Coverage for python/lsst/ip/isr/isrTask.py: 16%

892 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-19 04:27 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from lsst.daf.butler import DimensionGraph 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

75 Data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

95 

96 

97class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

98 dimensions={"instrument", "exposure", "detector"}, 

99 defaultTemplates={}): 

100 ccdExposure = cT.Input( 

101 name="raw", 

102 doc="Input exposure to process.", 

103 storageClass="Exposure", 

104 dimensions=["instrument", "exposure", "detector"], 

105 ) 

106 camera = cT.PrerequisiteInput( 

107 name="camera", 

108 storageClass="Camera", 

109 doc="Input camera to construct complete exposures.", 

110 dimensions=["instrument"], 

111 isCalibration=True, 

112 ) 

113 

114 crosstalk = cT.PrerequisiteInput( 

115 name="crosstalk", 

116 doc="Input crosstalk object", 

117 storageClass="CrosstalkCalib", 

118 dimensions=["instrument", "detector"], 

119 isCalibration=True, 

120 minimum=0, # can fall back to cameraGeom 

121 ) 

122 crosstalkSources = cT.PrerequisiteInput( 

123 name="isrOverscanCorrected", 

124 doc="Overscan corrected input images.", 

125 storageClass="Exposure", 

126 dimensions=["instrument", "exposure", "detector"], 

127 deferLoad=True, 

128 multiple=True, 

129 lookupFunction=crosstalkSourceLookup, 

130 minimum=0, # not needed for all instruments, no config to control this 

131 ) 

132 bias = cT.PrerequisiteInput( 

133 name="bias", 

134 doc="Input bias calibration.", 

135 storageClass="ExposureF", 

136 dimensions=["instrument", "detector"], 

137 isCalibration=True, 

138 ) 

139 dark = cT.PrerequisiteInput( 

140 name='dark', 

141 doc="Input dark calibration.", 

142 storageClass="ExposureF", 

143 dimensions=["instrument", "detector"], 

144 isCalibration=True, 

145 ) 

146 flat = cT.PrerequisiteInput( 

147 name="flat", 

148 doc="Input flat calibration.", 

149 storageClass="ExposureF", 

150 dimensions=["instrument", "physical_filter", "detector"], 

151 isCalibration=True, 

152 ) 

153 ptc = cT.PrerequisiteInput( 

154 name="ptc", 

155 doc="Input Photon Transfer Curve dataset", 

156 storageClass="PhotonTransferCurveDataset", 

157 dimensions=["instrument", "detector"], 

158 isCalibration=True, 

159 ) 

160 fringes = cT.PrerequisiteInput( 

161 name="fringe", 

162 doc="Input fringe calibration.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "physical_filter", "detector"], 

165 isCalibration=True, 

166 minimum=0, # only needed for some bands, even when enabled 

167 ) 

168 strayLightData = cT.PrerequisiteInput( 

169 name='yBackground', 

170 doc="Input stray light calibration.", 

171 storageClass="StrayLightData", 

172 dimensions=["instrument", "physical_filter", "detector"], 

173 deferLoad=True, 

174 isCalibration=True, 

175 minimum=0, # only needed for some bands, even when enabled 

176 ) 

177 bfKernel = cT.PrerequisiteInput( 

178 name='bfKernel', 

179 doc="Input brighter-fatter kernel.", 

180 storageClass="NumpyArray", 

181 dimensions=["instrument"], 

182 isCalibration=True, 

183 minimum=0, # can use either bfKernel or newBFKernel 

184 ) 

185 newBFKernel = cT.PrerequisiteInput( 

186 name='brighterFatterKernel', 

187 doc="Newer complete kernel + gain solutions.", 

188 storageClass="BrighterFatterKernel", 

189 dimensions=["instrument", "detector"], 

190 isCalibration=True, 

191 minimum=0, # can use either bfKernel or newBFKernel 

192 ) 

193 defects = cT.PrerequisiteInput( 

194 name='defects', 

195 doc="Input defect tables.", 

196 storageClass="Defects", 

197 dimensions=["instrument", "detector"], 

198 isCalibration=True, 

199 ) 

200 linearizer = cT.PrerequisiteInput( 

201 name='linearizer', 

202 storageClass="Linearizer", 

203 doc="Linearity correction calibration.", 

204 dimensions=["instrument", "detector"], 

205 isCalibration=True, 

206 minimum=0, # can fall back to cameraGeom 

207 ) 

208 opticsTransmission = cT.PrerequisiteInput( 

209 name="transmission_optics", 

210 storageClass="TransmissionCurve", 

211 doc="Transmission curve due to the optics.", 

212 dimensions=["instrument"], 

213 isCalibration=True, 

214 ) 

215 filterTransmission = cT.PrerequisiteInput( 

216 name="transmission_filter", 

217 storageClass="TransmissionCurve", 

218 doc="Transmission curve due to the filter.", 

219 dimensions=["instrument", "physical_filter"], 

220 isCalibration=True, 

221 ) 

222 sensorTransmission = cT.PrerequisiteInput( 

223 name="transmission_sensor", 

224 storageClass="TransmissionCurve", 

225 doc="Transmission curve due to the sensor.", 

226 dimensions=["instrument", "detector"], 

227 isCalibration=True, 

228 ) 

229 atmosphereTransmission = cT.PrerequisiteInput( 

230 name="transmission_atmosphere", 

231 storageClass="TransmissionCurve", 

232 doc="Transmission curve due to the atmosphere.", 

233 dimensions=["instrument"], 

234 isCalibration=True, 

235 ) 

236 illumMaskedImage = cT.PrerequisiteInput( 

237 name="illum", 

238 doc="Input illumination correction.", 

239 storageClass="MaskedImageF", 

240 dimensions=["instrument", "physical_filter", "detector"], 

241 isCalibration=True, 

242 ) 

243 deferredChargeCalib = cT.PrerequisiteInput( 

244 name="cpCtiCalib", 

245 doc="Deferred charge/CTI correction dataset.", 

246 storageClass="IsrCalib", 

247 dimensions=["instrument", "detector"], 

248 isCalibration=True, 

249 ) 

250 

251 outputExposure = cT.Output( 

252 name='postISRCCD', 

253 doc="Output ISR processed exposure.", 

254 storageClass="Exposure", 

255 dimensions=["instrument", "exposure", "detector"], 

256 ) 

257 preInterpExposure = cT.Output( 

258 name='preInterpISRCCD', 

259 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

260 storageClass="ExposureF", 

261 dimensions=["instrument", "exposure", "detector"], 

262 ) 

263 outputOssThumbnail = cT.Output( 

264 name="OssThumb", 

265 doc="Output Overscan-subtracted thumbnail image.", 

266 storageClass="Thumbnail", 

267 dimensions=["instrument", "exposure", "detector"], 

268 ) 

269 outputFlattenedThumbnail = cT.Output( 

270 name="FlattenedThumb", 

271 doc="Output flat-corrected thumbnail image.", 

272 storageClass="Thumbnail", 

273 dimensions=["instrument", "exposure", "detector"], 

274 ) 

275 outputStatistics = cT.Output( 

276 name="isrStatistics", 

277 doc="Output of additional statistics table.", 

278 storageClass="StructuredDataDict", 

279 dimensions=["instrument", "exposure", "detector"], 

280 ) 

281 

282 def __init__(self, *, config=None): 

283 super().__init__(config=config) 

284 

285 if config.doBias is not True: 

286 self.prerequisiteInputs.remove("bias") 

287 if config.doLinearize is not True: 

288 self.prerequisiteInputs.remove("linearizer") 

289 if config.doCrosstalk is not True: 

290 self.prerequisiteInputs.remove("crosstalkSources") 

291 self.prerequisiteInputs.remove("crosstalk") 

292 if config.doBrighterFatter is not True: 

293 self.prerequisiteInputs.remove("bfKernel") 

294 self.prerequisiteInputs.remove("newBFKernel") 

295 if config.doDefect is not True: 

296 self.prerequisiteInputs.remove("defects") 

297 if config.doDark is not True: 

298 self.prerequisiteInputs.remove("dark") 

299 if config.doFlat is not True: 

300 self.prerequisiteInputs.remove("flat") 

301 if config.doFringe is not True: 

302 self.prerequisiteInputs.remove("fringes") 

303 if config.doStrayLight is not True: 

304 self.prerequisiteInputs.remove("strayLightData") 

305 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

306 self.prerequisiteInputs.remove("ptc") 

307 if config.doAttachTransmissionCurve is not True: 

308 self.prerequisiteInputs.remove("opticsTransmission") 

309 self.prerequisiteInputs.remove("filterTransmission") 

310 self.prerequisiteInputs.remove("sensorTransmission") 

311 self.prerequisiteInputs.remove("atmosphereTransmission") 

312 else: 

313 if config.doUseOpticsTransmission is not True: 

314 self.prerequisiteInputs.remove("opticsTransmission") 

315 if config.doUseFilterTransmission is not True: 

316 self.prerequisiteInputs.remove("filterTransmission") 

317 if config.doUseSensorTransmission is not True: 

318 self.prerequisiteInputs.remove("sensorTransmission") 

319 if config.doUseAtmosphereTransmission is not True: 

320 self.prerequisiteInputs.remove("atmosphereTransmission") 

321 if config.doIlluminationCorrection is not True: 

322 self.prerequisiteInputs.remove("illumMaskedImage") 

323 if config.doDeferredCharge is not True: 

324 self.prerequisiteInputs.remove("deferredChargeCalib") 

325 

326 if config.doWrite is not True: 

327 self.outputs.remove("outputExposure") 

328 self.outputs.remove("preInterpExposure") 

329 self.outputs.remove("outputFlattenedThumbnail") 

330 self.outputs.remove("outputOssThumbnail") 

331 self.outputs.remove("outputStatistics") 

332 

333 if config.doSaveInterpPixels is not True: 

334 self.outputs.remove("preInterpExposure") 

335 if config.qa.doThumbnailOss is not True: 

336 self.outputs.remove("outputOssThumbnail") 

337 if config.qa.doThumbnailFlattened is not True: 

338 self.outputs.remove("outputFlattenedThumbnail") 

339 if config.doCalculateStatistics is not True: 

340 self.outputs.remove("outputStatistics") 

341 

342 

343class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

344 pipelineConnections=IsrTaskConnections): 

345 """Configuration parameters for IsrTask. 

346 

347 Items are grouped in the order in which they are executed by the task. 

348 """ 

349 datasetType = pexConfig.Field( 

350 dtype=str, 

351 doc="Dataset type for input data; users will typically leave this alone, " 

352 "but camera-specific ISR tasks will override it", 

353 default="raw", 

354 ) 

355 

356 fallbackFilterName = pexConfig.Field( 

357 dtype=str, 

358 doc="Fallback default filter name for calibrations.", 

359 optional=True 

360 ) 

361 useFallbackDate = pexConfig.Field( 

362 dtype=bool, 

363 doc="Pass observation date when using fallback filter.", 

364 default=False, 

365 ) 

366 expectWcs = pexConfig.Field( 

367 dtype=bool, 

368 default=True, 

369 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

370 ) 

371 fwhm = pexConfig.Field( 

372 dtype=float, 

373 doc="FWHM of PSF in arcseconds.", 

374 default=1.0, 

375 ) 

376 qa = pexConfig.ConfigField( 

377 dtype=isrQa.IsrQaConfig, 

378 doc="QA related configuration options.", 

379 ) 

380 doHeaderProvenance = pexConfig.Field( 

381 dtype=bool, 

382 default=True, 

383 doc="Write calibration identifiers into output exposure header?", 

384 ) 

385 

386 # Calib checking configuration: 

387 doRaiseOnCalibMismatch = pexConfig.Field( 

388 dtype=bool, 

389 default=False, 

390 doc="Should IsrTask halt if exposure and calibration header values do not match?", 

391 ) 

392 cameraKeywordsToCompare = pexConfig.ListField( 

393 dtype=str, 

394 doc="List of header keywords to compare between exposure and calibrations.", 

395 default=[], 

396 ) 

397 

398 # Image conversion configuration 

399 doConvertIntToFloat = pexConfig.Field( 

400 dtype=bool, 

401 doc="Convert integer raw images to floating point values?", 

402 default=True, 

403 ) 

404 

405 # Saturated pixel handling. 

406 doSaturation = pexConfig.Field( 

407 dtype=bool, 

408 doc="Mask saturated pixels? NB: this is totally independent of the" 

409 " interpolation option - this is ONLY setting the bits in the mask." 

410 " To have them interpolated make sure doSaturationInterpolation=True", 

411 default=True, 

412 ) 

413 saturatedMaskName = pexConfig.Field( 

414 dtype=str, 

415 doc="Name of mask plane to use in saturation detection and interpolation", 

416 default="SAT", 

417 ) 

418 saturation = pexConfig.Field( 

419 dtype=float, 

420 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

421 default=float("NaN"), 

422 ) 

423 growSaturationFootprintSize = pexConfig.Field( 

424 dtype=int, 

425 doc="Number of pixels by which to grow the saturation footprints", 

426 default=1, 

427 ) 

428 

429 # Suspect pixel handling. 

430 doSuspect = pexConfig.Field( 

431 dtype=bool, 

432 doc="Mask suspect pixels?", 

433 default=False, 

434 ) 

435 suspectMaskName = pexConfig.Field( 

436 dtype=str, 

437 doc="Name of mask plane to use for suspect pixels", 

438 default="SUSPECT", 

439 ) 

440 numEdgeSuspect = pexConfig.Field( 

441 dtype=int, 

442 doc="Number of edge pixels to be flagged as untrustworthy.", 

443 default=0, 

444 ) 

445 edgeMaskLevel = pexConfig.ChoiceField( 

446 dtype=str, 

447 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

448 default="DETECTOR", 

449 allowed={ 

450 'DETECTOR': 'Mask only the edges of the full detector.', 

451 'AMP': 'Mask edges of each amplifier.', 

452 }, 

453 ) 

454 

455 # Initial masking options. 

456 doSetBadRegions = pexConfig.Field( 

457 dtype=bool, 

458 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

459 default=True, 

460 ) 

461 badStatistic = pexConfig.ChoiceField( 

462 dtype=str, 

463 doc="How to estimate the average value for BAD regions.", 

464 default='MEANCLIP', 

465 allowed={ 

466 "MEANCLIP": "Correct using the (clipped) mean of good data", 

467 "MEDIAN": "Correct using the median of the good data", 

468 }, 

469 ) 

470 

471 # Overscan subtraction configuration. 

472 doOverscan = pexConfig.Field( 

473 dtype=bool, 

474 doc="Do overscan subtraction?", 

475 default=True, 

476 ) 

477 overscan = pexConfig.ConfigurableField( 

478 target=OverscanCorrectionTask, 

479 doc="Overscan subtraction task for image segments.", 

480 ) 

481 

482 # Amplifier to CCD assembly configuration 

483 doAssembleCcd = pexConfig.Field( 

484 dtype=bool, 

485 default=True, 

486 doc="Assemble amp-level exposures into a ccd-level exposure?" 

487 ) 

488 assembleCcd = pexConfig.ConfigurableField( 

489 target=AssembleCcdTask, 

490 doc="CCD assembly task", 

491 ) 

492 

493 # General calibration configuration. 

494 doAssembleIsrExposures = pexConfig.Field( 

495 dtype=bool, 

496 default=False, 

497 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

498 ) 

499 doTrimToMatchCalib = pexConfig.Field( 

500 dtype=bool, 

501 default=False, 

502 doc="Trim raw data to match calibration bounding boxes?" 

503 ) 

504 

505 # Bias subtraction. 

506 doBias = pexConfig.Field( 

507 dtype=bool, 

508 doc="Apply bias frame correction?", 

509 default=True, 

510 ) 

511 biasDataProductName = pexConfig.Field( 

512 dtype=str, 

513 doc="Name of the bias data product", 

514 default="bias", 

515 ) 

516 doBiasBeforeOverscan = pexConfig.Field( 

517 dtype=bool, 

518 doc="Reverse order of overscan and bias correction.", 

519 default=False 

520 ) 

521 

522 # Deferred charge correction. 

523 doDeferredCharge = pexConfig.Field( 

524 dtype=bool, 

525 doc="Apply deferred charge correction?", 

526 default=False, 

527 ) 

528 deferredChargeCorrection = pexConfig.ConfigurableField( 

529 target=DeferredChargeTask, 

530 doc="Deferred charge correction task.", 

531 ) 

532 

533 # Variance construction 

534 doVariance = pexConfig.Field( 

535 dtype=bool, 

536 doc="Calculate variance?", 

537 default=True 

538 ) 

539 gain = pexConfig.Field( 

540 dtype=float, 

541 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

542 default=float("NaN"), 

543 ) 

544 readNoise = pexConfig.Field( 

545 dtype=float, 

546 doc="The read noise to use if no Detector is present in the Exposure", 

547 default=0.0, 

548 ) 

549 doEmpiricalReadNoise = pexConfig.Field( 

550 dtype=bool, 

551 default=False, 

552 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

553 ) 

554 usePtcReadNoise = pexConfig.Field( 

555 dtype=bool, 

556 default=False, 

557 doc="Use readnoise values from the Photon Transfer Curve?" 

558 ) 

559 maskNegativeVariance = pexConfig.Field( 

560 dtype=bool, 

561 default=True, 

562 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

563 "in the measurement of the overscan at an edge due to the data falling off faster " 

564 "than the overscan model can account for it." 

565 ) 

566 negativeVarianceMaskName = pexConfig.Field( 

567 dtype=str, 

568 default="BAD", 

569 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

570 ) 

571 # Linearization. 

572 doLinearize = pexConfig.Field( 

573 dtype=bool, 

574 doc="Correct for nonlinearity of the detector's response?", 

575 default=True, 

576 ) 

577 

578 # Crosstalk. 

579 doCrosstalk = pexConfig.Field( 

580 dtype=bool, 

581 doc="Apply intra-CCD crosstalk correction?", 

582 default=False, 

583 ) 

584 doCrosstalkBeforeAssemble = pexConfig.Field( 

585 dtype=bool, 

586 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

587 default=False, 

588 ) 

589 crosstalk = pexConfig.ConfigurableField( 

590 target=CrosstalkTask, 

591 doc="Intra-CCD crosstalk correction", 

592 ) 

593 

594 # Masking options. 

595 doDefect = pexConfig.Field( 

596 dtype=bool, 

597 doc="Apply correction for CCD defects, e.g. hot pixels?", 

598 default=True, 

599 ) 

600 doNanMasking = pexConfig.Field( 

601 dtype=bool, 

602 doc="Mask non-finite (NAN, inf) pixels?", 

603 default=True, 

604 ) 

605 doWidenSaturationTrails = pexConfig.Field( 

606 dtype=bool, 

607 doc="Widen bleed trails based on their width?", 

608 default=True 

609 ) 

610 

611 # Brighter-Fatter correction. 

612 doBrighterFatter = pexConfig.Field( 

613 dtype=bool, 

614 default=False, 

615 doc="Apply the brighter-fatter correction?" 

616 ) 

617 brighterFatterLevel = pexConfig.ChoiceField( 

618 dtype=str, 

619 default="DETECTOR", 

620 doc="The level at which to correct for brighter-fatter.", 

621 allowed={ 

622 "AMP": "Every amplifier treated separately.", 

623 "DETECTOR": "One kernel per detector", 

624 } 

625 ) 

626 brighterFatterMaxIter = pexConfig.Field( 

627 dtype=int, 

628 default=10, 

629 doc="Maximum number of iterations for the brighter-fatter correction" 

630 ) 

631 brighterFatterThreshold = pexConfig.Field( 

632 dtype=float, 

633 default=1000, 

634 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

635 "absolute value of the difference between the current corrected image and the one " 

636 "from the previous iteration summed over all the pixels." 

637 ) 

638 brighterFatterApplyGain = pexConfig.Field( 

639 dtype=bool, 

640 default=True, 

641 doc="Should the gain be applied when applying the brighter-fatter correction?" 

642 ) 

643 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

644 dtype=str, 

645 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

646 "correction.", 

647 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

648 ) 

649 brighterFatterMaskGrowSize = pexConfig.Field( 

650 dtype=int, 

651 default=0, 

652 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

653 "when brighter-fatter correction is applied." 

654 ) 

655 

656 # Dark subtraction. 

657 doDark = pexConfig.Field( 

658 dtype=bool, 

659 doc="Apply dark frame correction?", 

660 default=True, 

661 ) 

662 darkDataProductName = pexConfig.Field( 

663 dtype=str, 

664 doc="Name of the dark data product", 

665 default="dark", 

666 ) 

667 

668 # Camera-specific stray light removal. 

669 doStrayLight = pexConfig.Field( 

670 dtype=bool, 

671 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

672 default=False, 

673 ) 

674 strayLight = pexConfig.ConfigurableField( 

675 target=StrayLightTask, 

676 doc="y-band stray light correction" 

677 ) 

678 

679 # Flat correction. 

680 doFlat = pexConfig.Field( 

681 dtype=bool, 

682 doc="Apply flat field correction?", 

683 default=True, 

684 ) 

685 flatDataProductName = pexConfig.Field( 

686 dtype=str, 

687 doc="Name of the flat data product", 

688 default="flat", 

689 ) 

690 flatScalingType = pexConfig.ChoiceField( 

691 dtype=str, 

692 doc="The method for scaling the flat on the fly.", 

693 default='USER', 

694 allowed={ 

695 "USER": "Scale by flatUserScale", 

696 "MEAN": "Scale by the inverse of the mean", 

697 "MEDIAN": "Scale by the inverse of the median", 

698 }, 

699 ) 

700 flatUserScale = pexConfig.Field( 

701 dtype=float, 

702 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

703 default=1.0, 

704 ) 

705 doTweakFlat = pexConfig.Field( 

706 dtype=bool, 

707 doc="Tweak flats to match observed amplifier ratios?", 

708 default=False 

709 ) 

710 

711 # Amplifier normalization based on gains instead of using flats 

712 # configuration. 

713 doApplyGains = pexConfig.Field( 

714 dtype=bool, 

715 doc="Correct the amplifiers for their gains instead of applying flat correction", 

716 default=False, 

717 ) 

718 usePtcGains = pexConfig.Field( 

719 dtype=bool, 

720 doc="Use the gain values from the Photon Transfer Curve?", 

721 default=False, 

722 ) 

723 normalizeGains = pexConfig.Field( 

724 dtype=bool, 

725 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

726 default=False, 

727 ) 

728 

729 # Fringe correction. 

730 doFringe = pexConfig.Field( 

731 dtype=bool, 

732 doc="Apply fringe correction?", 

733 default=True, 

734 ) 

735 fringe = pexConfig.ConfigurableField( 

736 target=FringeTask, 

737 doc="Fringe subtraction task", 

738 ) 

739 fringeAfterFlat = pexConfig.Field( 

740 dtype=bool, 

741 doc="Do fringe subtraction after flat-fielding?", 

742 default=True, 

743 ) 

744 

745 # Amp offset correction. 

746 doAmpOffset = pexConfig.Field( 

747 doc="Calculate and apply amp offset corrections?", 

748 dtype=bool, 

749 default=False, 

750 ) 

751 ampOffset = pexConfig.ConfigurableField( 

752 doc="Amp offset correction task.", 

753 target=AmpOffsetTask, 

754 ) 

755 

756 # Initial CCD-level background statistics options. 

757 doMeasureBackground = pexConfig.Field( 

758 dtype=bool, 

759 doc="Measure the background level on the reduced image?", 

760 default=False, 

761 ) 

762 

763 # Camera-specific masking configuration. 

764 doCameraSpecificMasking = pexConfig.Field( 

765 dtype=bool, 

766 doc="Mask camera-specific bad regions?", 

767 default=False, 

768 ) 

769 masking = pexConfig.ConfigurableField( 

770 target=MaskingTask, 

771 doc="Masking task." 

772 ) 

773 

774 # Interpolation options. 

775 doInterpolate = pexConfig.Field( 

776 dtype=bool, 

777 doc="Interpolate masked pixels?", 

778 default=True, 

779 ) 

780 doSaturationInterpolation = pexConfig.Field( 

781 dtype=bool, 

782 doc="Perform interpolation over pixels masked as saturated?" 

783 " NB: This is independent of doSaturation; if that is False this plane" 

784 " will likely be blank, resulting in a no-op here.", 

785 default=True, 

786 ) 

787 doNanInterpolation = pexConfig.Field( 

788 dtype=bool, 

789 doc="Perform interpolation over pixels masked as NaN?" 

790 " NB: This is independent of doNanMasking; if that is False this plane" 

791 " will likely be blank, resulting in a no-op here.", 

792 default=True, 

793 ) 

794 doNanInterpAfterFlat = pexConfig.Field( 

795 dtype=bool, 

796 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

797 "also have to interpolate them before flat-fielding."), 

798 default=False, 

799 ) 

800 maskListToInterpolate = pexConfig.ListField( 

801 dtype=str, 

802 doc="List of mask planes that should be interpolated.", 

803 default=['SAT', 'BAD'], 

804 ) 

805 doSaveInterpPixels = pexConfig.Field( 

806 dtype=bool, 

807 doc="Save a copy of the pre-interpolated pixel values?", 

808 default=False, 

809 ) 

810 

811 # Default photometric calibration options. 

812 fluxMag0T1 = pexConfig.DictField( 

813 keytype=str, 

814 itemtype=float, 

815 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

816 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

817 )) 

818 ) 

819 defaultFluxMag0T1 = pexConfig.Field( 

820 dtype=float, 

821 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

822 default=pow(10.0, 0.4*28.0) 

823 ) 

824 

825 # Vignette correction configuration. 

826 doVignette = pexConfig.Field( 

827 dtype=bool, 

828 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

829 "according to vignetting parameters?"), 

830 default=False, 

831 ) 

832 doMaskVignettePolygon = pexConfig.Field( 

833 dtype=bool, 

834 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

835 "is False"), 

836 default=True, 

837 ) 

838 vignetteValue = pexConfig.Field( 

839 dtype=float, 

840 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

841 optional=True, 

842 default=None, 

843 ) 

844 vignette = pexConfig.ConfigurableField( 

845 target=VignetteTask, 

846 doc="Vignetting task.", 

847 ) 

848 

849 # Transmission curve configuration. 

850 doAttachTransmissionCurve = pexConfig.Field( 

851 dtype=bool, 

852 default=False, 

853 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

854 ) 

855 doUseOpticsTransmission = pexConfig.Field( 

856 dtype=bool, 

857 default=True, 

858 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

859 ) 

860 doUseFilterTransmission = pexConfig.Field( 

861 dtype=bool, 

862 default=True, 

863 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

864 ) 

865 doUseSensorTransmission = pexConfig.Field( 

866 dtype=bool, 

867 default=True, 

868 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

869 ) 

870 doUseAtmosphereTransmission = pexConfig.Field( 

871 dtype=bool, 

872 default=True, 

873 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

874 ) 

875 

876 # Illumination correction. 

877 doIlluminationCorrection = pexConfig.Field( 

878 dtype=bool, 

879 default=False, 

880 doc="Perform illumination correction?" 

881 ) 

882 illuminationCorrectionDataProductName = pexConfig.Field( 

883 dtype=str, 

884 doc="Name of the illumination correction data product.", 

885 default="illumcor", 

886 ) 

887 illumScale = pexConfig.Field( 

888 dtype=float, 

889 doc="Scale factor for the illumination correction.", 

890 default=1.0, 

891 ) 

892 illumFilters = pexConfig.ListField( 

893 dtype=str, 

894 default=[], 

895 doc="Only perform illumination correction for these filters." 

896 ) 

897 

898 # Calculate image quality statistics? 

899 doStandardStatistics = pexConfig.Field( 

900 dtype=bool, 

901 doc="Should standard image quality statistics be calculated?", 

902 default=True, 

903 ) 

904 # Calculate additional statistics? 

905 doCalculateStatistics = pexConfig.Field( 

906 dtype=bool, 

907 doc="Should additional ISR statistics be calculated?", 

908 default=False, 

909 ) 

910 isrStats = pexConfig.ConfigurableField( 

911 target=IsrStatisticsTask, 

912 doc="Task to calculate additional statistics.", 

913 ) 

914 

915 # Write the outputs to disk. If ISR is run as a subtask, this may not 

916 # be needed. 

917 doWrite = pexConfig.Field( 

918 dtype=bool, 

919 doc="Persist postISRCCD?", 

920 default=True, 

921 ) 

922 

923 def validate(self): 

924 super().validate() 

925 if self.doFlat and self.doApplyGains: 

926 raise ValueError("You may not specify both doFlat and doApplyGains") 

927 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

928 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

929 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

930 self.maskListToInterpolate.append(self.saturatedMaskName) 

931 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

932 self.maskListToInterpolate.remove(self.saturatedMaskName) 

933 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

934 self.maskListToInterpolate.append("UNMASKEDNAN") 

935 

936 

937class IsrTask(pipeBase.PipelineTask): 

938 """Apply common instrument signature correction algorithms to a raw frame. 

939 

940 The process for correcting imaging data is very similar from 

941 camera to camera. This task provides a vanilla implementation of 

942 doing these corrections, including the ability to turn certain 

943 corrections off if they are not needed. The inputs to the primary 

944 method, `run()`, are a raw exposure to be corrected and the 

945 calibration data products. The raw input is a single chip sized 

946 mosaic of all amps including overscans and other non-science 

947 pixels. 

948 

949 The __init__ method sets up the subtasks for ISR processing, using 

950 the defaults from `lsst.ip.isr`. 

951 

952 Parameters 

953 ---------- 

954 args : `list` 

955 Positional arguments passed to the Task constructor. 

956 None used at this time. 

957 kwargs : `dict`, optional 

958 Keyword arguments passed on to the Task constructor. 

959 None used at this time. 

960 """ 

961 ConfigClass = IsrTaskConfig 

962 _DefaultName = "isr" 

963 

964 def __init__(self, **kwargs): 

965 super().__init__(**kwargs) 

966 self.makeSubtask("assembleCcd") 

967 self.makeSubtask("crosstalk") 

968 self.makeSubtask("strayLight") 

969 self.makeSubtask("fringe") 

970 self.makeSubtask("masking") 

971 self.makeSubtask("overscan") 

972 self.makeSubtask("vignette") 

973 self.makeSubtask("ampOffset") 

974 self.makeSubtask("deferredChargeCorrection") 

975 self.makeSubtask("isrStats") 

976 

977 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

978 inputs = butlerQC.get(inputRefs) 

979 

980 try: 

981 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

982 except Exception as e: 

983 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

984 (inputRefs, e)) 

985 

986 detector = inputs['ccdExposure'].getDetector() 

987 

988 if self.config.doCrosstalk is True: 

989 # Crosstalk sources need to be defined by the pipeline 

990 # yaml if they exist. 

991 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

992 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

993 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

994 else: 

995 coeffVector = (self.config.crosstalk.crosstalkValues 

996 if self.config.crosstalk.useConfigCoefficients else None) 

997 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

998 inputs['crosstalk'] = crosstalkCalib 

999 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1000 if 'crosstalkSources' not in inputs: 

1001 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1002 

1003 if self.doLinearize(detector) is True: 

1004 if 'linearizer' in inputs: 

1005 if isinstance(inputs['linearizer'], dict): 

1006 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1007 linearizer.fromYaml(inputs['linearizer']) 

1008 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1009 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1010 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1011 detector=detector, 

1012 log=self.log) 

1013 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1014 else: 

1015 linearizer = inputs['linearizer'] 

1016 linearizer.log = self.log 

1017 inputs['linearizer'] = linearizer 

1018 else: 

1019 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1020 self.log.warning("Constructing linearizer from cameraGeom information.") 

1021 

1022 if self.config.doDefect is True: 

1023 if "defects" in inputs and inputs['defects'] is not None: 

1024 # defects is loaded as a BaseCatalog with columns 

1025 # x0, y0, width, height. Masking expects a list of defects 

1026 # defined by their bounding box 

1027 if not isinstance(inputs["defects"], Defects): 

1028 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1029 

1030 # Load the correct style of brighter-fatter kernel, and repack 

1031 # the information as a numpy array. 

1032 if self.config.doBrighterFatter: 

1033 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1034 if brighterFatterKernel is None: 

1035 brighterFatterKernel = inputs.get('bfKernel', None) 

1036 

1037 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1038 # This is a ISR calib kernel 

1039 detName = detector.getName() 

1040 level = brighterFatterKernel.level 

1041 

1042 # This is expected to be a dictionary of amp-wise gains. 

1043 inputs['bfGains'] = brighterFatterKernel.gain 

1044 if self.config.brighterFatterLevel == 'DETECTOR': 

1045 if level == 'DETECTOR': 

1046 if detName in brighterFatterKernel.detKernels: 

1047 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1048 else: 

1049 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1050 elif level == 'AMP': 

1051 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1052 "fatter kernels.") 

1053 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1054 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1055 elif self.config.brighterFatterLevel == 'AMP': 

1056 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1057 

1058 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1059 expId = inputs['ccdExposure'].info.id 

1060 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1061 expId=expId, 

1062 assembler=self.assembleCcd 

1063 if self.config.doAssembleIsrExposures else None) 

1064 else: 

1065 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1066 

1067 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1068 if 'strayLightData' not in inputs: 

1069 inputs['strayLightData'] = None 

1070 

1071 if self.config.doHeaderProvenance: 

1072 # Add calibration provenanace info to header. 

1073 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1074 for inputName in sorted(inputs.keys()): 

1075 reference = getattr(inputRefs, inputName, None) 

1076 if reference is not None and hasattr(reference, "run"): 

1077 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1078 runValue = reference.run 

1079 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1080 idValue = str(reference.id) 

1081 

1082 exposureMetadata[runKey] = runValue 

1083 exposureMetadata[idKey] = idValue 

1084 

1085 outputs = self.run(**inputs) 

1086 butlerQC.put(outputs, outputRefs) 

1087 

1088 @timeMethod 

1089 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1090 crosstalk=None, crosstalkSources=None, 

1091 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1092 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1093 sensorTransmission=None, atmosphereTransmission=None, 

1094 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1095 deferredChargeCalib=None, 

1096 ): 

1097 """Perform instrument signature removal on an exposure. 

1098 

1099 Steps included in the ISR processing, in order performed, are: 

1100 

1101 - saturation and suspect pixel masking 

1102 - overscan subtraction 

1103 - CCD assembly of individual amplifiers 

1104 - bias subtraction 

1105 - variance image construction 

1106 - linearization of non-linear response 

1107 - crosstalk masking 

1108 - brighter-fatter correction 

1109 - dark subtraction 

1110 - fringe correction 

1111 - stray light subtraction 

1112 - flat correction 

1113 - masking of known defects and camera specific features 

1114 - vignette calculation 

1115 - appending transmission curve and distortion model 

1116 

1117 Parameters 

1118 ---------- 

1119 ccdExposure : `lsst.afw.image.Exposure` 

1120 The raw exposure that is to be run through ISR. The 

1121 exposure is modified by this method. 

1122 camera : `lsst.afw.cameraGeom.Camera`, optional 

1123 The camera geometry for this exposure. Required if 

1124 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1125 ``flat`` does not have an associated detector. 

1126 bias : `lsst.afw.image.Exposure`, optional 

1127 Bias calibration frame. 

1128 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1129 Functor for linearization. 

1130 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1131 Calibration for crosstalk. 

1132 crosstalkSources : `list`, optional 

1133 List of possible crosstalk sources. 

1134 dark : `lsst.afw.image.Exposure`, optional 

1135 Dark calibration frame. 

1136 flat : `lsst.afw.image.Exposure`, optional 

1137 Flat calibration frame. 

1138 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1139 Photon transfer curve dataset, with, e.g., gains 

1140 and read noise. 

1141 bfKernel : `numpy.ndarray`, optional 

1142 Brighter-fatter kernel. 

1143 bfGains : `dict` of `float`, optional 

1144 Gains used to override the detector's nominal gains for the 

1145 brighter-fatter correction. A dict keyed by amplifier name for 

1146 the detector in question. 

1147 defects : `lsst.ip.isr.Defects`, optional 

1148 List of defects. 

1149 fringes : `lsst.pipe.base.Struct`, optional 

1150 Struct containing the fringe correction data, with 

1151 elements: 

1152 

1153 ``fringes`` 

1154 fringe calibration frame (`lsst.afw.image.Exposure`) 

1155 ``seed`` 

1156 random seed derived from the ``ccdExposureId`` for random 

1157 number generator (`numpy.uint32`) 

1158 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1159 A ``TransmissionCurve`` that represents the throughput of the, 

1160 optics, to be evaluated in focal-plane coordinates. 

1161 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1162 A ``TransmissionCurve`` that represents the throughput of the 

1163 filter itself, to be evaluated in focal-plane coordinates. 

1164 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1165 A ``TransmissionCurve`` that represents the throughput of the 

1166 sensor itself, to be evaluated in post-assembly trimmed detector 

1167 coordinates. 

1168 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1169 A ``TransmissionCurve`` that represents the throughput of the 

1170 atmosphere, assumed to be spatially constant. 

1171 detectorNum : `int`, optional 

1172 The integer number for the detector to process. 

1173 strayLightData : `object`, optional 

1174 Opaque object containing calibration information for stray-light 

1175 correction. If `None`, no correction will be performed. 

1176 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1177 Illumination correction image. 

1178 

1179 Returns 

1180 ------- 

1181 result : `lsst.pipe.base.Struct` 

1182 Result struct with component: 

1183 

1184 ``exposure`` 

1185 The fully ISR corrected exposure. 

1186 (`lsst.afw.image.Exposure`) 

1187 ``outputExposure`` 

1188 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1189 ``ossThumb`` 

1190 Thumbnail image of the exposure after overscan subtraction. 

1191 (`numpy.ndarray`) 

1192 ``flattenedThumb`` 

1193 Thumbnail image of the exposure after flat-field correction. 

1194 (`numpy.ndarray`) 

1195 ``outputStatistics`` 

1196 Values of the additional statistics calculated. 

1197 

1198 Raises 

1199 ------ 

1200 RuntimeError 

1201 Raised if a configuration option is set to `True`, but the 

1202 required calibration data has not been specified. 

1203 

1204 Notes 

1205 ----- 

1206 The current processed exposure can be viewed by setting the 

1207 appropriate `lsstDebug` entries in the ``debug.display`` 

1208 dictionary. The names of these entries correspond to some of 

1209 the `IsrTaskConfig` Boolean options, with the value denoting the 

1210 frame to use. The exposure is shown inside the matching 

1211 option check and after the processing of that step has 

1212 finished. The steps with debug points are: 

1213 

1214 * doAssembleCcd 

1215 * doBias 

1216 * doCrosstalk 

1217 * doBrighterFatter 

1218 * doDark 

1219 * doFringe 

1220 * doStrayLight 

1221 * doFlat 

1222 

1223 In addition, setting the ``postISRCCD`` entry displays the 

1224 exposure after all ISR processing has finished. 

1225 """ 

1226 

1227 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1228 bias = self.ensureExposure(bias, camera, detectorNum) 

1229 dark = self.ensureExposure(dark, camera, detectorNum) 

1230 flat = self.ensureExposure(flat, camera, detectorNum) 

1231 

1232 ccd = ccdExposure.getDetector() 

1233 filterLabel = ccdExposure.getFilter() 

1234 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1235 

1236 if not ccd: 

1237 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1238 ccd = [FakeAmp(ccdExposure, self.config)] 

1239 

1240 # Validate Input 

1241 if self.config.doBias and bias is None: 

1242 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1243 if self.doLinearize(ccd) and linearizer is None: 

1244 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1245 if self.config.doBrighterFatter and bfKernel is None: 

1246 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1247 if self.config.doDark and dark is None: 

1248 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1249 if self.config.doFlat and flat is None: 

1250 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1251 if self.config.doDefect and defects is None: 

1252 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1253 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1254 and fringes.fringes is None): 

1255 # The `fringes` object needs to be a pipeBase.Struct, as 

1256 # we use it as a `dict` for the parameters of 

1257 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1258 # not be `None` if `doFringe=True`. Otherwise, raise. 

1259 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1260 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1261 and illumMaskedImage is None): 

1262 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1263 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1264 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1265 

1266 if self.config.doHeaderProvenance: 

1267 # Inputs have been validated, so we can add their date 

1268 # information to the output header. 

1269 exposureMetadata = ccdExposure.getMetadata() 

1270 if self.config.doBias: 

1271 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias) 

1272 self.compareCameraKeywords(exposureMetadata, bias, "bias") 

1273 if self.config.doBrighterFatter: 

1274 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel) 

1275 self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter") 

1276 if self.config.doCrosstalk: 

1277 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk) 

1278 self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk") 

1279 if self.config.doDark: 

1280 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark) 

1281 self.compareCameraKeywords(exposureMetadata, dark, "dark") 

1282 if self.config.doDefect: 

1283 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects) 

1284 self.compareCameraKeywords(exposureMetadata, defects, "defects") 

1285 if self.config.doDeferredCharge: 

1286 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib) 

1287 self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI") 

1288 if self.config.doFlat: 

1289 exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat) 

1290 self.compareCameraKeywords(exposureMetadata, flat, "flat") 

1291 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1292 exposureMetadata["LSST CALIB DATE FRINGE"] = self.extractCalibDate(fringes.fringes) 

1293 self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe") 

1294 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1295 exposureMetadata["LSST CALIB DATE ILLUMINATION"] = self.extractCalibDate(illumMaskedImage) 

1296 self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination") 

1297 if self.doLinearize(ccd): 

1298 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer) 

1299 self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer") 

1300 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1301 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc) 

1302 self.compareCameraKeywords(exposureMetadata, ptc, "PTC") 

1303 if self.config.doStrayLight: 

1304 exposureMetadata["LSST CALIB DATE STRAYLIGHT"] = self.extractCalibDate(strayLightData) 

1305 self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight") 

1306 if self.config.doAttachTransmissionCurve: 

1307 exposureMetadata["LSST CALIB DATE OPTICS_TR"] = self.extractCalibDate(opticsTransmission) 

1308 exposureMetadata["LSST CALIB DATE FILTER_TR"] = self.extractCalibDate(filterTransmission) 

1309 exposureMetadata["LSST CALIB DATE SENSOR_TR"] = self.extractCalibDate(sensorTransmission) 

1310 exposureMetadata["LSST CALIB DATE ATMOSP_TR"] = self.extractCalibDate(atmosphereTransmission) 

1311 

1312 # Begin ISR processing. 

1313 if self.config.doConvertIntToFloat: 

1314 self.log.info("Converting exposure to floating point values.") 

1315 ccdExposure = self.convertIntToFloat(ccdExposure) 

1316 

1317 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1318 self.log.info("Applying bias correction.") 

1319 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1320 trimToFit=self.config.doTrimToMatchCalib) 

1321 self.debugView(ccdExposure, "doBias") 

1322 

1323 # Amplifier level processing. 

1324 overscans = [] 

1325 

1326 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1327 # This will attempt to mask bleed pixels across all amplifiers. 

1328 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1329 

1330 for amp in ccd: 

1331 # if ccdExposure is one amp, 

1332 # check for coverage to prevent performing ops multiple times 

1333 if ccdExposure.getBBox().contains(amp.getBBox()): 

1334 # Check for fully masked bad amplifiers, 

1335 # and generate masks for SUSPECT and SATURATED values. 

1336 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1337 

1338 if self.config.doOverscan and not badAmp: 

1339 # Overscan correction on amp-by-amp basis. 

1340 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1341 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1342 if overscanResults is not None and \ 

1343 self.config.qa is not None and self.config.qa.saveStats is True: 

1344 if isinstance(overscanResults.overscanMean, float): 

1345 # Only serial overscan was run 

1346 mean = overscanResults.overscanMean 

1347 sigma = overscanResults.overscanSigma 

1348 residMean = overscanResults.residualMean 

1349 residSigma = overscanResults.residualSigma 

1350 else: 

1351 # Both serial and parallel overscan were 

1352 # run. Only report serial here. 

1353 mean = overscanResults.overscanMean[0] 

1354 sigma = overscanResults.overscanSigma[0] 

1355 residMean = overscanResults.residualMean[0] 

1356 residSigma = overscanResults.residualSigma[0] 

1357 

1358 self.metadata[f"FIT MEDIAN {amp.getName()}"] = mean 

1359 self.metadata[f"FIT STDEV {amp.getName()}"] = sigma 

1360 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1361 amp.getName(), mean, sigma) 

1362 

1363 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = residMean 

1364 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = residSigma 

1365 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1366 amp.getName(), residMean, residSigma) 

1367 

1368 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1369 else: 

1370 if badAmp: 

1371 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1372 overscanResults = None 

1373 

1374 overscans.append(overscanResults if overscanResults is not None else None) 

1375 else: 

1376 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1377 

1378 if self.config.doDeferredCharge: 

1379 self.log.info("Applying deferred charge/CTI correction.") 

1380 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1381 self.debugView(ccdExposure, "doDeferredCharge") 

1382 

1383 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1384 self.log.info("Applying crosstalk correction.") 

1385 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1386 crosstalkSources=crosstalkSources, camera=camera) 

1387 self.debugView(ccdExposure, "doCrosstalk") 

1388 

1389 if self.config.doAssembleCcd: 

1390 self.log.info("Assembling CCD from amplifiers.") 

1391 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1392 

1393 if self.config.expectWcs and not ccdExposure.getWcs(): 

1394 self.log.warning("No WCS found in input exposure.") 

1395 self.debugView(ccdExposure, "doAssembleCcd") 

1396 

1397 ossThumb = None 

1398 if self.config.qa.doThumbnailOss: 

1399 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1400 

1401 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1402 self.log.info("Applying bias correction.") 

1403 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1404 trimToFit=self.config.doTrimToMatchCalib) 

1405 self.debugView(ccdExposure, "doBias") 

1406 

1407 if self.config.doVariance: 

1408 for amp, overscanResults in zip(ccd, overscans): 

1409 if ccdExposure.getBBox().contains(amp.getBBox()): 

1410 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1411 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1412 if overscanResults is not None: 

1413 self.updateVariance(ampExposure, amp, 

1414 overscanImage=overscanResults.overscanImage, 

1415 ptcDataset=ptc) 

1416 else: 

1417 self.updateVariance(ampExposure, amp, 

1418 overscanImage=None, 

1419 ptcDataset=ptc) 

1420 if self.config.qa is not None and self.config.qa.saveStats is True: 

1421 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1422 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1423 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1424 qaStats.getValue(afwMath.MEDIAN) 

1425 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1426 qaStats.getValue(afwMath.STDEVCLIP) 

1427 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1428 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1429 qaStats.getValue(afwMath.STDEVCLIP)) 

1430 if self.config.maskNegativeVariance: 

1431 self.maskNegativeVariance(ccdExposure) 

1432 

1433 if self.doLinearize(ccd): 

1434 self.log.info("Applying linearizer.") 

1435 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1436 detector=ccd, log=self.log) 

1437 

1438 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1439 self.log.info("Applying crosstalk correction.") 

1440 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1441 crosstalkSources=crosstalkSources, isTrimmed=True) 

1442 self.debugView(ccdExposure, "doCrosstalk") 

1443 

1444 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1445 # widen trails, and do anything else the camera needs. Saturated and 

1446 # suspect pixels have already been masked. 

1447 if self.config.doDefect: 

1448 self.log.info("Masking defects.") 

1449 self.maskDefect(ccdExposure, defects) 

1450 

1451 if self.config.numEdgeSuspect > 0: 

1452 self.log.info("Masking edges as SUSPECT.") 

1453 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1454 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1455 

1456 if self.config.doNanMasking: 

1457 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1458 self.maskNan(ccdExposure) 

1459 

1460 if self.config.doWidenSaturationTrails: 

1461 self.log.info("Widening saturation trails.") 

1462 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1463 

1464 if self.config.doCameraSpecificMasking: 

1465 self.log.info("Masking regions for camera specific reasons.") 

1466 self.masking.run(ccdExposure) 

1467 

1468 if self.config.doBrighterFatter: 

1469 # We need to apply flats and darks before we can interpolate, and 

1470 # we need to interpolate before we do B-F, but we do B-F without 

1471 # the flats and darks applied so we can work in units of electrons 

1472 # or holes. This context manager applies and then removes the darks 

1473 # and flats. 

1474 # 

1475 # We also do not want to interpolate values here, so operate on 

1476 # temporary images so we can apply only the BF-correction and roll 

1477 # back the interpolation. 

1478 interpExp = ccdExposure.clone() 

1479 with self.flatContext(interpExp, flat, dark): 

1480 isrFunctions.interpolateFromMask( 

1481 maskedImage=interpExp.getMaskedImage(), 

1482 fwhm=self.config.fwhm, 

1483 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1484 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1485 ) 

1486 bfExp = interpExp.clone() 

1487 

1488 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1489 type(bfKernel), type(bfGains)) 

1490 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1491 self.config.brighterFatterMaxIter, 

1492 self.config.brighterFatterThreshold, 

1493 self.config.brighterFatterApplyGain, 

1494 bfGains) 

1495 if bfResults[1] == self.config.brighterFatterMaxIter: 

1496 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1497 bfResults[0]) 

1498 else: 

1499 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1500 bfResults[1]) 

1501 image = ccdExposure.getMaskedImage().getImage() 

1502 bfCorr = bfExp.getMaskedImage().getImage() 

1503 bfCorr -= interpExp.getMaskedImage().getImage() 

1504 image += bfCorr 

1505 

1506 # Applying the brighter-fatter correction applies a 

1507 # convolution to the science image. At the edges this 

1508 # convolution may not have sufficient valid pixels to 

1509 # produce a valid correction. Mark pixels within the size 

1510 # of the brighter-fatter kernel as EDGE to warn of this 

1511 # fact. 

1512 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1513 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1514 maskPlane="EDGE") 

1515 

1516 if self.config.brighterFatterMaskGrowSize > 0: 

1517 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1518 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1519 isrFunctions.growMasks(ccdExposure.getMask(), 

1520 radius=self.config.brighterFatterMaskGrowSize, 

1521 maskNameList=maskPlane, 

1522 maskValue=maskPlane) 

1523 

1524 self.debugView(ccdExposure, "doBrighterFatter") 

1525 

1526 if self.config.doDark: 

1527 self.log.info("Applying dark correction.") 

1528 self.darkCorrection(ccdExposure, dark) 

1529 self.debugView(ccdExposure, "doDark") 

1530 

1531 if self.config.doFringe and not self.config.fringeAfterFlat: 

1532 self.log.info("Applying fringe correction before flat.") 

1533 self.fringe.run(ccdExposure, **fringes.getDict()) 

1534 self.debugView(ccdExposure, "doFringe") 

1535 

1536 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1537 self.log.info("Checking strayLight correction.") 

1538 self.strayLight.run(ccdExposure, strayLightData) 

1539 self.debugView(ccdExposure, "doStrayLight") 

1540 

1541 if self.config.doFlat: 

1542 self.log.info("Applying flat correction.") 

1543 self.flatCorrection(ccdExposure, flat) 

1544 self.debugView(ccdExposure, "doFlat") 

1545 

1546 if self.config.doApplyGains: 

1547 self.log.info("Applying gain correction instead of flat.") 

1548 if self.config.usePtcGains: 

1549 self.log.info("Using gains from the Photon Transfer Curve.") 

1550 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1551 ptcGains=ptc.gain) 

1552 else: 

1553 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1554 

1555 if self.config.doFringe and self.config.fringeAfterFlat: 

1556 self.log.info("Applying fringe correction after flat.") 

1557 self.fringe.run(ccdExposure, **fringes.getDict()) 

1558 

1559 if self.config.doVignette: 

1560 if self.config.doMaskVignettePolygon: 

1561 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1562 else: 

1563 self.log.info("Constructing and attaching vignette polygon.") 

1564 self.vignettePolygon = self.vignette.run( 

1565 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1566 vignetteValue=self.config.vignetteValue, log=self.log) 

1567 

1568 if self.config.doAttachTransmissionCurve: 

1569 self.log.info("Adding transmission curves.") 

1570 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1571 filterTransmission=filterTransmission, 

1572 sensorTransmission=sensorTransmission, 

1573 atmosphereTransmission=atmosphereTransmission) 

1574 

1575 flattenedThumb = None 

1576 if self.config.qa.doThumbnailFlattened: 

1577 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1578 

1579 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1580 self.log.info("Performing illumination correction.") 

1581 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1582 illumMaskedImage, illumScale=self.config.illumScale, 

1583 trimToFit=self.config.doTrimToMatchCalib) 

1584 

1585 preInterpExp = None 

1586 if self.config.doSaveInterpPixels: 

1587 preInterpExp = ccdExposure.clone() 

1588 

1589 # Reset and interpolate bad pixels. 

1590 # 

1591 # Large contiguous bad regions (which should have the BAD mask 

1592 # bit set) should have their values set to the image median. 

1593 # This group should include defects and bad amplifiers. As the 

1594 # area covered by these defects are large, there's little 

1595 # reason to expect that interpolation would provide a more 

1596 # useful value. 

1597 # 

1598 # Smaller defects can be safely interpolated after the larger 

1599 # regions have had their pixel values reset. This ensures 

1600 # that the remaining defects adjacent to bad amplifiers (as an 

1601 # example) do not attempt to interpolate extreme values. 

1602 if self.config.doSetBadRegions: 

1603 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1604 if badPixelCount > 0: 

1605 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1606 

1607 if self.config.doInterpolate: 

1608 self.log.info("Interpolating masked pixels.") 

1609 isrFunctions.interpolateFromMask( 

1610 maskedImage=ccdExposure.getMaskedImage(), 

1611 fwhm=self.config.fwhm, 

1612 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1613 maskNameList=list(self.config.maskListToInterpolate) 

1614 ) 

1615 

1616 self.roughZeroPoint(ccdExposure) 

1617 

1618 # correct for amp offsets within the CCD 

1619 if self.config.doAmpOffset: 

1620 self.log.info("Correcting amp offsets.") 

1621 self.ampOffset.run(ccdExposure) 

1622 

1623 if self.config.doMeasureBackground: 

1624 self.log.info("Measuring background level.") 

1625 self.measureBackground(ccdExposure, self.config.qa) 

1626 

1627 if self.config.qa is not None and self.config.qa.saveStats is True: 

1628 for amp in ccd: 

1629 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1630 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1631 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1632 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1633 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1634 qaStats.getValue(afwMath.STDEVCLIP) 

1635 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1636 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1637 qaStats.getValue(afwMath.STDEVCLIP)) 

1638 

1639 # Calculate standard image quality statistics 

1640 if self.config.doStandardStatistics: 

1641 metadata = ccdExposure.getMetadata() 

1642 for amp in ccd: 

1643 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1644 ampName = amp.getName() 

1645 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1646 ampExposure.getMaskedImage(), 

1647 [self.config.saturatedMaskName] 

1648 ) 

1649 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1650 ampExposure.getMaskedImage(), 

1651 ["BAD"] 

1652 ) 

1653 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1654 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1655 

1656 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1657 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1658 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1659 

1660 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1661 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1662 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1663 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1664 else: 

1665 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1666 

1667 # calculate additional statistics. 

1668 outputStatistics = None 

1669 if self.config.doCalculateStatistics: 

1670 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1671 ptc=ptc).results 

1672 

1673 self.debugView(ccdExposure, "postISRCCD") 

1674 

1675 return pipeBase.Struct( 

1676 exposure=ccdExposure, 

1677 ossThumb=ossThumb, 

1678 flattenedThumb=flattenedThumb, 

1679 

1680 preInterpExposure=preInterpExp, 

1681 outputExposure=ccdExposure, 

1682 outputOssThumbnail=ossThumb, 

1683 outputFlattenedThumbnail=flattenedThumb, 

1684 outputStatistics=outputStatistics, 

1685 ) 

1686 

1687 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1688 """Ensure that the data returned by Butler is a fully constructed exp. 

1689 

1690 ISR requires exposure-level image data for historical reasons, so if we 

1691 did not recieve that from Butler, construct it from what we have, 

1692 modifying the input in place. 

1693 

1694 Parameters 

1695 ---------- 

1696 inputExp : `lsst.afw.image` image-type. 

1697 The input data structure obtained from Butler. 

1698 Can be `lsst.afw.image.Exposure`, 

1699 `lsst.afw.image.DecoratedImageU`, 

1700 or `lsst.afw.image.ImageF` 

1701 camera : `lsst.afw.cameraGeom.camera`, optional 

1702 The camera associated with the image. Used to find the appropriate 

1703 detector if detector is not already set. 

1704 detectorNum : `int`, optional 

1705 The detector in the camera to attach, if the detector is not 

1706 already set. 

1707 

1708 Returns 

1709 ------- 

1710 inputExp : `lsst.afw.image.Exposure` 

1711 The re-constructed exposure, with appropriate detector parameters. 

1712 

1713 Raises 

1714 ------ 

1715 TypeError 

1716 Raised if the input data cannot be used to construct an exposure. 

1717 """ 

1718 if isinstance(inputExp, afwImage.DecoratedImageU): 

1719 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1720 elif isinstance(inputExp, afwImage.ImageF): 

1721 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1722 elif isinstance(inputExp, afwImage.MaskedImageF): 

1723 inputExp = afwImage.makeExposure(inputExp) 

1724 elif isinstance(inputExp, afwImage.Exposure): 

1725 pass 

1726 elif inputExp is None: 

1727 # Assume this will be caught by the setup if it is a problem. 

1728 return inputExp 

1729 else: 

1730 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1731 (type(inputExp), )) 

1732 

1733 if inputExp.getDetector() is None: 

1734 if camera is None or detectorNum is None: 

1735 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1736 'without a detector set.') 

1737 inputExp.setDetector(camera[detectorNum]) 

1738 

1739 return inputExp 

1740 

1741 @staticmethod 

1742 def extractCalibDate(calib): 

1743 """Extract common calibration metadata values that will be written to 

1744 output header. 

1745 

1746 Parameters 

1747 ---------- 

1748 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1749 Calibration to pull date information from. 

1750 

1751 Returns 

1752 ------- 

1753 dateString : `str` 

1754 Calibration creation date string to add to header. 

1755 """ 

1756 if hasattr(calib, "getMetadata"): 

1757 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1758 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1759 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1760 else: 

1761 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1762 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1763 else: 

1764 return "Unknown Unknown" 

1765 

1766 def compareCameraKeywords(self, exposureMetadata, calib, calibName): 

1767 """Compare header keywords to confirm camera states match. 

1768 

1769 Parameters 

1770 ---------- 

1771 exposureMetadata : `lsst.daf.base.PropertySet` 

1772 Header for the exposure being processed. 

1773 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1774 Calibration to be applied. 

1775 calibName : `str` 

1776 Calib type for log message. 

1777 """ 

1778 try: 

1779 calibMetadata = calib.getMetadata() 

1780 except AttributeError: 

1781 return 

1782 for keyword in self.config.cameraKeywordsToCompare: 

1783 if keyword in exposureMetadata and keyword in calibMetadata: 

1784 if exposureMetadata[keyword] != calibMetadata[keyword]: 

1785 if self.config.doRaiseOnCalibMismatch: 

1786 raise RuntimeError("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1787 calibName, keyword, 

1788 exposureMetadata[keyword], calibMetadata[keyword]) 

1789 else: 

1790 self.log.warning("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1791 calibName, keyword, 

1792 exposureMetadata[keyword], calibMetadata[keyword]) 

1793 else: 

1794 self.log.debug("Sequencer keyword %s not found.", keyword) 

1795 

1796 def convertIntToFloat(self, exposure): 

1797 """Convert exposure image from uint16 to float. 

1798 

1799 If the exposure does not need to be converted, the input is 

1800 immediately returned. For exposures that are converted to use 

1801 floating point pixels, the variance is set to unity and the 

1802 mask to zero. 

1803 

1804 Parameters 

1805 ---------- 

1806 exposure : `lsst.afw.image.Exposure` 

1807 The raw exposure to be converted. 

1808 

1809 Returns 

1810 ------- 

1811 newexposure : `lsst.afw.image.Exposure` 

1812 The input ``exposure``, converted to floating point pixels. 

1813 

1814 Raises 

1815 ------ 

1816 RuntimeError 

1817 Raised if the exposure type cannot be converted to float. 

1818 

1819 """ 

1820 if isinstance(exposure, afwImage.ExposureF): 

1821 # Nothing to be done 

1822 self.log.debug("Exposure already of type float.") 

1823 return exposure 

1824 if not hasattr(exposure, "convertF"): 

1825 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1826 

1827 newexposure = exposure.convertF() 

1828 newexposure.variance[:] = 1 

1829 newexposure.mask[:] = 0x0 

1830 

1831 return newexposure 

1832 

1833 def maskAmplifier(self, ccdExposure, amp, defects): 

1834 """Identify bad amplifiers, saturated and suspect pixels. 

1835 

1836 Parameters 

1837 ---------- 

1838 ccdExposure : `lsst.afw.image.Exposure` 

1839 Input exposure to be masked. 

1840 amp : `lsst.afw.cameraGeom.Amplifier` 

1841 Catalog of parameters defining the amplifier on this 

1842 exposure to mask. 

1843 defects : `lsst.ip.isr.Defects` 

1844 List of defects. Used to determine if the entire 

1845 amplifier is bad. 

1846 

1847 Returns 

1848 ------- 

1849 badAmp : `Bool` 

1850 If this is true, the entire amplifier area is covered by 

1851 defects and unusable. 

1852 

1853 """ 

1854 maskedImage = ccdExposure.getMaskedImage() 

1855 

1856 badAmp = False 

1857 

1858 # Check if entire amp region is defined as a defect 

1859 # NB: need to use amp.getBBox() for correct comparison with current 

1860 # defects definition. 

1861 if defects is not None: 

1862 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1863 

1864 # In the case of a bad amp, we will set mask to "BAD" 

1865 # (here use amp.getRawBBox() for correct association with pixels in 

1866 # current ccdExposure). 

1867 if badAmp: 

1868 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1869 afwImage.PARENT) 

1870 maskView = dataView.getMask() 

1871 maskView |= maskView.getPlaneBitMask("BAD") 

1872 del maskView 

1873 return badAmp 

1874 

1875 # Mask remaining defects after assembleCcd() to allow for defects that 

1876 # cross amplifier boundaries. Saturation and suspect pixels can be 

1877 # masked now, though. 

1878 limits = dict() 

1879 if self.config.doSaturation and not badAmp: 

1880 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1881 if self.config.doSuspect and not badAmp: 

1882 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1883 if math.isfinite(self.config.saturation): 

1884 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1885 

1886 for maskName, maskThreshold in limits.items(): 

1887 if not math.isnan(maskThreshold): 

1888 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1889 isrFunctions.makeThresholdMask( 

1890 maskedImage=dataView, 

1891 threshold=maskThreshold, 

1892 growFootprints=0, 

1893 maskName=maskName 

1894 ) 

1895 

1896 # Determine if we've fully masked this amplifier with SUSPECT and 

1897 # SAT pixels. 

1898 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1899 afwImage.PARENT) 

1900 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1901 self.config.suspectMaskName]) 

1902 if numpy.all(maskView.getArray() & maskVal > 0): 

1903 badAmp = True 

1904 maskView |= maskView.getPlaneBitMask("BAD") 

1905 

1906 return badAmp 

1907 

1908 def overscanCorrection(self, ccdExposure, amp): 

1909 """Apply overscan correction in place. 

1910 

1911 This method does initial pixel rejection of the overscan 

1912 region. The overscan can also be optionally segmented to 

1913 allow for discontinuous overscan responses to be fit 

1914 separately. The actual overscan subtraction is performed by 

1915 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

1916 after the amplifier is preprocessed. 

1917 

1918 Parameters 

1919 ---------- 

1920 ccdExposure : `lsst.afw.image.Exposure` 

1921 Exposure to have overscan correction performed. 

1922 amp : `lsst.afw.cameraGeom.Amplifer` 

1923 The amplifier to consider while correcting the overscan. 

1924 

1925 Returns 

1926 ------- 

1927 overscanResults : `lsst.pipe.base.Struct` 

1928 Result struct with components: 

1929 

1930 ``imageFit`` 

1931 Value or fit subtracted from the amplifier image data. 

1932 (scalar or `lsst.afw.image.Image`) 

1933 ``overscanFit`` 

1934 Value or fit subtracted from the overscan image data. 

1935 (scalar or `lsst.afw.image.Image`) 

1936 ``overscanImage`` 

1937 Image of the overscan region with the overscan 

1938 correction applied. This quantity is used to estimate 

1939 the amplifier read noise empirically. 

1940 (`lsst.afw.image.Image`) 

1941 ``edgeMask`` 

1942 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

1943 ``overscanMean`` 

1944 Median overscan fit value. (`float`) 

1945 ``overscanSigma`` 

1946 Clipped standard deviation of the overscan after 

1947 correction. (`float`) 

1948 

1949 Raises 

1950 ------ 

1951 RuntimeError 

1952 Raised if the ``amp`` does not contain raw pixel information. 

1953 

1954 See Also 

1955 -------- 

1956 lsst.ip.isr.overscan.OverscanTask 

1957 """ 

1958 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1959 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1960 return None 

1961 

1962 # Perform overscan correction on subregions. 

1963 overscanResults = self.overscan.run(ccdExposure, amp) 

1964 

1965 metadata = ccdExposure.getMetadata() 

1966 ampName = amp.getName() 

1967 

1968 keyBase = "LSST ISR OVERSCAN" 

1969 # Updated quantities 

1970 if isinstance(overscanResults.overscanMean, float): 

1971 # Serial overscan correction only: 

1972 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

1973 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

1974 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

1975 

1976 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

1977 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

1978 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

1979 elif isinstance(overscanResults.overscanMean, tuple): 

1980 # Both serial and parallel overscan have run: 

1981 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

1982 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

1983 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

1984 

1985 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

1986 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

1987 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

1988 

1989 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

1990 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

1991 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

1992 

1993 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

1994 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

1995 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

1996 else: 

1997 self.log.warning("Unexpected type for overscan values; none added to header.") 

1998 

1999 return overscanResults 

2000 

2001 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2002 """Set the variance plane using the gain and read noise 

2003 

2004 The read noise is calculated from the ``overscanImage`` if the 

2005 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2006 the value from the amplifier data is used. 

2007 

2008 Parameters 

2009 ---------- 

2010 ampExposure : `lsst.afw.image.Exposure` 

2011 Exposure to process. 

2012 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

2013 Amplifier detector data. 

2014 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2015 Image of overscan, required only for empirical read noise. 

2016 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2017 PTC dataset containing the gains and read noise. 

2018 

2019 Raises 

2020 ------ 

2021 RuntimeError 

2022 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2023 are ``True``, but ptcDataset is not provided. 

2024 

2025 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2026 ``overscanImage`` is ``None``. 

2027 

2028 See also 

2029 -------- 

2030 lsst.ip.isr.isrFunctions.updateVariance 

2031 """ 

2032 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2033 if self.config.usePtcGains: 

2034 if ptcDataset is None: 

2035 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2036 else: 

2037 gain = ptcDataset.gain[amp.getName()] 

2038 self.log.info("Using gain from Photon Transfer Curve.") 

2039 else: 

2040 gain = amp.getGain() 

2041 

2042 if math.isnan(gain): 

2043 gain = 1.0 

2044 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2045 elif gain <= 0: 

2046 patchedGain = 1.0 

2047 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2048 amp.getName(), gain, patchedGain) 

2049 gain = patchedGain 

2050 

2051 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2052 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(), 

2053 [self.config.saturatedMaskName, 

2054 self.config.suspectMaskName, 

2055 "BAD", "NO_DATA"]) 

2056 allPixels = ampExposure.getWidth() * ampExposure.getHeight() 

2057 if allPixels == badPixels: 

2058 # If the image is bad, do not raise. 

2059 self.log.info("Skipping empirical read noise for amp %s. No good pixels.", 

2060 amp.getName()) 

2061 else: 

2062 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2063 

2064 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2065 stats = afwMath.StatisticsControl() 

2066 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2067 readNoise = afwMath.makeStatistics(overscanImage.getImage(), 

2068 afwMath.STDEVCLIP, stats).getValue() 

2069 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2070 amp.getName(), readNoise) 

2071 elif self.config.usePtcReadNoise: 

2072 if ptcDataset is None: 

2073 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2074 else: 

2075 readNoise = ptcDataset.noise[amp.getName()] 

2076 self.log.info("Using read noise from Photon Transfer Curve.") 

2077 else: 

2078 readNoise = amp.getReadNoise() 

2079 

2080 metadata = ampExposure.getMetadata() 

2081 metadata[f'LSST GAIN {amp.getName()}'] = gain 

2082 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise 

2083 

2084 isrFunctions.updateVariance( 

2085 maskedImage=ampExposure.getMaskedImage(), 

2086 gain=gain, 

2087 readNoise=readNoise, 

2088 ) 

2089 

2090 def maskNegativeVariance(self, exposure): 

2091 """Identify and mask pixels with negative variance values. 

2092 

2093 Parameters 

2094 ---------- 

2095 exposure : `lsst.afw.image.Exposure` 

2096 Exposure to process. 

2097 

2098 See Also 

2099 -------- 

2100 lsst.ip.isr.isrFunctions.updateVariance 

2101 """ 

2102 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2103 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2104 exposure.mask.array[bad] |= maskPlane 

2105 

2106 def darkCorrection(self, exposure, darkExposure, invert=False): 

2107 """Apply dark correction in place. 

2108 

2109 Parameters 

2110 ---------- 

2111 exposure : `lsst.afw.image.Exposure` 

2112 Exposure to process. 

2113 darkExposure : `lsst.afw.image.Exposure` 

2114 Dark exposure of the same size as ``exposure``. 

2115 invert : `Bool`, optional 

2116 If True, re-add the dark to an already corrected image. 

2117 

2118 Raises 

2119 ------ 

2120 RuntimeError 

2121 Raised if either ``exposure`` or ``darkExposure`` do not 

2122 have their dark time defined. 

2123 

2124 See Also 

2125 -------- 

2126 lsst.ip.isr.isrFunctions.darkCorrection 

2127 """ 

2128 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2129 if math.isnan(expScale): 

2130 raise RuntimeError("Exposure darktime is NAN.") 

2131 if darkExposure.getInfo().getVisitInfo() is not None \ 

2132 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2133 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2134 else: 

2135 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2136 # so getDarkTime() does not exist. 

2137 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2138 darkScale = 1.0 

2139 

2140 isrFunctions.darkCorrection( 

2141 maskedImage=exposure.getMaskedImage(), 

2142 darkMaskedImage=darkExposure.getMaskedImage(), 

2143 expScale=expScale, 

2144 darkScale=darkScale, 

2145 invert=invert, 

2146 trimToFit=self.config.doTrimToMatchCalib 

2147 ) 

2148 

2149 def doLinearize(self, detector): 

2150 """Check if linearization is needed for the detector cameraGeom. 

2151 

2152 Checks config.doLinearize and the linearity type of the first 

2153 amplifier. 

2154 

2155 Parameters 

2156 ---------- 

2157 detector : `lsst.afw.cameraGeom.Detector` 

2158 Detector to get linearity type from. 

2159 

2160 Returns 

2161 ------- 

2162 doLinearize : `Bool` 

2163 If True, linearization should be performed. 

2164 """ 

2165 return self.config.doLinearize and \ 

2166 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2167 

2168 def flatCorrection(self, exposure, flatExposure, invert=False): 

2169 """Apply flat correction in place. 

2170 

2171 Parameters 

2172 ---------- 

2173 exposure : `lsst.afw.image.Exposure` 

2174 Exposure to process. 

2175 flatExposure : `lsst.afw.image.Exposure` 

2176 Flat exposure of the same size as ``exposure``. 

2177 invert : `Bool`, optional 

2178 If True, unflatten an already flattened image. 

2179 

2180 See Also 

2181 -------- 

2182 lsst.ip.isr.isrFunctions.flatCorrection 

2183 """ 

2184 isrFunctions.flatCorrection( 

2185 maskedImage=exposure.getMaskedImage(), 

2186 flatMaskedImage=flatExposure.getMaskedImage(), 

2187 scalingType=self.config.flatScalingType, 

2188 userScale=self.config.flatUserScale, 

2189 invert=invert, 

2190 trimToFit=self.config.doTrimToMatchCalib 

2191 ) 

2192 

2193 def saturationDetection(self, exposure, amp): 

2194 """Detect and mask saturated pixels in config.saturatedMaskName. 

2195 

2196 Parameters 

2197 ---------- 

2198 exposure : `lsst.afw.image.Exposure` 

2199 Exposure to process. Only the amplifier DataSec is processed. 

2200 amp : `lsst.afw.cameraGeom.Amplifier` 

2201 Amplifier detector data. 

2202 

2203 See Also 

2204 -------- 

2205 lsst.ip.isr.isrFunctions.makeThresholdMask 

2206 """ 

2207 if not math.isnan(amp.getSaturation()): 

2208 maskedImage = exposure.getMaskedImage() 

2209 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2210 isrFunctions.makeThresholdMask( 

2211 maskedImage=dataView, 

2212 threshold=amp.getSaturation(), 

2213 growFootprints=0, 

2214 maskName=self.config.saturatedMaskName, 

2215 ) 

2216 

2217 def saturationInterpolation(self, exposure): 

2218 """Interpolate over saturated pixels, in place. 

2219 

2220 This method should be called after `saturationDetection`, to 

2221 ensure that the saturated pixels have been identified in the 

2222 SAT mask. It should also be called after `assembleCcd`, since 

2223 saturated regions may cross amplifier boundaries. 

2224 

2225 Parameters 

2226 ---------- 

2227 exposure : `lsst.afw.image.Exposure` 

2228 Exposure to process. 

2229 

2230 See Also 

2231 -------- 

2232 lsst.ip.isr.isrTask.saturationDetection 

2233 lsst.ip.isr.isrFunctions.interpolateFromMask 

2234 """ 

2235 isrFunctions.interpolateFromMask( 

2236 maskedImage=exposure.getMaskedImage(), 

2237 fwhm=self.config.fwhm, 

2238 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2239 maskNameList=list(self.config.saturatedMaskName), 

2240 ) 

2241 

2242 def suspectDetection(self, exposure, amp): 

2243 """Detect and mask suspect pixels in config.suspectMaskName. 

2244 

2245 Parameters 

2246 ---------- 

2247 exposure : `lsst.afw.image.Exposure` 

2248 Exposure to process. Only the amplifier DataSec is processed. 

2249 amp : `lsst.afw.cameraGeom.Amplifier` 

2250 Amplifier detector data. 

2251 

2252 See Also 

2253 -------- 

2254 lsst.ip.isr.isrFunctions.makeThresholdMask 

2255 

2256 Notes 

2257 ----- 

2258 Suspect pixels are pixels whose value is greater than 

2259 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2260 affected by unknown systematics; for example if non-linearity 

2261 corrections above a certain level are unstable then that would be a 

2262 useful value for suspectLevel. A value of `nan` indicates that no such 

2263 level exists and no pixels are to be masked as suspicious. 

2264 """ 

2265 suspectLevel = amp.getSuspectLevel() 

2266 if math.isnan(suspectLevel): 

2267 return 

2268 

2269 maskedImage = exposure.getMaskedImage() 

2270 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2271 isrFunctions.makeThresholdMask( 

2272 maskedImage=dataView, 

2273 threshold=suspectLevel, 

2274 growFootprints=0, 

2275 maskName=self.config.suspectMaskName, 

2276 ) 

2277 

2278 def maskDefect(self, exposure, defectBaseList): 

2279 """Mask defects using mask plane "BAD", in place. 

2280 

2281 Parameters 

2282 ---------- 

2283 exposure : `lsst.afw.image.Exposure` 

2284 Exposure to process. 

2285 defectBaseList : defect-type 

2286 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2287 or `list` of `lsst.afw.image.DefectBase`. 

2288 

2289 Notes 

2290 ----- 

2291 Call this after CCD assembly, since defects may cross amplifier 

2292 boundaries. 

2293 """ 

2294 maskedImage = exposure.getMaskedImage() 

2295 if not isinstance(defectBaseList, Defects): 

2296 # Promotes DefectBase to Defect 

2297 defectList = Defects(defectBaseList) 

2298 else: 

2299 defectList = defectBaseList 

2300 defectList.maskPixels(maskedImage, maskName="BAD") 

2301 

2302 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2303 """Mask edge pixels with applicable mask plane. 

2304 

2305 Parameters 

2306 ---------- 

2307 exposure : `lsst.afw.image.Exposure` 

2308 Exposure to process. 

2309 numEdgePixels : `int`, optional 

2310 Number of edge pixels to mask. 

2311 maskPlane : `str`, optional 

2312 Mask plane name to use. 

2313 level : `str`, optional 

2314 Level at which to mask edges. 

2315 """ 

2316 maskedImage = exposure.getMaskedImage() 

2317 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2318 

2319 if numEdgePixels > 0: 

2320 if level == 'DETECTOR': 

2321 boxes = [maskedImage.getBBox()] 

2322 elif level == 'AMP': 

2323 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2324 

2325 for box in boxes: 

2326 # This makes a bbox numEdgeSuspect pixels smaller than the 

2327 # image on each side 

2328 subImage = maskedImage[box] 

2329 box.grow(-numEdgePixels) 

2330 # Mask pixels outside box 

2331 SourceDetectionTask.setEdgeBits( 

2332 subImage, 

2333 box, 

2334 maskBitMask) 

2335 

2336 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2337 """Mask and interpolate defects using mask plane "BAD", in place. 

2338 

2339 Parameters 

2340 ---------- 

2341 exposure : `lsst.afw.image.Exposure` 

2342 Exposure to process. 

2343 defectBaseList : defects-like 

2344 List of defects to mask and interpolate. Can be 

2345 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2346 

2347 See Also 

2348 -------- 

2349 lsst.ip.isr.isrTask.maskDefect 

2350 """ 

2351 self.maskDefect(exposure, defectBaseList) 

2352 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2353 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2354 isrFunctions.interpolateFromMask( 

2355 maskedImage=exposure.getMaskedImage(), 

2356 fwhm=self.config.fwhm, 

2357 growSaturatedFootprints=0, 

2358 maskNameList=["BAD"], 

2359 ) 

2360 

2361 def maskNan(self, exposure): 

2362 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2363 

2364 Parameters 

2365 ---------- 

2366 exposure : `lsst.afw.image.Exposure` 

2367 Exposure to process. 

2368 

2369 Notes 

2370 ----- 

2371 We mask over all non-finite values (NaN, inf), including those 

2372 that are masked with other bits (because those may or may not be 

2373 interpolated over later, and we want to remove all NaN/infs). 

2374 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2375 preserve the historical name. 

2376 """ 

2377 maskedImage = exposure.getMaskedImage() 

2378 

2379 # Find and mask NaNs 

2380 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2381 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2382 numNans = maskNans(maskedImage, maskVal) 

2383 self.metadata["NUMNANS"] = numNans 

2384 if numNans > 0: 

2385 self.log.warning("There were %d unmasked NaNs.", numNans) 

2386 

2387 def maskAndInterpolateNan(self, exposure): 

2388 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2389 in place. 

2390 

2391 Parameters 

2392 ---------- 

2393 exposure : `lsst.afw.image.Exposure` 

2394 Exposure to process. 

2395 

2396 See Also 

2397 -------- 

2398 lsst.ip.isr.isrTask.maskNan 

2399 """ 

2400 self.maskNan(exposure) 

2401 isrFunctions.interpolateFromMask( 

2402 maskedImage=exposure.getMaskedImage(), 

2403 fwhm=self.config.fwhm, 

2404 growSaturatedFootprints=0, 

2405 maskNameList=["UNMASKEDNAN"], 

2406 ) 

2407 

2408 def measureBackground(self, exposure, IsrQaConfig=None): 

2409 """Measure the image background in subgrids, for quality control. 

2410 

2411 Parameters 

2412 ---------- 

2413 exposure : `lsst.afw.image.Exposure` 

2414 Exposure to process. 

2415 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2416 Configuration object containing parameters on which background 

2417 statistics and subgrids to use. 

2418 """ 

2419 if IsrQaConfig is not None: 

2420 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2421 IsrQaConfig.flatness.nIter) 

2422 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2423 statsControl.setAndMask(maskVal) 

2424 maskedImage = exposure.getMaskedImage() 

2425 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2426 skyLevel = stats.getValue(afwMath.MEDIAN) 

2427 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2428 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2429 metadata = exposure.getMetadata() 

2430 metadata["SKYLEVEL"] = skyLevel 

2431 metadata["SKYSIGMA"] = skySigma 

2432 

2433 # calcluating flatlevel over the subgrids 

2434 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2435 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2436 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2437 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2438 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2439 skyLevels = numpy.zeros((nX, nY)) 

2440 

2441 for j in range(nY): 

2442 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2443 for i in range(nX): 

2444 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2445 

2446 xLLC = xc - meshXHalf 

2447 yLLC = yc - meshYHalf 

2448 xURC = xc + meshXHalf - 1 

2449 yURC = yc + meshYHalf - 1 

2450 

2451 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2452 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2453 

2454 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2455 

2456 good = numpy.where(numpy.isfinite(skyLevels)) 

2457 skyMedian = numpy.median(skyLevels[good]) 

2458 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2459 flatness_rms = numpy.std(flatness) 

2460 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2461 

2462 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2463 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2464 nX, nY, flatness_pp, flatness_rms) 

2465 

2466 metadata["FLATNESS_PP"] = float(flatness_pp) 

2467 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2468 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2469 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2470 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2471 

2472 def roughZeroPoint(self, exposure): 

2473 """Set an approximate magnitude zero point for the exposure. 

2474 

2475 Parameters 

2476 ---------- 

2477 exposure : `lsst.afw.image.Exposure` 

2478 Exposure to process. 

2479 """ 

2480 filterLabel = exposure.getFilter() 

2481 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2482 

2483 if physicalFilter in self.config.fluxMag0T1: 

2484 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2485 else: 

2486 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2487 fluxMag0 = self.config.defaultFluxMag0T1 

2488 

2489 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2490 if not expTime > 0: # handle NaN as well as <= 0 

2491 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2492 return 

2493 

2494 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2495 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2496 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2497 

2498 @contextmanager 

2499 def flatContext(self, exp, flat, dark=None): 

2500 """Context manager that applies and removes flats and darks, 

2501 if the task is configured to apply them. 

2502 

2503 Parameters 

2504 ---------- 

2505 exp : `lsst.afw.image.Exposure` 

2506 Exposure to process. 

2507 flat : `lsst.afw.image.Exposure` 

2508 Flat exposure the same size as ``exp``. 

2509 dark : `lsst.afw.image.Exposure`, optional 

2510 Dark exposure the same size as ``exp``. 

2511 

2512 Yields 

2513 ------ 

2514 exp : `lsst.afw.image.Exposure` 

2515 The flat and dark corrected exposure. 

2516 """ 

2517 if self.config.doDark and dark is not None: 

2518 self.darkCorrection(exp, dark) 

2519 if self.config.doFlat: 

2520 self.flatCorrection(exp, flat) 

2521 try: 

2522 yield exp 

2523 finally: 

2524 if self.config.doFlat: 

2525 self.flatCorrection(exp, flat, invert=True) 

2526 if self.config.doDark and dark is not None: 

2527 self.darkCorrection(exp, dark, invert=True) 

2528 

2529 def debugView(self, exposure, stepname): 

2530 """Utility function to examine ISR exposure at different stages. 

2531 

2532 Parameters 

2533 ---------- 

2534 exposure : `lsst.afw.image.Exposure` 

2535 Exposure to view. 

2536 stepname : `str` 

2537 State of processing to view. 

2538 """ 

2539 frame = getDebugFrame(self._display, stepname) 

2540 if frame: 

2541 display = getDisplay(frame) 

2542 display.scale('asinh', 'zscale') 

2543 display.mtv(exposure) 

2544 prompt = "Press Enter to continue [c]... " 

2545 while True: 

2546 ans = input(prompt).lower() 

2547 if ans in ("", "c",): 

2548 break 

2549 

2550 

2551class FakeAmp(object): 

2552 """A Detector-like object that supports returning gain and saturation level 

2553 

2554 This is used when the input exposure does not have a detector. 

2555 

2556 Parameters 

2557 ---------- 

2558 exposure : `lsst.afw.image.Exposure` 

2559 Exposure to generate a fake amplifier for. 

2560 config : `lsst.ip.isr.isrTaskConfig` 

2561 Configuration to apply to the fake amplifier. 

2562 """ 

2563 

2564 def __init__(self, exposure, config): 

2565 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2566 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2567 self._gain = config.gain 

2568 self._readNoise = config.readNoise 

2569 self._saturation = config.saturation 

2570 

2571 def getBBox(self): 

2572 return self._bbox 

2573 

2574 def getRawBBox(self): 

2575 return self._bbox 

2576 

2577 def getRawHorizontalOverscanBBox(self): 

2578 return self._RawHorizontalOverscanBBox 

2579 

2580 def getGain(self): 

2581 return self._gain 

2582 

2583 def getReadNoise(self): 

2584 return self._readNoise 

2585 

2586 def getSaturation(self): 

2587 return self._saturation 

2588 

2589 def getSuspectLevel(self): 

2590 return float("NaN")