Coverage for python/lsst/ip/isr/isrTaskLSST.py: 19%

475 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-11 04:06 -0700

1__all__ = ["IsrTaskLSST", "IsrTaskLSSTConfig"] 

2 

3import numpy 

4import math 

5 

6from . import isrFunctions 

7from . import isrQa 

8from . import linearize 

9from .defects import Defects 

10 

11from contextlib import contextmanager 

12from lsst.afw.cameraGeom import NullLinearityType 

13import lsst.pex.config as pexConfig 

14import lsst.afw.math as afwMath 

15import lsst.pipe.base as pipeBase 

16import lsst.afw.image as afwImage 

17import lsst.pipe.base.connectionTypes as cT 

18from lsst.meas.algorithms.detection import SourceDetectionTask 

19 

20from .overscan import SerialOverscanCorrectionTask, ParallelOverscanCorrectionTask 

21from .overscanAmpConfig import OverscanCameraConfig 

22from .assembleCcdTask import AssembleCcdTask 

23from .deferredCharge import DeferredChargeTask 

24from .crosstalk import CrosstalkTask 

25from .masking import MaskingTask 

26from .isrStatistics import IsrStatisticsTask 

27from .isr import maskNans 

28 

29 

30class IsrTaskLSSTConnections(pipeBase.PipelineTaskConnections, 

31 dimensions={"instrument", "exposure", "detector"}, 

32 defaultTemplates={}): 

33 ccdExposure = cT.Input( 

34 name="raw", 

35 doc="Input exposure to process.", 

36 storageClass="Exposure", 

37 dimensions=["instrument", "exposure", "detector"], 

38 ) 

39 camera = cT.PrerequisiteInput( 

40 name="camera", 

41 storageClass="Camera", 

42 doc="Input camera to construct complete exposures.", 

43 dimensions=["instrument"], 

44 isCalibration=True, 

45 ) 

46 dnlLUT = cT.PrerequisiteInput( 

47 name="dnlLUT", 

48 doc="Look-up table for differential non-linearity.", 

49 storageClass="IsrCalib", 

50 dimensions=["instrument", "exposure", "detector"], 

51 isCalibration=True, 

52 # TODO DM 36636 

53 ) 

54 bias = cT.PrerequisiteInput( 

55 name="bias", 

56 doc="Input bias calibration.", 

57 storageClass="ExposureF", 

58 dimensions=["instrument", "detector"], 

59 isCalibration=True, 

60 ) 

61 deferredChargeCalib = cT.PrerequisiteInput( 

62 name="cpCtiCalib", 

63 doc="Deferred charge/CTI correction dataset.", 

64 storageClass="IsrCalib", 

65 dimensions=["instrument", "detector"], 

66 isCalibration=True, 

67 ) 

68 linearizer = cT.PrerequisiteInput( 

69 name='linearizer', 

70 storageClass="Linearizer", 

71 doc="Linearity correction calibration.", 

72 dimensions=["instrument", "detector"], 

73 isCalibration=True, 

74 ) 

75 ptc = cT.PrerequisiteInput( 

76 name="ptc", 

77 doc="Input Photon Transfer Curve dataset", 

78 storageClass="PhotonTransferCurveDataset", 

79 dimensions=["instrument", "detector"], 

80 isCalibration=True, 

81 ) 

82 crosstalk = cT.PrerequisiteInput( 

83 name="crosstalk", 

84 doc="Input crosstalk object", 

85 storageClass="CrosstalkCalib", 

86 dimensions=["instrument", "detector"], 

87 isCalibration=True, 

88 ) 

89 defects = cT.PrerequisiteInput( 

90 name='defects', 

91 doc="Input defect tables.", 

92 storageClass="Defects", 

93 dimensions=["instrument", "detector"], 

94 isCalibration=True, 

95 ) 

96 bfKernel = cT.PrerequisiteInput( 

97 name='brighterFatterKernel', 

98 doc="Complete kernel + gain solutions.", 

99 storageClass="BrighterFatterKernel", 

100 dimensions=["instrument", "detector"], 

101 isCalibration=True, 

102 ) 

103 dark = cT.PrerequisiteInput( 

104 name='dark', 

105 doc="Input dark calibration.", 

106 storageClass="ExposureF", 

107 dimensions=["instrument", "detector"], 

108 isCalibration=True, 

109 ) 

110 outputExposure = cT.Output( 

111 name='postISRCCD', 

112 doc="Output ISR processed exposure.", 

113 storageClass="Exposure", 

114 dimensions=["instrument", "exposure", "detector"], 

115 ) 

116 preInterpExposure = cT.Output( 

117 name='preInterpISRCCD', 

118 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

119 storageClass="ExposureF", 

120 dimensions=["instrument", "exposure", "detector"], 

121 ) 

122 outputBin1Exposure = cT.Output( 

123 name="postIsrBin1", 

124 doc="First binned image.", 

125 storageClass="ExposureF", 

126 dimensions=["instrument", "exposure", "detector"], 

127 ) 

128 outputBin2Exposure = cT.Output( 

129 name="postIsrBin2", 

130 doc="Second binned image.", 

131 storageClass="ExposureF", 

132 dimensions=["instrument", "exposure", "detector"], 

133 ) 

134 

135 outputStatistics = cT.Output( 

136 name="isrStatistics", 

137 doc="Output of additional statistics table.", 

138 storageClass="StructuredDataDict", 

139 dimensions=["instrument", "exposure", "detector"], 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if config.doDiffNonLinearCorrection is not True: 

146 del self.dnlLUT 

147 if config.doBias is not True: 

148 del self.bias 

149 if config.doDeferredCharge is not True: 

150 del self.deferredChargeCalib 

151 if config.doLinearize is not True: 

152 del self.linearizer 

153 if not config.doCrosstalk and not config.overscanCamera.doAnyParallelOverscanCrosstalk: 

154 del self.crosstalk 

155 if config.doDefect is not True: 

156 del self.defects 

157 if config.doBrighterFatter is not True: 

158 del self.bfKernel 

159 if config.doDark is not True: 

160 del self.dark 

161 

162 if config.doBinnedExposures is not True: 

163 del self.outputBin1Exposure 

164 del self.outputBin2Exposure 

165 if config.doSaveInterpPixels is not True: 

166 del self.preInterpExposure 

167 

168 if config.doCalculateStatistics is not True: 

169 del self.outputStatistics 

170 

171 

172class IsrTaskLSSTConfig(pipeBase.PipelineTaskConfig, 

173 pipelineConnections=IsrTaskLSSTConnections): 

174 """Configuration parameters for IsrTaskLSST. 

175 

176 Items are grouped in the order in which they are executed by the task. 

177 """ 

178 expectWcs = pexConfig.Field( 

179 dtype=bool, 

180 default=True, 

181 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

182 ) 

183 qa = pexConfig.ConfigField( 

184 dtype=isrQa.IsrQaConfig, 

185 doc="QA related configuration options.", 

186 ) 

187 doHeaderProvenance = pexConfig.Field( 

188 dtype=bool, 

189 default=True, 

190 doc="Write calibration identifiers into output exposure header.", 

191 ) 

192 

193 # Differential non-linearity correction. 

194 doDiffNonLinearCorrection = pexConfig.Field( 

195 dtype=bool, 

196 doc="Do differential non-linearity correction?", 

197 default=False, 

198 ) 

199 

200 overscanCamera = pexConfig.ConfigField( 

201 dtype=OverscanCameraConfig, 

202 doc="Per-detector and per-amplifier overscan configurations.", 

203 ) 

204 

205 # Amplifier to CCD assembly configuration. 

206 doAssembleCcd = pexConfig.Field( 

207 dtype=bool, 

208 default=True, 

209 doc="Assemble amp-level exposures into a ccd-level exposure?" 

210 ) 

211 assembleCcd = pexConfig.ConfigurableField( 

212 target=AssembleCcdTask, 

213 doc="CCD assembly task.", 

214 ) 

215 

216 # Bias subtraction. 

217 doBias = pexConfig.Field( 

218 dtype=bool, 

219 doc="Apply bias frame correction?", 

220 default=True, 

221 ) 

222 

223 # Deferred charge correction. 

224 doDeferredCharge = pexConfig.Field( 

225 dtype=bool, 

226 doc="Apply deferred charge correction?", 

227 default=True, 

228 ) 

229 deferredChargeCorrection = pexConfig.ConfigurableField( 

230 target=DeferredChargeTask, 

231 doc="Deferred charge correction task.", 

232 ) 

233 

234 # Linearization. 

235 doLinearize = pexConfig.Field( 

236 dtype=bool, 

237 doc="Correct for nonlinearity of the detector's response?", 

238 default=True, 

239 ) 

240 

241 # Gains. 

242 doGainsCorrection = pexConfig.Field( 

243 dtype=bool, 

244 doc="Apply temperature correction to the gains?", 

245 default=False, 

246 ) 

247 doApplyGains = pexConfig.Field( 

248 dtype=bool, 

249 doc="Apply gains to the image?", 

250 default=True, 

251 ) 

252 

253 # Variance construction. 

254 doVariance = pexConfig.Field( 

255 dtype=bool, 

256 doc="Calculate variance?", 

257 default=True 

258 ) 

259 gain = pexConfig.Field( 

260 dtype=float, 

261 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN).", 

262 default=float("NaN"), 

263 ) 

264 maskNegativeVariance = pexConfig.Field( 

265 dtype=bool, 

266 doc="Mask pixels that claim a negative variance. This likely indicates a failure " 

267 "in the measurement of the overscan at an edge due to the data falling off faster " 

268 "than the overscan model can account for it.", 

269 default=True, 

270 ) 

271 negativeVarianceMaskName = pexConfig.Field( 

272 dtype=str, 

273 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

274 default="BAD", 

275 ) 

276 saturatedMaskName = pexConfig.Field( 

277 dtype=str, 

278 doc="Name of mask plane to use in saturation detection and interpolation.", 

279 default="SAT", 

280 ) 

281 suspectMaskName = pexConfig.Field( 

282 dtype=str, 

283 doc="Name of mask plane to use for suspect pixels.", 

284 default="SUSPECT", 

285 ) 

286 

287 # Crosstalk. 

288 doCrosstalk = pexConfig.Field( 

289 dtype=bool, 

290 doc="Apply intra-CCD crosstalk correction?", 

291 default=True, 

292 ) 

293 crosstalk = pexConfig.ConfigurableField( 

294 target=CrosstalkTask, 

295 doc="Intra-CCD crosstalk correction.", 

296 ) 

297 

298 # Masking options. 

299 doDefect = pexConfig.Field( 

300 dtype=bool, 

301 doc="Apply correction for CCD defects, e.g. hot pixels?", 

302 default=True, 

303 ) 

304 doNanMasking = pexConfig.Field( 

305 dtype=bool, 

306 doc="Mask non-finite (NAN, inf) pixels.", 

307 default=True, 

308 ) 

309 doWidenSaturationTrails = pexConfig.Field( 

310 dtype=bool, 

311 doc="Widen bleed trails based on their width.", 

312 default=True, 

313 ) 

314 masking = pexConfig.ConfigurableField( 

315 target=MaskingTask, 

316 doc="Masking task." 

317 ) 

318 

319 # Interpolation options. 

320 doInterpolate = pexConfig.Field( 

321 dtype=bool, 

322 doc="Interpolate masked pixels?", 

323 default=True, 

324 ) 

325 maskListToInterpolate = pexConfig.ListField( 

326 dtype=str, 

327 doc="List of mask planes that should be interpolated.", 

328 default=['SAT', 'BAD'], 

329 ) 

330 doSaveInterpPixels = pexConfig.Field( 

331 dtype=bool, 

332 doc="Save a copy of the pre-interpolated pixel values?", 

333 default=False, 

334 ) 

335 

336 # Initial masking options. 

337 doSetBadRegions = pexConfig.Field( 

338 dtype=bool, 

339 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

340 default=True, 

341 ) 

342 

343 # Brighter-Fatter correction. 

344 doBrighterFatter = pexConfig.Field( 

345 dtype=bool, 

346 doc="Apply the brighter-fatter correction?", 

347 default=True, 

348 ) 

349 brighterFatterLevel = pexConfig.ChoiceField( 

350 dtype=str, 

351 doc="The level at which to correct for brighter-fatter.", 

352 allowed={ 

353 "AMP": "Every amplifier treated separately.", 

354 "DETECTOR": "One kernel per detector.", 

355 }, 

356 default="DETECTOR", 

357 ) 

358 brighterFatterMaxIter = pexConfig.Field( 

359 dtype=int, 

360 doc="Maximum number of iterations for the brighter-fatter correction.", 

361 default=10, 

362 ) 

363 brighterFatterThreshold = pexConfig.Field( 

364 dtype=float, 

365 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

366 "absolute value of the difference between the current corrected image and the one " 

367 "from the previous iteration summed over all the pixels.", 

368 default=1000, 

369 ) 

370 brighterFatterApplyGain = pexConfig.Field( 

371 dtype=bool, 

372 doc="Should the gain be applied when applying the brighter-fatter correction?", 

373 default=True, 

374 ) 

375 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

376 dtype=str, 

377 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

378 "correction.", 

379 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

380 ) 

381 brighterFatterMaskGrowSize = pexConfig.Field( 

382 dtype=int, 

383 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

384 "when brighter-fatter correction is applied.", 

385 default=0, 

386 ) 

387 brighterFatterFwhmForInterpolation = pexConfig.Field( 

388 dtype=float, 

389 doc="FWHM of PSF in arcseconds used for interpolation in brighter-fatter correction " 

390 "(currently unused).", 

391 default=1.0, 

392 ) 

393 growSaturationFootprintSize = pexConfig.Field( 

394 dtype=int, 

395 doc="Number of pixels by which to grow the saturation footprints.", 

396 default=1, 

397 ) 

398 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

399 dtype=str, 

400 doc="List of mask planes that should be interpolated over when applying the brighter-fatter." 

401 "correction.", 

402 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

403 ) 

404 

405 # Dark subtraction. 

406 doDark = pexConfig.Field( 

407 dtype=bool, 

408 doc="Apply dark frame correction.", 

409 default=True, 

410 ) 

411 

412 # Flat correction. 

413 doFlat = pexConfig.Field( 

414 dtype=bool, 

415 doc="Apply flat field correction.", 

416 default=True, 

417 ) 

418 flatScalingType = pexConfig.ChoiceField( 

419 dtype=str, 

420 doc="The method for scaling the flat on the fly.", 

421 default='USER', 

422 allowed={ 

423 "USER": "Scale by flatUserScale", 

424 "MEAN": "Scale by the inverse of the mean", 

425 "MEDIAN": "Scale by the inverse of the median", 

426 }, 

427 ) 

428 flatUserScale = pexConfig.Field( 

429 dtype=float, 

430 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise.", 

431 default=1.0, 

432 ) 

433 

434 # Calculate image quality statistics? 

435 doStandardStatistics = pexConfig.Field( 

436 dtype=bool, 

437 doc="Should standard image quality statistics be calculated?", 

438 default=True, 

439 ) 

440 # Calculate additional statistics? 

441 doCalculateStatistics = pexConfig.Field( 

442 dtype=bool, 

443 doc="Should additional ISR statistics be calculated?", 

444 default=True, 

445 ) 

446 isrStats = pexConfig.ConfigurableField( 

447 target=IsrStatisticsTask, 

448 doc="Task to calculate additional statistics.", 

449 ) 

450 

451 # Make binned images? 

452 doBinnedExposures = pexConfig.Field( 

453 dtype=bool, 

454 doc="Should binned exposures be calculated?", 

455 default=False, 

456 ) 

457 binFactor1 = pexConfig.Field( 457 ↛ exitline 457 didn't jump to the function exit

458 dtype=int, 

459 doc="Binning factor for first binned exposure. This is intended for a finely binned output.", 

460 default=8, 

461 check=lambda x: x > 1, 

462 ) 

463 binFactor2 = pexConfig.Field( 463 ↛ exitline 463 didn't jump to the function exit

464 dtype=int, 

465 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.", 

466 default=64, 

467 check=lambda x: x > 1, 

468 ) 

469 

470 def validate(self): 

471 super().validate() 

472 

473 if self.doCalculateStatistics and self.isrStats.doCtiStatistics: 

474 # DM-41912: Implement doApplyGains in LSST IsrTask 

475 # if self.doApplyGains != 

476 # self.isrStats.doApplyGainsForCtiStatistics: 

477 raise ValueError("doApplyGains must match isrStats.applyGainForCtiStatistics.") 

478 

479 def setDefaults(self): 

480 super().setDefaults() 

481 

482 

483class IsrTaskLSST(pipeBase.PipelineTask): 

484 ConfigClass = IsrTaskLSSTConfig 

485 _DefaultName = "isr" 

486 

487 def __init__(self, **kwargs): 

488 super().__init__(**kwargs) 

489 self.makeSubtask("assembleCcd") 

490 self.makeSubtask("deferredChargeCorrection") 

491 self.makeSubtask("crosstalk") 

492 self.makeSubtask("masking") 

493 self.makeSubtask("isrStats") 

494 

495 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

496 

497 inputs = butlerQC.get(inputRefs) 

498 self.validateInput(inputs) 

499 super().runQuantum(butlerQC, inputRefs, outputRefs) 

500 

501 def validateInput(self, inputs): 

502 """ 

503 This is a check that all the inputs required by the config 

504 are available. 

505 """ 

506 

507 doCrosstalk = self.config.doCrosstalk or self.config.overscanCamera.doAnyParallelOverscanCrosstalk 

508 

509 inputMap = {'dnlLUT': self.config.doDiffNonLinearCorrection, 

510 'bias': self.config.doBias, 

511 'deferredChargeCalib': self.config.doDeferredCharge, 

512 'linearizer': self.config.doLinearize, 

513 'ptc': self.config.doApplyGains, 

514 'crosstalk': doCrosstalk, 

515 'defects': self.config.doDefect, 

516 'bfKernel': self.config.doBrighterFatter, 

517 'dark': self.config.doDark, 

518 } 

519 

520 for calibrationFile, configValue in inputMap.items(): 

521 if configValue and inputs[calibrationFile] is None: 

522 raise RuntimeError("Must supply ", calibrationFile) 

523 

524 def diffNonLinearCorrection(self, ccdExposure, dnlLUT, **kwargs): 

525 # TODO DM 36636 

526 # isrFunctions.diffNonLinearCorrection 

527 pass 

528 

529 def maskFullDefectAmplifiers(self, ccdExposure, detector, defects): 

530 """ 

531 Check for fully masked bad amplifiers and mask them. 

532 

533 Full defect masking happens later to allow for defects which 

534 cross amplifier boundaries. 

535 

536 Parameters 

537 ---------- 

538 ccdExposure : `lsst.afw.image.Exposure` 

539 Input exposure to be masked. 

540 detector : `lsst.afw.cameraGeom.Detector` 

541 Detector object. 

542 defects : `lsst.ip.isr.Defects` 

543 List of defects. Used to determine if an entire 

544 amplifier is bad. 

545 

546 Returns 

547 ------- 

548 badAmpDict : `str`[`bool`] 

549 Dictionary of amplifiers, keyed by name, value is True if 

550 amplifier is fully masked. 

551 """ 

552 badAmpDict = {} 

553 

554 maskedImage = ccdExposure.getMaskedImage() 

555 

556 for amp in detector: 

557 ampName = amp.getName() 

558 badAmpDict[ampName] = False 

559 

560 # Check if entire amp region is defined as a defect 

561 # NB: need to use amp.getBBox() for correct comparison with current 

562 # defects definition. 

563 if defects is not None: 

564 badAmpDict[ampName] = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

565 

566 # In the case of a bad amp, we will set mask to "BAD" 

567 # (here use amp.getRawBBox() for correct association with pixels in 

568 # current ccdExposure). 

569 if badAmpDict[ampName]: 

570 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

571 afwImage.PARENT) 

572 maskView = dataView.getMask() 

573 maskView |= maskView.getPlaneBitMask("BAD") 

574 del maskView 

575 

576 self.log.warning("Amplifier %s is bad (completely covered with defects)", ampName) 

577 

578 return badAmpDict 

579 

580 def maskSaturatedPixels(self, badAmpDict, ccdExposure, detector): 

581 """ 

582 Mask SATURATED and SUSPECT pixels and check if any amplifiers 

583 are fully masked. 

584 

585 Parameters 

586 ---------- 

587 badAmpDict : `str` [`bool`] 

588 Dictionary of amplifiers, keyed by name, value is True if 

589 amplifier is fully masked. 

590 ccdExposure : `lsst.afw.image.Exposure` 

591 Input exposure to be masked. 

592 detector : `lsst.afw.cameraGeom.Detector` 

593 Detector object. 

594 defects : `lsst.ip.isr.Defects` 

595 List of defects. Used to determine if an entire 

596 amplifier is bad. 

597 

598 Returns 

599 ------- 

600 badAmpDict : `str`[`bool`] 

601 Dictionary of amplifiers, keyed by name. 

602 """ 

603 maskedImage = ccdExposure.getMaskedImage() 

604 

605 for amp in detector: 

606 ampName = amp.getName() 

607 

608 if badAmpDict[ampName]: 

609 # No need to check fully bad amplifiers. 

610 continue 

611 

612 # Mask saturated and suspect pixels. 

613 limits = {} 

614 if self.config.doSaturation: 

615 # Set to the default from the camera model. 

616 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

617 # And update if it is set in the config. 

618 if math.isfinite(self.config.saturation): 

619 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

620 if self.config.doSuspect: 

621 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

622 

623 for maskName, maskThreshold in limits.items(): 

624 if not math.isnan(maskThreshold): 

625 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

626 isrFunctions.makeThresholdMask( 

627 maskedImage=dataView, 

628 threshold=maskThreshold, 

629 growFootprints=0, 

630 maskName=maskName 

631 ) 

632 

633 # Determine if we've fully masked this amplifier with SUSPECT and 

634 # SAT pixels. 

635 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

636 afwImage.PARENT) 

637 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

638 self.config.suspectMaskName]) 

639 if numpy.all(maskView.getArray() & maskVal > 0): 

640 self.log.warning("Amplifier %s is bad (completely SATURATED or SUSPECT)", ampName) 

641 badAmpDict[ampName] = True 

642 maskView |= maskView.getPlaneBitMask("BAD") 

643 

644 return badAmpDict 

645 

646 def overscanCorrection(self, mode, detectorConfig, detector, badAmpDict, ccdExposure): 

647 """Apply serial overscan correction in place to all amps. 

648 

649 The actual overscan subtraction is performed by the 

650 `lsst.ip.isr.overscan.OverscanTask`, which is called here. 

651 

652 Parameters 

653 ---------- 

654 mode : `str` 

655 Must be `SERIAL` or `PARALLEL`. 

656 detectorConfig : `lsst.ip.isr.OverscanDetectorConfig` 

657 Per-amplifier configurations. 

658 detector : `lsst.afw.cameraGeom.Detector` 

659 Detector object. 

660 badAmpDict : `dict` 

661 Dictionary of amp name to whether it is a bad amp. 

662 ccdExposure : `lsst.afw.image.Exposure` 

663 Exposure to have overscan correction performed. 

664 

665 Returns 

666 ------- 

667 overscans : `list` [`lsst.pipe.base.Struct` or None] 

668 Each result struct has components: 

669 

670 ``imageFit`` 

671 Value or fit subtracted from the amplifier image data. 

672 (scalar or `lsst.afw.image.Image`) 

673 ``overscanFit`` 

674 Value or fit subtracted from the overscan image data. 

675 (scalar or `lsst.afw.image.Image`) 

676 ``overscanImage`` 

677 Image of the overscan region with the overscan 

678 correction applied. This quantity is used to estimate 

679 the amplifier read noise empirically. 

680 (`lsst.afw.image.Image`) 

681 ``overscanMean`` 

682 Mean overscan fit value. (`float`) 

683 ``overscanMedian`` 

684 Median overscan fit value. (`float`) 

685 ``overscanSigma`` 

686 Clipped standard deviation of the overscan fit. (`float`) 

687 ``residualMean`` 

688 Mean of the overscan after fit subtraction. (`float`) 

689 ``residualMedian`` 

690 Median of the overscan after fit subtraction. (`float`) 

691 ``residualSigma`` 

692 Clipped standard deviation of the overscan after fit 

693 subtraction. (`float`) 

694 

695 See Also 

696 -------- 

697 lsst.ip.isr.overscan.OverscanTask 

698 """ 

699 if mode not in ["SERIAL", "PARALLEL"]: 

700 raise ValueError("Mode must be SERIAL or PARALLEL") 

701 

702 # This returns a list in amp order, with None for uncorrected amps. 

703 overscans = [] 

704 

705 for i, amp in enumerate(detector): 

706 ampName = amp.getName() 

707 

708 ampConfig = detectorConfig.getOverscanAmpConfig(ampName) 

709 

710 if mode == "SERIAL" and not ampConfig.doSerialOverscan: 

711 self.log.debug( 

712 "ISR_OSCAN: Amplifier %s/%s configured to skip serial overscan.", 

713 detector.getName(), 

714 ampName, 

715 ) 

716 results = None 

717 elif mode == "PARALLEL" and not ampConfig.doParallelOverscan: 

718 self.log.debug( 

719 "ISR_OSCAN: Amplifier %s configured to skip parallel overscan.", 

720 detector.getName(), 

721 ampName, 

722 ) 

723 results = None 

724 elif badAmpDict[ampName] or not ccdExposure.getBBox().contains(amp.getBBox()): 

725 results = None 

726 else: 

727 # This check is to confirm that we are not trying to run 

728 # overscan on an already trimmed image. Therefore, always 

729 # checking just the horizontal overscan bounding box is 

730 # sufficient. 

731 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

732 self.log.warning( 

733 "ISR_OSCAN: No overscan region for amp %s. Not performing overscan correction.", 

734 ampName, 

735 ) 

736 results = None 

737 else: 

738 if mode == "SERIAL": 

739 # We need to set up the subtask here with a custom 

740 # configuration. 

741 serialOverscan = SerialOverscanCorrectionTask(config=ampConfig.serialOverscanConfig) 

742 results = serialOverscan.run(ccdExposure, amp) 

743 else: 

744 parallelOverscan = ParallelOverscanCorrectionTask( 

745 config=ampConfig.parallelOverscanConfig, 

746 ) 

747 results = parallelOverscan.run(ccdExposure, amp) 

748 

749 metadata = ccdExposure.getMetadata() 

750 keyBase = "LSST ISR OVERSCAN" 

751 metadata[f"{keyBase} {mode} MEAN {ampName}"] = results.overscanMean 

752 metadata[f"{keyBase} {mode} MEDIAN {ampName}"] = results.overscanMedian 

753 metadata[f"{keyBase} {mode} STDEV {ampName}"] = results.overscanSigma 

754 

755 metadata[f"{keyBase} RESIDUAL {mode} MEAN {ampName}"] = results.residualMean 

756 metadata[f"{keyBase} RESIDUAL {mode} MEDIAN {ampName}"] = results.residualMedian 

757 metadata[f"{keyBase} RESIDUAL {mode} STDEV {ampName}"] = results.residualSigma 

758 

759 overscans[i] = results 

760 

761 # Question: should this be finer grained? 

762 ccdExposure.getMetadata().set("OVERSCAN", "Overscan corrected") 

763 

764 return overscans 

765 

766 def getLinearizer(self, detector): 

767 # Here we assume linearizer as dict or LUT are not supported 

768 # TODO DM 28741 

769 

770 # TODO construct isrcalib input 

771 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

772 self.log.warning("Constructing linearizer from cameraGeom information.") 

773 

774 return linearizer 

775 

776 def gainsCorrection(self, **kwargs): 

777 # TODO DM 36639 

778 gains = [] 

779 readNoise = [] 

780 

781 return gains, readNoise 

782 

783 def updateVariance(self, ampExposure, amp, ptcDataset=None): 

784 """Set the variance plane using the gain and read noise. 

785 

786 Parameters 

787 ---------- 

788 ampExposure : `lsst.afw.image.Exposure` 

789 Exposure to process. 

790 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

791 Amplifier detector data. 

792 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

793 PTC dataset containing the gains and read noise. 

794 

795 Raises 

796 ------ 

797 RuntimeError 

798 Raised if ptcDataset is not provided. 

799 

800 See also 

801 -------- 

802 lsst.ip.isr.isrFunctions.updateVariance 

803 """ 

804 # Get gains from PTC 

805 if ptcDataset is None: 

806 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

807 else: 

808 gain = ptcDataset.gain[amp.getName()] 

809 self.log.debug("Getting gain from Photon Transfer Curve.") 

810 

811 if math.isnan(gain): 

812 gain = 1.0 

813 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

814 elif gain <= 0: 

815 patchedGain = 1.0 

816 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

817 amp.getName(), gain, patchedGain) 

818 gain = patchedGain 

819 

820 # Get read noise from PTC 

821 if ptcDataset is None: 

822 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

823 else: 

824 readNoise = ptcDataset.noise[amp.getName()] 

825 self.log.debug("Getting read noise from Photon Transfer Curve.") 

826 

827 metadata = ampExposure.getMetadata() 

828 metadata[f'LSST GAIN {amp.getName()}'] = gain 

829 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise 

830 

831 isrFunctions.updateVariance( 

832 maskedImage=ampExposure.getMaskedImage(), 

833 gain=gain, 

834 readNoise=readNoise, 

835 ) 

836 

837 def maskNegativeVariance(self, exposure): 

838 """Identify and mask pixels with negative variance values. 

839 

840 Parameters 

841 ---------- 

842 exposure : `lsst.afw.image.Exposure` 

843 Exposure to process. 

844 

845 See Also 

846 -------- 

847 lsst.ip.isr.isrFunctions.updateVariance 

848 """ 

849 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

850 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

851 exposure.mask.array[bad] |= maskPlane 

852 

853 def variancePlane(self, ccdExposure, ccd, ptc): 

854 for amp in ccd: 

855 if ccdExposure.getBBox().contains(amp.getBBox()): 

856 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

857 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

858 

859 self.updateVariance(ampExposure, amp, ptcDataset=ptc) 

860 

861 if self.config.qa is not None and self.config.qa.saveStats is True: 

862 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

863 afwMath.MEDIAN | afwMath.STDEVCLIP) 

864 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

865 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

866 qaStats.getValue(afwMath.STDEVCLIP)) 

867 if self.config.maskNegativeVariance: 

868 self.maskNegativeVariance(ccdExposure) 

869 

870 def maskDefect(self, exposure, defectBaseList): 

871 """Mask defects using mask plane "BAD", in place. 

872 

873 Parameters 

874 ---------- 

875 exposure : `lsst.afw.image.Exposure` 

876 Exposure to process. 

877 

878 defectBaseList : defect-type 

879 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

880 or `list` of `lsst.afw.image.DefectBase`. 

881 """ 

882 maskedImage = exposure.getMaskedImage() 

883 if not isinstance(defectBaseList, Defects): 

884 # Promotes DefectBase to Defect 

885 defectList = Defects(defectBaseList) 

886 else: 

887 defectList = defectBaseList 

888 defectList.maskPixels(maskedImage, maskName="BAD") 

889 

890 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

891 """Mask edge pixels with applicable mask plane. 

892 

893 Parameters 

894 ---------- 

895 exposure : `lsst.afw.image.Exposure` 

896 Exposure to process. 

897 numEdgePixels : `int`, optional 

898 Number of edge pixels to mask. 

899 maskPlane : `str`, optional 

900 Mask plane name to use. 

901 level : `str`, optional 

902 Level at which to mask edges. 

903 """ 

904 maskedImage = exposure.getMaskedImage() 

905 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

906 

907 if numEdgePixels > 0: 

908 if level == 'DETECTOR': 

909 boxes = [maskedImage.getBBox()] 

910 elif level == 'AMP': 

911 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

912 

913 for box in boxes: 

914 # This makes a bbox numEdgeSuspect pixels smaller than the 

915 # image on each side 

916 subImage = maskedImage[box] 

917 box.grow(-numEdgePixels) 

918 # Mask pixels outside box 

919 SourceDetectionTask.setEdgeBits( 

920 subImage, 

921 box, 

922 maskBitMask) 

923 

924 def maskNan(self, exposure): 

925 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

926 

927 Parameters 

928 ---------- 

929 exposure : `lsst.afw.image.Exposure` 

930 Exposure to process. 

931 

932 Notes 

933 ----- 

934 We mask over all non-finite values (NaN, inf), including those 

935 that are masked with other bits (because those may or may not be 

936 interpolated over later, and we want to remove all NaN/infs). 

937 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

938 preserve the historical name. 

939 """ 

940 maskedImage = exposure.getMaskedImage() 

941 

942 # Find and mask NaNs 

943 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

944 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

945 numNans = maskNans(maskedImage, maskVal) 

946 self.metadata["NUMNANS"] = numNans 

947 if numNans > 0: 

948 self.log.warning("There were %d unmasked NaNs.", numNans) 

949 

950 def countBadPixels(self, exposure): 

951 """ 

952 Notes 

953 ----- 

954 Reset and interpolate bad pixels. 

955 

956 Large contiguous bad regions (which should have the BAD mask 

957 bit set) should have their values set to the image median. 

958 This group should include defects and bad amplifiers. As the 

959 area covered by these defects are large, there's little 

960 reason to expect that interpolation would provide a more 

961 useful value. 

962 

963 Smaller defects can be safely interpolated after the larger 

964 regions have had their pixel values reset. This ensures 

965 that the remaining defects adjacent to bad amplifiers (as an 

966 example) do not attempt to interpolate extreme values. 

967 """ 

968 badPixelCount, badPixelValue = isrFunctions.setBadRegions(exposure) 

969 if badPixelCount > 0: 

970 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

971 

972 @contextmanager 

973 def flatContext(self, exp, flat, dark=None): 

974 """Context manager that applies and removes flats and darks, 

975 if the task is configured to apply them. 

976 

977 Parameters 

978 ---------- 

979 exp : `lsst.afw.image.Exposure` 

980 Exposure to process. 

981 flat : `lsst.afw.image.Exposure` 

982 Flat exposure the same size as ``exp``. 

983 dark : `lsst.afw.image.Exposure`, optional 

984 Dark exposure the same size as ``exp``. 

985 

986 Yields 

987 ------ 

988 exp : `lsst.afw.image.Exposure` 

989 The flat and dark corrected exposure. 

990 """ 

991 if self.config.doDark and dark is not None: 

992 self.darkCorrection(exp, dark) 

993 if self.config.doFlat: 

994 self.flatCorrection(exp, flat) 

995 try: 

996 yield exp 

997 finally: 

998 if self.config.doFlat: 

999 self.flatCorrection(exp, flat, invert=True) 

1000 if self.config.doDark and dark is not None: 

1001 self.darkCorrection(exp, dark, invert=True) 

1002 

1003 def getBrighterFatterKernel(self, detector, bfKernel): 

1004 detName = detector.getName() 

1005 

1006 # This is expected to be a dictionary of amp-wise gains. 

1007 bfGains = bfKernel.gain 

1008 if bfKernel.level == 'DETECTOR': 

1009 if detName in bfKernel.detKernels: 

1010 bfKernelOut = bfKernel.detKernels[detName] 

1011 return bfKernelOut, bfGains 

1012 else: 

1013 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1014 elif bfKernel.level == 'AMP': 

1015 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1016 "fatter kernels.") 

1017 bfKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1018 bfKernelOut = bfKernel.detKernels[detName] 

1019 return bfKernelOut, bfGains 

1020 

1021 def applyBrighterFatterCorrection(self, ccdExposure, flat, dark, bfKernel, bfGains): 

1022 # We need to apply flats and darks before we can interpolate, and 

1023 # we need to interpolate before we do B-F, but we do B-F without 

1024 # the flats and darks applied so we can work in units of electrons 

1025 # or holes. This context manager applies and then removes the darks 

1026 # and flats. 

1027 # 

1028 # We also do not want to interpolate values here, so operate on 

1029 # temporary images so we can apply only the BF-correction and roll 

1030 # back the interpolation. 

1031 # This won't be necessary once the gain normalization 

1032 # is done appropriately. 

1033 interpExp = ccdExposure.clone() 

1034 with self.flatContext(interpExp, flat, dark): 

1035 isrFunctions.interpolateFromMask( 

1036 maskedImage=interpExp.getMaskedImage(), 

1037 fwhm=self.config.brighterFatterFwhmForInterpolation, 

1038 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1039 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1040 ) 

1041 bfExp = interpExp.clone() 

1042 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1043 type(bfKernel), type(bfGains)) 

1044 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1045 self.config.brighterFatterMaxIter, 

1046 self.config.brighterFatterThreshold, 

1047 self.config.brighterFatterApplyGain, 

1048 bfGains) 

1049 if bfResults[1] == self.config.brighterFatterMaxIter: 

1050 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1051 bfResults[0]) 

1052 else: 

1053 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1054 bfResults[1]) 

1055 

1056 image = ccdExposure.getMaskedImage().getImage() 

1057 bfCorr = bfExp.getMaskedImage().getImage() 

1058 bfCorr -= interpExp.getMaskedImage().getImage() 

1059 image += bfCorr 

1060 

1061 # Applying the brighter-fatter correction applies a 

1062 # convolution to the science image. At the edges this 

1063 # convolution may not have sufficient valid pixels to 

1064 # produce a valid correction. Mark pixels within the size 

1065 # of the brighter-fatter kernel as EDGE to warn of this 

1066 # fact. 

1067 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1068 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1069 maskPlane="EDGE") 

1070 

1071 if self.config.brighterFatterMaskGrowSize > 0: 

1072 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1073 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1074 isrFunctions.growMasks(ccdExposure.getMask(), 

1075 radius=self.config.brighterFatterMaskGrowSize, 

1076 maskNameList=maskPlane, 

1077 maskValue=maskPlane) 

1078 

1079 return ccdExposure 

1080 

1081 def darkCorrection(self, exposure, darkExposure, invert=False): 

1082 """Apply dark correction in place. 

1083 

1084 Parameters 

1085 ---------- 

1086 exposure : `lsst.afw.image.Exposure` 

1087 Exposure to process. 

1088 darkExposure : `lsst.afw.image.Exposure` 

1089 Dark exposure of the same size as ``exposure``. 

1090 invert : `Bool`, optional 

1091 If True, re-add the dark to an already corrected image. 

1092 

1093 Raises 

1094 ------ 

1095 RuntimeError 

1096 Raised if either ``exposure`` or ``darkExposure`` do not 

1097 have their dark time defined. 

1098 

1099 See Also 

1100 -------- 

1101 lsst.ip.isr.isrFunctions.darkCorrection 

1102 """ 

1103 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

1104 if math.isnan(expScale): 

1105 raise RuntimeError("Exposure darktime is NAN.") 

1106 if darkExposure.getInfo().getVisitInfo() is not None \ 

1107 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

1108 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

1109 else: 

1110 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

1111 # so getDarkTime() does not exist. 

1112 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

1113 darkScale = 1.0 

1114 

1115 isrFunctions.darkCorrection( 

1116 maskedImage=exposure.getMaskedImage(), 

1117 darkMaskedImage=darkExposure.getMaskedImage(), 

1118 expScale=expScale, 

1119 darkScale=darkScale, 

1120 invert=invert, 

1121 ) 

1122 

1123 @staticmethod 

1124 def extractCalibDate(calib): 

1125 """Extract common calibration metadata values that will be written to 

1126 output header. 

1127 

1128 Parameters 

1129 ---------- 

1130 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1131 Calibration to pull date information from. 

1132 

1133 Returns 

1134 ------- 

1135 dateString : `str` 

1136 Calibration creation date string to add to header. 

1137 """ 

1138 if hasattr(calib, "getMetadata"): 

1139 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1140 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1141 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1142 else: 

1143 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1144 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1145 else: 

1146 return "Unknown Unknown" 

1147 

1148 def doLinearize(self, detector): 

1149 """Check if linearization is needed for the detector cameraGeom. 

1150 

1151 Checks config.doLinearize and the linearity type of the first 

1152 amplifier. 

1153 

1154 Parameters 

1155 ---------- 

1156 detector : `lsst.afw.cameraGeom.Detector` 

1157 Detector to get linearity type from. 

1158 

1159 Returns 

1160 ------- 

1161 doLinearize : `Bool` 

1162 If True, linearization should be performed. 

1163 """ 

1164 return self.config.doLinearize and \ 

1165 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

1166 

1167 def flatCorrection(self, exposure, flatExposure, invert=False): 

1168 """Apply flat correction in place. 

1169 

1170 Parameters 

1171 ---------- 

1172 exposure : `lsst.afw.image.Exposure` 

1173 Exposure to process. 

1174 flatExposure : `lsst.afw.image.Exposure` 

1175 Flat exposure of the same size as ``exposure``. 

1176 invert : `Bool`, optional 

1177 If True, unflatten an already flattened image. 

1178 

1179 See Also 

1180 -------- 

1181 lsst.ip.isr.isrFunctions.flatCorrection 

1182 """ 

1183 isrFunctions.flatCorrection( 

1184 maskedImage=exposure.getMaskedImage(), 

1185 flatMaskedImage=flatExposure.getMaskedImage(), 

1186 scalingType=self.config.flatScalingType, 

1187 userScale=self.config.flatUserScale, 

1188 invert=invert 

1189 ) 

1190 

1191 def makeBinnedImages(self, exposure): 

1192 """Make visualizeVisit style binned exposures. 

1193 

1194 Parameters 

1195 ---------- 

1196 exposure : `lsst.afw.image.Exposure` 

1197 Exposure to bin. 

1198 

1199 Returns 

1200 ------- 

1201 bin1 : `lsst.afw.image.Exposure` 

1202 Binned exposure using binFactor1. 

1203 bin2 : `lsst.afw.image.Exposure` 

1204 Binned exposure using binFactor2. 

1205 """ 

1206 mi = exposure.getMaskedImage() 

1207 

1208 bin1 = afwMath.binImage(mi, self.config.binFactor1) 

1209 bin2 = afwMath.binImage(mi, self.config.binFactor2) 

1210 

1211 return bin1, bin2 

1212 

1213 def run(self, *, ccdExposure, dnlLUT=None, bias=None, deferredChargeCalib=None, linearizer=None, 

1214 ptc=None, crosstalk=None, defects=None, bfKernel=None, bfGains=None, dark=None, 

1215 flat=None, camera=None, **kwargs 

1216 ): 

1217 

1218 detector = ccdExposure.getDetector() 

1219 

1220 overscanDetectorConfig = self.config.overscanCamera.getOverscanDetectorConfig(detector) 

1221 

1222 gains = ptc.gain 

1223 

1224 if self.config.doHeaderProvenance: 

1225 # Inputs have been validated, so we can add their date 

1226 # information to the output header. 

1227 exposureMetadata = ccdExposure.getMetadata() 

1228 exposureMetadata["LSST CALIB OVERSCAN HASH"] = overscanDetectorConfig.md5 

1229 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc) 

1230 if self.config.doDiffNonLinearCorrection: 

1231 exposureMetadata["LSST CALIB DATE DNL"] = self.extractCalibDate(dnlLUT) 

1232 if self.config.doBias: 

1233 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias) 

1234 if self.config.doDeferredCharge: 

1235 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib) 

1236 if self.doLinearize(detector): 

1237 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer) 

1238 if self.config.doCrosstalk or overscanDetectorConfig.doAnyParallelOverscanCrosstalk: 

1239 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk) 

1240 if self.config.doDefect: 

1241 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects) 

1242 if self.config.doBrighterFatter: 

1243 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel) 

1244 if self.config.doDark: 

1245 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark) 

1246 if self.config.doFlat: 

1247 exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat) 

1248 

1249 # First we mark which amplifiers are completely bad from defects. 

1250 badAmpDict = self.maskFullDefectAmplifiers(ccdExposure, detector, defects) 

1251 

1252 if self.config.doDiffNonLinearCorrection: 

1253 self.diffNonLinearCorrection(ccdExposure, dnlLUT) 

1254 

1255 if overscanDetectorConfig.doAnySerialOverscan: 

1256 # Input units: ADU 

1257 serialOverscans = self.overscanCorrection( 

1258 "SERIAL", 

1259 overscanDetectorConfig, 

1260 detector, 

1261 badAmpDict, 

1262 ccdExposure, 

1263 ) 

1264 else: 

1265 serialOverscans = [None]*len(detector) 

1266 

1267 if overscanDetectorConfig.doAnyParallelOverscanCrosstalk: 

1268 # Input units: ADU 

1269 # Make sure that the units here are consistent with later 

1270 # application. 

1271 self.crosstalk.run( 

1272 ccdExposure, 

1273 crosstalk=crosstalk, 

1274 camera=camera, 

1275 parallelOverscanRegion=True, 

1276 detectorConfig=overscanDetectorConfig, 

1277 ) 

1278 

1279 # After serial overscan correction, we can mask SATURATED and 

1280 # SUSPECT pixels. This updates badAmpDict if any amplifier 

1281 # is fully saturated after serial overscan correction. 

1282 badAmpDict = self.maskSaturatedPixels(badAmpDict, ccdExposure, detector) 

1283 

1284 if overscanDetectorConfig.doAnyParallelOverscan: 

1285 # Input units: ADU 

1286 # At the moment we do not use the parallelOverscans return value. 

1287 _ = self.overscanCorrection( 

1288 "PARALLEL", 

1289 overscanDetectorConfig, 

1290 detector, 

1291 badAmpDict, 

1292 ccdExposure, 

1293 ) 

1294 

1295 if self.config.doAssembleCcd: 

1296 # Input units: ADU 

1297 self.log.info("Assembling CCD from amplifiers.") 

1298 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1299 

1300 if self.config.expectWcs and not ccdExposure.getWcs(): 

1301 self.log.warning("No WCS found in input exposure.") 

1302 

1303 if self.config.doLinearize: 

1304 # Input units: ADU 

1305 self.log.info("Applying linearizer.") 

1306 linearizer = self.getLinearizer(detector=detector) 

1307 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1308 detector=detector, log=self.log) 

1309 

1310 if self.config.doCrosstalk: 

1311 # Input units: ADU 

1312 self.log.info("Applying crosstalk correction.") 

1313 self.crosstalk.run(ccdExposure, crosstalk=crosstalk) 

1314 

1315 if self.config.doBias: 

1316 # Input units: ADU 

1317 self.log.info("Applying bias correction.") 

1318 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage()) 

1319 

1320 if self.config.doGainsCorrection: 

1321 # TODO DM 36639 

1322 self.log.info("Apply temperature dependence to the gains.") 

1323 gains, readNoise = self.gainsCorrection(**kwargs) 

1324 

1325 if self.config.doApplyGains: 

1326 # Input units: ADU 

1327 # Output units: electrons 

1328 self.log.info("Apply PTC gains (temperature corrected or not) to the image.") 

1329 isrFunctions.applyGains(ccdExposure, normalizeGains=False, ptcGains=gains) 

1330 

1331 if self.config.doDeferredCharge: 

1332 # Input units: electrons 

1333 self.log.info("Applying deferred charge/CTI correction.") 

1334 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1335 

1336 if self.config.doVariance: 

1337 # Input units: electrons 

1338 self.variancePlane(ccdExposure, detector, ptc) 

1339 

1340 # Masking block (defects, NAN pixels and trails). 

1341 # Saturated and suspect pixels have already been masked. 

1342 if self.config.doDefect: 

1343 # Input units: electrons 

1344 self.log.info("Applying defects masking.") 

1345 self.maskDefect(ccdExposure, defects) 

1346 

1347 if self.config.doNanMasking: 

1348 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1349 self.maskNan(ccdExposure) 

1350 

1351 if self.config.doWidenSaturationTrails: 

1352 self.log.info("Widening saturation trails.") 

1353 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1354 

1355 preInterpExp = None 

1356 if self.config.doSaveInterpPixels: 

1357 preInterpExp = ccdExposure.clone() 

1358 

1359 if self.config.doSetBadRegions: 

1360 self.log.info('Counting pixels in BAD regions.') 

1361 self.countBadPixels(ccdExposure) 

1362 

1363 if self.config.doInterpolate: 

1364 self.log.info("Interpolating masked pixels.") 

1365 isrFunctions.interpolateFromMask( 

1366 maskedImage=ccdExposure.getMaskedImage(), 

1367 fwhm=self.config.brighterFatterFwhmForInterpolation, 

1368 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1369 maskNameList=list(self.config.maskListToInterpolate) 

1370 ) 

1371 

1372 if self.config.doDark: 

1373 # Input units: electrons 

1374 self.log.info("Applying dark subtraction.") 

1375 self.darkCorrection(ccdExposure, dark) 

1376 

1377 if self.config.doBrighterFatter: 

1378 # Input units: electrons 

1379 self.log.info("Applying Bright-Fatter kernels.") 

1380 bfKernelOut, bfGains = self.getBrighterFatterKernel(detector, bfKernel) 

1381 ccdExposure = self.applyBrighterFatterCorrection(ccdExposure, flat, dark, bfKernelOut, bfGains) 

1382 

1383 if self.config.doFlat: 

1384 # Input units: electrons 

1385 self.log.info("Applying flat correction.") 

1386 # Placeholder while the LSST flat procedure is done. 

1387 # The flat here would be a background flat. 

1388 self.flatCorrection(ccdExposure, flat) 

1389 

1390 # Calculate standard image quality statistics 

1391 if self.config.doStandardStatistics: 

1392 metadata = ccdExposure.getMetadata() 

1393 for amp in detector: 

1394 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1395 ampName = amp.getName() 

1396 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1397 ampExposure.getMaskedImage(), 

1398 [self.config.saturatedMaskName] 

1399 ) 

1400 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1401 ampExposure.getMaskedImage(), 

1402 ["BAD"] 

1403 ) 

1404 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1405 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1406 

1407 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1408 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1409 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1410 

1411 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1412 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1413 if overscanDetectorConfig.doAnySerialOverscan and k1 in metadata and k2 in metadata: 

1414 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1415 else: 

1416 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1417 

1418 # calculate additional statistics. 

1419 outputStatistics = None 

1420 if self.config.doCalculateStatistics: 

1421 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=serialOverscans, 

1422 bias=bias, dark=dark, flat=flat, ptc=ptc, 

1423 defects=defects).results 

1424 

1425 # do image binning. 

1426 outputBin1Exposure = None 

1427 outputBin2Exposure = None 

1428 if self.config.doBinnedExposures: 

1429 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure) 

1430 

1431 return pipeBase.Struct( 

1432 exposure=ccdExposure, 

1433 

1434 outputBin1Exposure=outputBin1Exposure, 

1435 outputBin2Exposure=outputBin2Exposure, 

1436 

1437 preInterpExposure=preInterpExp, 

1438 outputExposure=ccdExposure, 

1439 outputStatistics=outputStatistics, 

1440 )