Coverage for python/lsst/ip/isr/isrTask.py: 16%

899 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-04 22:12 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from lsst.daf.butler import DimensionGraph 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

75 Data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

95 

96 

97class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

98 dimensions={"instrument", "exposure", "detector"}, 

99 defaultTemplates={}): 

100 ccdExposure = cT.Input( 

101 name="raw", 

102 doc="Input exposure to process.", 

103 storageClass="Exposure", 

104 dimensions=["instrument", "exposure", "detector"], 

105 ) 

106 camera = cT.PrerequisiteInput( 

107 name="camera", 

108 storageClass="Camera", 

109 doc="Input camera to construct complete exposures.", 

110 dimensions=["instrument"], 

111 isCalibration=True, 

112 ) 

113 

114 crosstalk = cT.PrerequisiteInput( 

115 name="crosstalk", 

116 doc="Input crosstalk object", 

117 storageClass="CrosstalkCalib", 

118 dimensions=["instrument", "detector"], 

119 isCalibration=True, 

120 minimum=0, # can fall back to cameraGeom 

121 ) 

122 crosstalkSources = cT.PrerequisiteInput( 

123 name="isrOverscanCorrected", 

124 doc="Overscan corrected input images.", 

125 storageClass="Exposure", 

126 dimensions=["instrument", "exposure", "detector"], 

127 deferLoad=True, 

128 multiple=True, 

129 lookupFunction=crosstalkSourceLookup, 

130 minimum=0, # not needed for all instruments, no config to control this 

131 ) 

132 bias = cT.PrerequisiteInput( 

133 name="bias", 

134 doc="Input bias calibration.", 

135 storageClass="ExposureF", 

136 dimensions=["instrument", "detector"], 

137 isCalibration=True, 

138 ) 

139 dark = cT.PrerequisiteInput( 

140 name='dark', 

141 doc="Input dark calibration.", 

142 storageClass="ExposureF", 

143 dimensions=["instrument", "detector"], 

144 isCalibration=True, 

145 ) 

146 flat = cT.PrerequisiteInput( 

147 name="flat", 

148 doc="Input flat calibration.", 

149 storageClass="ExposureF", 

150 dimensions=["instrument", "physical_filter", "detector"], 

151 isCalibration=True, 

152 ) 

153 ptc = cT.PrerequisiteInput( 

154 name="ptc", 

155 doc="Input Photon Transfer Curve dataset", 

156 storageClass="PhotonTransferCurveDataset", 

157 dimensions=["instrument", "detector"], 

158 isCalibration=True, 

159 ) 

160 fringes = cT.PrerequisiteInput( 

161 name="fringe", 

162 doc="Input fringe calibration.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "physical_filter", "detector"], 

165 isCalibration=True, 

166 minimum=0, # only needed for some bands, even when enabled 

167 ) 

168 strayLightData = cT.PrerequisiteInput( 

169 name='yBackground', 

170 doc="Input stray light calibration.", 

171 storageClass="StrayLightData", 

172 dimensions=["instrument", "physical_filter", "detector"], 

173 deferLoad=True, 

174 isCalibration=True, 

175 minimum=0, # only needed for some bands, even when enabled 

176 ) 

177 bfKernel = cT.PrerequisiteInput( 

178 name='bfKernel', 

179 doc="Input brighter-fatter kernel.", 

180 storageClass="NumpyArray", 

181 dimensions=["instrument"], 

182 isCalibration=True, 

183 minimum=0, # can use either bfKernel or newBFKernel 

184 ) 

185 newBFKernel = cT.PrerequisiteInput( 

186 name='brighterFatterKernel', 

187 doc="Newer complete kernel + gain solutions.", 

188 storageClass="BrighterFatterKernel", 

189 dimensions=["instrument", "detector"], 

190 isCalibration=True, 

191 minimum=0, # can use either bfKernel or newBFKernel 

192 ) 

193 defects = cT.PrerequisiteInput( 

194 name='defects', 

195 doc="Input defect tables.", 

196 storageClass="Defects", 

197 dimensions=["instrument", "detector"], 

198 isCalibration=True, 

199 ) 

200 linearizer = cT.PrerequisiteInput( 

201 name='linearizer', 

202 storageClass="Linearizer", 

203 doc="Linearity correction calibration.", 

204 dimensions=["instrument", "detector"], 

205 isCalibration=True, 

206 minimum=0, # can fall back to cameraGeom 

207 ) 

208 opticsTransmission = cT.PrerequisiteInput( 

209 name="transmission_optics", 

210 storageClass="TransmissionCurve", 

211 doc="Transmission curve due to the optics.", 

212 dimensions=["instrument"], 

213 isCalibration=True, 

214 ) 

215 filterTransmission = cT.PrerequisiteInput( 

216 name="transmission_filter", 

217 storageClass="TransmissionCurve", 

218 doc="Transmission curve due to the filter.", 

219 dimensions=["instrument", "physical_filter"], 

220 isCalibration=True, 

221 ) 

222 sensorTransmission = cT.PrerequisiteInput( 

223 name="transmission_sensor", 

224 storageClass="TransmissionCurve", 

225 doc="Transmission curve due to the sensor.", 

226 dimensions=["instrument", "detector"], 

227 isCalibration=True, 

228 ) 

229 atmosphereTransmission = cT.PrerequisiteInput( 

230 name="transmission_atmosphere", 

231 storageClass="TransmissionCurve", 

232 doc="Transmission curve due to the atmosphere.", 

233 dimensions=["instrument"], 

234 isCalibration=True, 

235 ) 

236 illumMaskedImage = cT.PrerequisiteInput( 

237 name="illum", 

238 doc="Input illumination correction.", 

239 storageClass="MaskedImageF", 

240 dimensions=["instrument", "physical_filter", "detector"], 

241 isCalibration=True, 

242 ) 

243 deferredChargeCalib = cT.PrerequisiteInput( 

244 name="cpCtiCalib", 

245 doc="Deferred charge/CTI correction dataset.", 

246 storageClass="IsrCalib", 

247 dimensions=["instrument", "detector"], 

248 isCalibration=True, 

249 ) 

250 

251 outputExposure = cT.Output( 

252 name='postISRCCD', 

253 doc="Output ISR processed exposure.", 

254 storageClass="Exposure", 

255 dimensions=["instrument", "exposure", "detector"], 

256 ) 

257 preInterpExposure = cT.Output( 

258 name='preInterpISRCCD', 

259 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

260 storageClass="ExposureF", 

261 dimensions=["instrument", "exposure", "detector"], 

262 ) 

263 outputOssThumbnail = cT.Output( 

264 name="OssThumb", 

265 doc="Output Overscan-subtracted thumbnail image.", 

266 storageClass="Thumbnail", 

267 dimensions=["instrument", "exposure", "detector"], 

268 ) 

269 outputFlattenedThumbnail = cT.Output( 

270 name="FlattenedThumb", 

271 doc="Output flat-corrected thumbnail image.", 

272 storageClass="Thumbnail", 

273 dimensions=["instrument", "exposure", "detector"], 

274 ) 

275 outputStatistics = cT.Output( 

276 name="isrStatistics", 

277 doc="Output of additional statistics table.", 

278 storageClass="StructuredDataDict", 

279 dimensions=["instrument", "exposure", "detector"], 

280 ) 

281 

282 def __init__(self, *, config=None): 

283 super().__init__(config=config) 

284 

285 if config.doBias is not True: 

286 self.prerequisiteInputs.remove("bias") 

287 if config.doLinearize is not True: 

288 self.prerequisiteInputs.remove("linearizer") 

289 if config.doCrosstalk is not True: 

290 self.prerequisiteInputs.remove("crosstalkSources") 

291 self.prerequisiteInputs.remove("crosstalk") 

292 if config.doBrighterFatter is not True: 

293 self.prerequisiteInputs.remove("bfKernel") 

294 self.prerequisiteInputs.remove("newBFKernel") 

295 if config.doDefect is not True: 

296 self.prerequisiteInputs.remove("defects") 

297 if config.doDark is not True: 

298 self.prerequisiteInputs.remove("dark") 

299 if config.doFlat is not True: 

300 self.prerequisiteInputs.remove("flat") 

301 if config.doFringe is not True: 

302 self.prerequisiteInputs.remove("fringes") 

303 if config.doStrayLight is not True: 

304 self.prerequisiteInputs.remove("strayLightData") 

305 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

306 self.prerequisiteInputs.remove("ptc") 

307 if config.doAttachTransmissionCurve is not True: 

308 self.prerequisiteInputs.remove("opticsTransmission") 

309 self.prerequisiteInputs.remove("filterTransmission") 

310 self.prerequisiteInputs.remove("sensorTransmission") 

311 self.prerequisiteInputs.remove("atmosphereTransmission") 

312 else: 

313 if config.doUseOpticsTransmission is not True: 

314 self.prerequisiteInputs.remove("opticsTransmission") 

315 if config.doUseFilterTransmission is not True: 

316 self.prerequisiteInputs.remove("filterTransmission") 

317 if config.doUseSensorTransmission is not True: 

318 self.prerequisiteInputs.remove("sensorTransmission") 

319 if config.doUseAtmosphereTransmission is not True: 

320 self.prerequisiteInputs.remove("atmosphereTransmission") 

321 if config.doIlluminationCorrection is not True: 

322 self.prerequisiteInputs.remove("illumMaskedImage") 

323 if config.doDeferredCharge is not True: 

324 self.prerequisiteInputs.remove("deferredChargeCalib") 

325 

326 if config.doWrite is not True: 

327 self.outputs.remove("outputExposure") 

328 self.outputs.remove("preInterpExposure") 

329 self.outputs.remove("outputFlattenedThumbnail") 

330 self.outputs.remove("outputOssThumbnail") 

331 self.outputs.remove("outputStatistics") 

332 

333 if config.doSaveInterpPixels is not True: 

334 self.outputs.remove("preInterpExposure") 

335 if config.qa.doThumbnailOss is not True: 

336 self.outputs.remove("outputOssThumbnail") 

337 if config.qa.doThumbnailFlattened is not True: 

338 self.outputs.remove("outputFlattenedThumbnail") 

339 if config.doCalculateStatistics is not True: 

340 self.outputs.remove("outputStatistics") 

341 

342 

343class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

344 pipelineConnections=IsrTaskConnections): 

345 """Configuration parameters for IsrTask. 

346 

347 Items are grouped in the order in which they are executed by the task. 

348 """ 

349 datasetType = pexConfig.Field( 

350 dtype=str, 

351 doc="Dataset type for input data; users will typically leave this alone, " 

352 "but camera-specific ISR tasks will override it", 

353 default="raw", 

354 ) 

355 

356 fallbackFilterName = pexConfig.Field( 

357 dtype=str, 

358 doc="Fallback default filter name for calibrations.", 

359 optional=True 

360 ) 

361 useFallbackDate = pexConfig.Field( 

362 dtype=bool, 

363 doc="Pass observation date when using fallback filter.", 

364 default=False, 

365 ) 

366 expectWcs = pexConfig.Field( 

367 dtype=bool, 

368 default=True, 

369 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

370 ) 

371 fwhm = pexConfig.Field( 

372 dtype=float, 

373 doc="FWHM of PSF in arcseconds.", 

374 default=1.0, 

375 ) 

376 qa = pexConfig.ConfigField( 

377 dtype=isrQa.IsrQaConfig, 

378 doc="QA related configuration options.", 

379 ) 

380 doHeaderProvenance = pexConfig.Field( 

381 dtype=bool, 

382 default=True, 

383 doc="Write calibration identifiers into output exposure header?", 

384 ) 

385 

386 # Calib checking configuration: 

387 doRaiseOnCalibMismatch = pexConfig.Field( 

388 dtype=bool, 

389 default=False, 

390 doc="Should IsrTask halt if exposure and calibration header values do not match?", 

391 ) 

392 cameraKeywordsToCompare = pexConfig.ListField( 

393 dtype=str, 

394 doc="List of header keywords to compare between exposure and calibrations.", 

395 default=[], 

396 ) 

397 

398 # Image conversion configuration 

399 doConvertIntToFloat = pexConfig.Field( 

400 dtype=bool, 

401 doc="Convert integer raw images to floating point values?", 

402 default=True, 

403 ) 

404 

405 # Saturated pixel handling. 

406 doSaturation = pexConfig.Field( 

407 dtype=bool, 

408 doc="Mask saturated pixels? NB: this is totally independent of the" 

409 " interpolation option - this is ONLY setting the bits in the mask." 

410 " To have them interpolated make sure doSaturationInterpolation=True", 

411 default=True, 

412 ) 

413 saturatedMaskName = pexConfig.Field( 

414 dtype=str, 

415 doc="Name of mask plane to use in saturation detection and interpolation", 

416 default="SAT", 

417 ) 

418 saturation = pexConfig.Field( 

419 dtype=float, 

420 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

421 default=float("NaN"), 

422 ) 

423 growSaturationFootprintSize = pexConfig.Field( 

424 dtype=int, 

425 doc="Number of pixels by which to grow the saturation footprints", 

426 default=1, 

427 ) 

428 

429 # Suspect pixel handling. 

430 doSuspect = pexConfig.Field( 

431 dtype=bool, 

432 doc="Mask suspect pixels?", 

433 default=False, 

434 ) 

435 suspectMaskName = pexConfig.Field( 

436 dtype=str, 

437 doc="Name of mask plane to use for suspect pixels", 

438 default="SUSPECT", 

439 ) 

440 numEdgeSuspect = pexConfig.Field( 

441 dtype=int, 

442 doc="Number of edge pixels to be flagged as untrustworthy.", 

443 default=0, 

444 ) 

445 edgeMaskLevel = pexConfig.ChoiceField( 

446 dtype=str, 

447 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

448 default="DETECTOR", 

449 allowed={ 

450 'DETECTOR': 'Mask only the edges of the full detector.', 

451 'AMP': 'Mask edges of each amplifier.', 

452 }, 

453 ) 

454 

455 # Initial masking options. 

456 doSetBadRegions = pexConfig.Field( 

457 dtype=bool, 

458 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

459 default=True, 

460 ) 

461 badStatistic = pexConfig.ChoiceField( 

462 dtype=str, 

463 doc="How to estimate the average value for BAD regions.", 

464 default='MEANCLIP', 

465 allowed={ 

466 "MEANCLIP": "Correct using the (clipped) mean of good data", 

467 "MEDIAN": "Correct using the median of the good data", 

468 }, 

469 ) 

470 

471 # Overscan subtraction configuration. 

472 doOverscan = pexConfig.Field( 

473 dtype=bool, 

474 doc="Do overscan subtraction?", 

475 default=True, 

476 ) 

477 overscan = pexConfig.ConfigurableField( 

478 target=OverscanCorrectionTask, 

479 doc="Overscan subtraction task for image segments.", 

480 ) 

481 

482 # Amplifier to CCD assembly configuration 

483 doAssembleCcd = pexConfig.Field( 

484 dtype=bool, 

485 default=True, 

486 doc="Assemble amp-level exposures into a ccd-level exposure?" 

487 ) 

488 assembleCcd = pexConfig.ConfigurableField( 

489 target=AssembleCcdTask, 

490 doc="CCD assembly task", 

491 ) 

492 

493 # General calibration configuration. 

494 doAssembleIsrExposures = pexConfig.Field( 

495 dtype=bool, 

496 default=False, 

497 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

498 ) 

499 doTrimToMatchCalib = pexConfig.Field( 

500 dtype=bool, 

501 default=False, 

502 doc="Trim raw data to match calibration bounding boxes?" 

503 ) 

504 

505 # Bias subtraction. 

506 doBias = pexConfig.Field( 

507 dtype=bool, 

508 doc="Apply bias frame correction?", 

509 default=True, 

510 ) 

511 biasDataProductName = pexConfig.Field( 

512 dtype=str, 

513 doc="Name of the bias data product", 

514 default="bias", 

515 ) 

516 doBiasBeforeOverscan = pexConfig.Field( 

517 dtype=bool, 

518 doc="Reverse order of overscan and bias correction.", 

519 default=False 

520 ) 

521 

522 # Deferred charge correction. 

523 doDeferredCharge = pexConfig.Field( 

524 dtype=bool, 

525 doc="Apply deferred charge correction?", 

526 default=False, 

527 ) 

528 deferredChargeCorrection = pexConfig.ConfigurableField( 

529 target=DeferredChargeTask, 

530 doc="Deferred charge correction task.", 

531 ) 

532 

533 # Variance construction 

534 doVariance = pexConfig.Field( 

535 dtype=bool, 

536 doc="Calculate variance?", 

537 default=True 

538 ) 

539 gain = pexConfig.Field( 

540 dtype=float, 

541 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

542 default=float("NaN"), 

543 ) 

544 readNoise = pexConfig.Field( 

545 dtype=float, 

546 doc="The read noise to use if no Detector is present in the Exposure", 

547 default=0.0, 

548 ) 

549 doEmpiricalReadNoise = pexConfig.Field( 

550 dtype=bool, 

551 default=False, 

552 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

553 ) 

554 usePtcReadNoise = pexConfig.Field( 

555 dtype=bool, 

556 default=False, 

557 doc="Use readnoise values from the Photon Transfer Curve?" 

558 ) 

559 maskNegativeVariance = pexConfig.Field( 

560 dtype=bool, 

561 default=True, 

562 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

563 "in the measurement of the overscan at an edge due to the data falling off faster " 

564 "than the overscan model can account for it." 

565 ) 

566 negativeVarianceMaskName = pexConfig.Field( 

567 dtype=str, 

568 default="BAD", 

569 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

570 ) 

571 # Linearization. 

572 doLinearize = pexConfig.Field( 

573 dtype=bool, 

574 doc="Correct for nonlinearity of the detector's response?", 

575 default=True, 

576 ) 

577 

578 # Crosstalk. 

579 doCrosstalk = pexConfig.Field( 

580 dtype=bool, 

581 doc="Apply intra-CCD crosstalk correction?", 

582 default=False, 

583 ) 

584 doCrosstalkBeforeAssemble = pexConfig.Field( 

585 dtype=bool, 

586 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

587 default=False, 

588 ) 

589 crosstalk = pexConfig.ConfigurableField( 

590 target=CrosstalkTask, 

591 doc="Intra-CCD crosstalk correction", 

592 ) 

593 

594 # Masking options. 

595 doDefect = pexConfig.Field( 

596 dtype=bool, 

597 doc="Apply correction for CCD defects, e.g. hot pixels?", 

598 default=True, 

599 ) 

600 doNanMasking = pexConfig.Field( 

601 dtype=bool, 

602 doc="Mask non-finite (NAN, inf) pixels?", 

603 default=True, 

604 ) 

605 doWidenSaturationTrails = pexConfig.Field( 

606 dtype=bool, 

607 doc="Widen bleed trails based on their width?", 

608 default=True 

609 ) 

610 

611 # Brighter-Fatter correction. 

612 doBrighterFatter = pexConfig.Field( 

613 dtype=bool, 

614 default=False, 

615 doc="Apply the brighter-fatter correction?" 

616 ) 

617 doFluxConservingBrighterFatterCorrection = pexConfig.Field( 

618 dtype=bool, 

619 default=False, 

620 doc="Apply the flux-conserving BFE correction by Miller et al.?" 

621 ) 

622 brighterFatterLevel = pexConfig.ChoiceField( 

623 dtype=str, 

624 default="DETECTOR", 

625 doc="The level at which to correct for brighter-fatter.", 

626 allowed={ 

627 "AMP": "Every amplifier treated separately.", 

628 "DETECTOR": "One kernel per detector", 

629 } 

630 ) 

631 brighterFatterMaxIter = pexConfig.Field( 

632 dtype=int, 

633 default=10, 

634 doc="Maximum number of iterations for the brighter-fatter correction" 

635 ) 

636 brighterFatterThreshold = pexConfig.Field( 

637 dtype=float, 

638 default=1000, 

639 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

640 "absolute value of the difference between the current corrected image and the one " 

641 "from the previous iteration summed over all the pixels." 

642 ) 

643 brighterFatterApplyGain = pexConfig.Field( 

644 dtype=bool, 

645 default=True, 

646 doc="Should the gain be applied when applying the brighter-fatter correction?" 

647 ) 

648 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

649 dtype=str, 

650 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

651 "correction.", 

652 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

653 ) 

654 brighterFatterMaskGrowSize = pexConfig.Field( 

655 dtype=int, 

656 default=0, 

657 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

658 "when brighter-fatter correction is applied." 

659 ) 

660 

661 # Dark subtraction. 

662 doDark = pexConfig.Field( 

663 dtype=bool, 

664 doc="Apply dark frame correction?", 

665 default=True, 

666 ) 

667 darkDataProductName = pexConfig.Field( 

668 dtype=str, 

669 doc="Name of the dark data product", 

670 default="dark", 

671 ) 

672 

673 # Camera-specific stray light removal. 

674 doStrayLight = pexConfig.Field( 

675 dtype=bool, 

676 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

677 default=False, 

678 ) 

679 strayLight = pexConfig.ConfigurableField( 

680 target=StrayLightTask, 

681 doc="y-band stray light correction" 

682 ) 

683 

684 # Flat correction. 

685 doFlat = pexConfig.Field( 

686 dtype=bool, 

687 doc="Apply flat field correction?", 

688 default=True, 

689 ) 

690 flatDataProductName = pexConfig.Field( 

691 dtype=str, 

692 doc="Name of the flat data product", 

693 default="flat", 

694 ) 

695 flatScalingType = pexConfig.ChoiceField( 

696 dtype=str, 

697 doc="The method for scaling the flat on the fly.", 

698 default='USER', 

699 allowed={ 

700 "USER": "Scale by flatUserScale", 

701 "MEAN": "Scale by the inverse of the mean", 

702 "MEDIAN": "Scale by the inverse of the median", 

703 }, 

704 ) 

705 flatUserScale = pexConfig.Field( 

706 dtype=float, 

707 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

708 default=1.0, 

709 ) 

710 doTweakFlat = pexConfig.Field( 

711 dtype=bool, 

712 doc="Tweak flats to match observed amplifier ratios?", 

713 default=False 

714 ) 

715 

716 # Amplifier normalization based on gains instead of using flats 

717 # configuration. 

718 doApplyGains = pexConfig.Field( 

719 dtype=bool, 

720 doc="Correct the amplifiers for their gains instead of applying flat correction", 

721 default=False, 

722 ) 

723 usePtcGains = pexConfig.Field( 

724 dtype=bool, 

725 doc="Use the gain values from the Photon Transfer Curve?", 

726 default=False, 

727 ) 

728 normalizeGains = pexConfig.Field( 

729 dtype=bool, 

730 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

731 default=False, 

732 ) 

733 

734 # Fringe correction. 

735 doFringe = pexConfig.Field( 

736 dtype=bool, 

737 doc="Apply fringe correction?", 

738 default=True, 

739 ) 

740 fringe = pexConfig.ConfigurableField( 

741 target=FringeTask, 

742 doc="Fringe subtraction task", 

743 ) 

744 fringeAfterFlat = pexConfig.Field( 

745 dtype=bool, 

746 doc="Do fringe subtraction after flat-fielding?", 

747 default=True, 

748 ) 

749 

750 # Amp offset correction. 

751 doAmpOffset = pexConfig.Field( 

752 doc="Calculate and apply amp offset corrections?", 

753 dtype=bool, 

754 default=False, 

755 ) 

756 ampOffset = pexConfig.ConfigurableField( 

757 doc="Amp offset correction task.", 

758 target=AmpOffsetTask, 

759 ) 

760 

761 # Initial CCD-level background statistics options. 

762 doMeasureBackground = pexConfig.Field( 

763 dtype=bool, 

764 doc="Measure the background level on the reduced image?", 

765 default=False, 

766 ) 

767 

768 # Camera-specific masking configuration. 

769 doCameraSpecificMasking = pexConfig.Field( 

770 dtype=bool, 

771 doc="Mask camera-specific bad regions?", 

772 default=False, 

773 ) 

774 masking = pexConfig.ConfigurableField( 

775 target=MaskingTask, 

776 doc="Masking task." 

777 ) 

778 

779 # Interpolation options. 

780 doInterpolate = pexConfig.Field( 

781 dtype=bool, 

782 doc="Interpolate masked pixels?", 

783 default=True, 

784 ) 

785 doSaturationInterpolation = pexConfig.Field( 

786 dtype=bool, 

787 doc="Perform interpolation over pixels masked as saturated?" 

788 " NB: This is independent of doSaturation; if that is False this plane" 

789 " will likely be blank, resulting in a no-op here.", 

790 default=True, 

791 ) 

792 doNanInterpolation = pexConfig.Field( 

793 dtype=bool, 

794 doc="Perform interpolation over pixels masked as NaN?" 

795 " NB: This is independent of doNanMasking; if that is False this plane" 

796 " will likely be blank, resulting in a no-op here.", 

797 default=True, 

798 ) 

799 doNanInterpAfterFlat = pexConfig.Field( 

800 dtype=bool, 

801 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

802 "also have to interpolate them before flat-fielding."), 

803 default=False, 

804 ) 

805 maskListToInterpolate = pexConfig.ListField( 

806 dtype=str, 

807 doc="List of mask planes that should be interpolated.", 

808 default=['SAT', 'BAD'], 

809 ) 

810 doSaveInterpPixels = pexConfig.Field( 

811 dtype=bool, 

812 doc="Save a copy of the pre-interpolated pixel values?", 

813 default=False, 

814 ) 

815 

816 # Default photometric calibration options. 

817 fluxMag0T1 = pexConfig.DictField( 

818 keytype=str, 

819 itemtype=float, 

820 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

821 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

822 )) 

823 ) 

824 defaultFluxMag0T1 = pexConfig.Field( 

825 dtype=float, 

826 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

827 default=pow(10.0, 0.4*28.0) 

828 ) 

829 

830 # Vignette correction configuration. 

831 doVignette = pexConfig.Field( 

832 dtype=bool, 

833 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

834 "according to vignetting parameters?"), 

835 default=False, 

836 ) 

837 doMaskVignettePolygon = pexConfig.Field( 

838 dtype=bool, 

839 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

840 "is False"), 

841 default=True, 

842 ) 

843 vignetteValue = pexConfig.Field( 

844 dtype=float, 

845 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

846 optional=True, 

847 default=None, 

848 ) 

849 vignette = pexConfig.ConfigurableField( 

850 target=VignetteTask, 

851 doc="Vignetting task.", 

852 ) 

853 

854 # Transmission curve configuration. 

855 doAttachTransmissionCurve = pexConfig.Field( 

856 dtype=bool, 

857 default=False, 

858 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

859 ) 

860 doUseOpticsTransmission = pexConfig.Field( 

861 dtype=bool, 

862 default=True, 

863 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

864 ) 

865 doUseFilterTransmission = pexConfig.Field( 

866 dtype=bool, 

867 default=True, 

868 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

869 ) 

870 doUseSensorTransmission = pexConfig.Field( 

871 dtype=bool, 

872 default=True, 

873 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

874 ) 

875 doUseAtmosphereTransmission = pexConfig.Field( 

876 dtype=bool, 

877 default=True, 

878 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

879 ) 

880 

881 # Illumination correction. 

882 doIlluminationCorrection = pexConfig.Field( 

883 dtype=bool, 

884 default=False, 

885 doc="Perform illumination correction?" 

886 ) 

887 illuminationCorrectionDataProductName = pexConfig.Field( 

888 dtype=str, 

889 doc="Name of the illumination correction data product.", 

890 default="illumcor", 

891 ) 

892 illumScale = pexConfig.Field( 

893 dtype=float, 

894 doc="Scale factor for the illumination correction.", 

895 default=1.0, 

896 ) 

897 illumFilters = pexConfig.ListField( 

898 dtype=str, 

899 default=[], 

900 doc="Only perform illumination correction for these filters." 

901 ) 

902 

903 # Calculate image quality statistics? 

904 doStandardStatistics = pexConfig.Field( 

905 dtype=bool, 

906 doc="Should standard image quality statistics be calculated?", 

907 default=True, 

908 ) 

909 # Calculate additional statistics? 

910 doCalculateStatistics = pexConfig.Field( 

911 dtype=bool, 

912 doc="Should additional ISR statistics be calculated?", 

913 default=False, 

914 ) 

915 isrStats = pexConfig.ConfigurableField( 

916 target=IsrStatisticsTask, 

917 doc="Task to calculate additional statistics.", 

918 ) 

919 

920 # Write the outputs to disk. If ISR is run as a subtask, this may not 

921 # be needed. 

922 doWrite = pexConfig.Field( 

923 dtype=bool, 

924 doc="Persist postISRCCD?", 

925 default=True, 

926 ) 

927 

928 def validate(self): 

929 super().validate() 

930 if self.doFlat and self.doApplyGains: 

931 raise ValueError("You may not specify both doFlat and doApplyGains") 

932 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

933 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

934 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

935 self.maskListToInterpolate.append(self.saturatedMaskName) 

936 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

937 self.maskListToInterpolate.remove(self.saturatedMaskName) 

938 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

939 self.maskListToInterpolate.append("UNMASKEDNAN") 

940 

941 

942class IsrTask(pipeBase.PipelineTask): 

943 """Apply common instrument signature correction algorithms to a raw frame. 

944 

945 The process for correcting imaging data is very similar from 

946 camera to camera. This task provides a vanilla implementation of 

947 doing these corrections, including the ability to turn certain 

948 corrections off if they are not needed. The inputs to the primary 

949 method, `run()`, are a raw exposure to be corrected and the 

950 calibration data products. The raw input is a single chip sized 

951 mosaic of all amps including overscans and other non-science 

952 pixels. 

953 

954 The __init__ method sets up the subtasks for ISR processing, using 

955 the defaults from `lsst.ip.isr`. 

956 

957 Parameters 

958 ---------- 

959 args : `list` 

960 Positional arguments passed to the Task constructor. 

961 None used at this time. 

962 kwargs : `dict`, optional 

963 Keyword arguments passed on to the Task constructor. 

964 None used at this time. 

965 """ 

966 ConfigClass = IsrTaskConfig 

967 _DefaultName = "isr" 

968 

969 def __init__(self, **kwargs): 

970 super().__init__(**kwargs) 

971 self.makeSubtask("assembleCcd") 

972 self.makeSubtask("crosstalk") 

973 self.makeSubtask("strayLight") 

974 self.makeSubtask("fringe") 

975 self.makeSubtask("masking") 

976 self.makeSubtask("overscan") 

977 self.makeSubtask("vignette") 

978 self.makeSubtask("ampOffset") 

979 self.makeSubtask("deferredChargeCorrection") 

980 self.makeSubtask("isrStats") 

981 

982 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

983 inputs = butlerQC.get(inputRefs) 

984 

985 try: 

986 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

987 except Exception as e: 

988 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

989 (inputRefs, e)) 

990 

991 detector = inputs['ccdExposure'].getDetector() 

992 

993 if self.config.doCrosstalk is True: 

994 # Crosstalk sources need to be defined by the pipeline 

995 # yaml if they exist. 

996 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

997 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

998 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

999 else: 

1000 coeffVector = (self.config.crosstalk.crosstalkValues 

1001 if self.config.crosstalk.useConfigCoefficients else None) 

1002 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1003 inputs['crosstalk'] = crosstalkCalib 

1004 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1005 if 'crosstalkSources' not in inputs: 

1006 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1007 

1008 if self.doLinearize(detector) is True: 

1009 if 'linearizer' in inputs: 

1010 if isinstance(inputs['linearizer'], dict): 

1011 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1012 linearizer.fromYaml(inputs['linearizer']) 

1013 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1014 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1015 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1016 detector=detector, 

1017 log=self.log) 

1018 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1019 else: 

1020 linearizer = inputs['linearizer'] 

1021 linearizer.log = self.log 

1022 inputs['linearizer'] = linearizer 

1023 else: 

1024 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1025 self.log.warning("Constructing linearizer from cameraGeom information.") 

1026 

1027 if self.config.doDefect is True: 

1028 if "defects" in inputs and inputs['defects'] is not None: 

1029 # defects is loaded as a BaseCatalog with columns 

1030 # x0, y0, width, height. Masking expects a list of defects 

1031 # defined by their bounding box 

1032 if not isinstance(inputs["defects"], Defects): 

1033 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1034 

1035 # Load the correct style of brighter-fatter kernel, and repack 

1036 # the information as a numpy array. 

1037 if self.config.doBrighterFatter: 

1038 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1039 if brighterFatterKernel is None: 

1040 # This type of kernel must be in (y, x) index 

1041 # ordering, as it used directly as the .array 

1042 # component of the afwImage kernel. 

1043 brighterFatterKernel = inputs.get('bfKernel', None) 

1044 

1045 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1046 # This is a ISR calib kernel. These kernels are 

1047 # generated in (x, y) index ordering, and need to be 

1048 # transposed to be used directly as the .array 

1049 # component of the afwImage kernel. This is done 

1050 # explicitly below when setting the ``bfKernel`` 

1051 # input. 

1052 detName = detector.getName() 

1053 level = brighterFatterKernel.level 

1054 

1055 # This is expected to be a dictionary of amp-wise gains. 

1056 inputs['bfGains'] = brighterFatterKernel.gain 

1057 if self.config.brighterFatterLevel == 'DETECTOR': 

1058 kernel = None 

1059 if level == 'DETECTOR': 

1060 if detName in brighterFatterKernel.detKernels: 

1061 kernel = brighterFatterKernel.detKernels[detName] 

1062 else: 

1063 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1064 elif level == 'AMP': 

1065 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1066 "fatter kernels.") 

1067 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1068 kernel = brighterFatterKernel.detKernels[detName] 

1069 if kernel is None: 

1070 raise RuntimeError("Could not identify brighter-fatter kernel!") 

1071 # Do the one single transpose here so the kernel 

1072 # can be directly loaded into the afwImage .array 

1073 # component. 

1074 inputs['bfKernel'] = numpy.transpose(kernel) 

1075 elif self.config.brighterFatterLevel == 'AMP': 

1076 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1077 

1078 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1079 expId = inputs['ccdExposure'].info.id 

1080 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1081 expId=expId, 

1082 assembler=self.assembleCcd 

1083 if self.config.doAssembleIsrExposures else None) 

1084 else: 

1085 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1086 

1087 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1088 if 'strayLightData' not in inputs: 

1089 inputs['strayLightData'] = None 

1090 

1091 if self.config.doHeaderProvenance: 

1092 # Add calibration provenanace info to header. 

1093 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1094 for inputName in sorted(inputs.keys()): 

1095 reference = getattr(inputRefs, inputName, None) 

1096 if reference is not None and hasattr(reference, "run"): 

1097 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1098 runValue = reference.run 

1099 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1100 idValue = str(reference.id) 

1101 

1102 exposureMetadata[runKey] = runValue 

1103 exposureMetadata[idKey] = idValue 

1104 

1105 outputs = self.run(**inputs) 

1106 butlerQC.put(outputs, outputRefs) 

1107 

1108 @timeMethod 

1109 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1110 crosstalk=None, crosstalkSources=None, 

1111 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1112 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1113 sensorTransmission=None, atmosphereTransmission=None, 

1114 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1115 deferredChargeCalib=None, 

1116 ): 

1117 """Perform instrument signature removal on an exposure. 

1118 

1119 Steps included in the ISR processing, in order performed, are: 

1120 

1121 - saturation and suspect pixel masking 

1122 - overscan subtraction 

1123 - CCD assembly of individual amplifiers 

1124 - bias subtraction 

1125 - variance image construction 

1126 - linearization of non-linear response 

1127 - crosstalk masking 

1128 - brighter-fatter correction 

1129 - dark subtraction 

1130 - fringe correction 

1131 - stray light subtraction 

1132 - flat correction 

1133 - masking of known defects and camera specific features 

1134 - vignette calculation 

1135 - appending transmission curve and distortion model 

1136 

1137 Parameters 

1138 ---------- 

1139 ccdExposure : `lsst.afw.image.Exposure` 

1140 The raw exposure that is to be run through ISR. The 

1141 exposure is modified by this method. 

1142 camera : `lsst.afw.cameraGeom.Camera`, optional 

1143 The camera geometry for this exposure. Required if 

1144 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1145 ``flat`` does not have an associated detector. 

1146 bias : `lsst.afw.image.Exposure`, optional 

1147 Bias calibration frame. 

1148 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1149 Functor for linearization. 

1150 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1151 Calibration for crosstalk. 

1152 crosstalkSources : `list`, optional 

1153 List of possible crosstalk sources. 

1154 dark : `lsst.afw.image.Exposure`, optional 

1155 Dark calibration frame. 

1156 flat : `lsst.afw.image.Exposure`, optional 

1157 Flat calibration frame. 

1158 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1159 Photon transfer curve dataset, with, e.g., gains 

1160 and read noise. 

1161 bfKernel : `numpy.ndarray`, optional 

1162 Brighter-fatter kernel. 

1163 bfGains : `dict` of `float`, optional 

1164 Gains used to override the detector's nominal gains for the 

1165 brighter-fatter correction. A dict keyed by amplifier name for 

1166 the detector in question. 

1167 defects : `lsst.ip.isr.Defects`, optional 

1168 List of defects. 

1169 fringes : `lsst.pipe.base.Struct`, optional 

1170 Struct containing the fringe correction data, with 

1171 elements: 

1172 

1173 ``fringes`` 

1174 fringe calibration frame (`lsst.afw.image.Exposure`) 

1175 ``seed`` 

1176 random seed derived from the ``ccdExposureId`` for random 

1177 number generator (`numpy.uint32`) 

1178 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1179 A ``TransmissionCurve`` that represents the throughput of the, 

1180 optics, to be evaluated in focal-plane coordinates. 

1181 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1182 A ``TransmissionCurve`` that represents the throughput of the 

1183 filter itself, to be evaluated in focal-plane coordinates. 

1184 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1185 A ``TransmissionCurve`` that represents the throughput of the 

1186 sensor itself, to be evaluated in post-assembly trimmed detector 

1187 coordinates. 

1188 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1189 A ``TransmissionCurve`` that represents the throughput of the 

1190 atmosphere, assumed to be spatially constant. 

1191 detectorNum : `int`, optional 

1192 The integer number for the detector to process. 

1193 strayLightData : `object`, optional 

1194 Opaque object containing calibration information for stray-light 

1195 correction. If `None`, no correction will be performed. 

1196 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1197 Illumination correction image. 

1198 

1199 Returns 

1200 ------- 

1201 result : `lsst.pipe.base.Struct` 

1202 Result struct with component: 

1203 

1204 ``exposure`` 

1205 The fully ISR corrected exposure. 

1206 (`lsst.afw.image.Exposure`) 

1207 ``outputExposure`` 

1208 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1209 ``ossThumb`` 

1210 Thumbnail image of the exposure after overscan subtraction. 

1211 (`numpy.ndarray`) 

1212 ``flattenedThumb`` 

1213 Thumbnail image of the exposure after flat-field correction. 

1214 (`numpy.ndarray`) 

1215 ``outputStatistics`` 

1216 Values of the additional statistics calculated. 

1217 

1218 Raises 

1219 ------ 

1220 RuntimeError 

1221 Raised if a configuration option is set to `True`, but the 

1222 required calibration data has not been specified. 

1223 

1224 Notes 

1225 ----- 

1226 The current processed exposure can be viewed by setting the 

1227 appropriate `lsstDebug` entries in the ``debug.display`` 

1228 dictionary. The names of these entries correspond to some of 

1229 the `IsrTaskConfig` Boolean options, with the value denoting the 

1230 frame to use. The exposure is shown inside the matching 

1231 option check and after the processing of that step has 

1232 finished. The steps with debug points are: 

1233 

1234 * doAssembleCcd 

1235 * doBias 

1236 * doCrosstalk 

1237 * doBrighterFatter 

1238 * doDark 

1239 * doFringe 

1240 * doStrayLight 

1241 * doFlat 

1242 

1243 In addition, setting the ``postISRCCD`` entry displays the 

1244 exposure after all ISR processing has finished. 

1245 """ 

1246 

1247 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1248 bias = self.ensureExposure(bias, camera, detectorNum) 

1249 dark = self.ensureExposure(dark, camera, detectorNum) 

1250 flat = self.ensureExposure(flat, camera, detectorNum) 

1251 

1252 ccd = ccdExposure.getDetector() 

1253 filterLabel = ccdExposure.getFilter() 

1254 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1255 

1256 if not ccd: 

1257 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1258 ccd = [FakeAmp(ccdExposure, self.config)] 

1259 

1260 # Validate Input 

1261 if self.config.doBias and bias is None: 

1262 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1263 if self.doLinearize(ccd) and linearizer is None: 

1264 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1265 if self.config.doBrighterFatter and bfKernel is None: 

1266 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1267 if self.config.doDark and dark is None: 

1268 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1269 if self.config.doFlat and flat is None: 

1270 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1271 if self.config.doDefect and defects is None: 

1272 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1273 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1274 and fringes.fringes is None): 

1275 # The `fringes` object needs to be a pipeBase.Struct, as 

1276 # we use it as a `dict` for the parameters of 

1277 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1278 # not be `None` if `doFringe=True`. Otherwise, raise. 

1279 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1280 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1281 and illumMaskedImage is None): 

1282 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1283 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1284 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1285 

1286 if self.config.doHeaderProvenance: 

1287 # Inputs have been validated, so we can add their date 

1288 # information to the output header. 

1289 exposureMetadata = ccdExposure.getMetadata() 

1290 if self.config.doBias: 

1291 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias) 

1292 self.compareCameraKeywords(exposureMetadata, bias, "bias") 

1293 if self.config.doBrighterFatter: 

1294 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel) 

1295 self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter") 

1296 if self.config.doCrosstalk: 

1297 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk) 

1298 self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk") 

1299 if self.config.doDark: 

1300 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark) 

1301 self.compareCameraKeywords(exposureMetadata, dark, "dark") 

1302 if self.config.doDefect: 

1303 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects) 

1304 self.compareCameraKeywords(exposureMetadata, defects, "defects") 

1305 if self.config.doDeferredCharge: 

1306 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib) 

1307 self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI") 

1308 if self.config.doFlat: 

1309 exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat) 

1310 self.compareCameraKeywords(exposureMetadata, flat, "flat") 

1311 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1312 exposureMetadata["LSST CALIB DATE FRINGE"] = self.extractCalibDate(fringes.fringes) 

1313 self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe") 

1314 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1315 exposureMetadata["LSST CALIB DATE ILLUMINATION"] = self.extractCalibDate(illumMaskedImage) 

1316 self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination") 

1317 if self.doLinearize(ccd): 

1318 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer) 

1319 self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer") 

1320 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1321 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc) 

1322 self.compareCameraKeywords(exposureMetadata, ptc, "PTC") 

1323 if self.config.doStrayLight: 

1324 exposureMetadata["LSST CALIB DATE STRAYLIGHT"] = self.extractCalibDate(strayLightData) 

1325 self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight") 

1326 if self.config.doAttachTransmissionCurve: 

1327 exposureMetadata["LSST CALIB DATE OPTICS_TR"] = self.extractCalibDate(opticsTransmission) 

1328 exposureMetadata["LSST CALIB DATE FILTER_TR"] = self.extractCalibDate(filterTransmission) 

1329 exposureMetadata["LSST CALIB DATE SENSOR_TR"] = self.extractCalibDate(sensorTransmission) 

1330 exposureMetadata["LSST CALIB DATE ATMOSP_TR"] = self.extractCalibDate(atmosphereTransmission) 

1331 

1332 # Begin ISR processing. 

1333 if self.config.doConvertIntToFloat: 

1334 self.log.info("Converting exposure to floating point values.") 

1335 ccdExposure = self.convertIntToFloat(ccdExposure) 

1336 

1337 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1338 self.log.info("Applying bias correction.") 

1339 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1340 trimToFit=self.config.doTrimToMatchCalib) 

1341 self.debugView(ccdExposure, "doBias") 

1342 

1343 # Amplifier level processing. 

1344 overscans = [] 

1345 

1346 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1347 # This will attempt to mask bleed pixels across all amplifiers. 

1348 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1349 

1350 for amp in ccd: 

1351 # if ccdExposure is one amp, 

1352 # check for coverage to prevent performing ops multiple times 

1353 if ccdExposure.getBBox().contains(amp.getBBox()): 

1354 # Check for fully masked bad amplifiers, 

1355 # and generate masks for SUSPECT and SATURATED values. 

1356 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1357 

1358 if self.config.doOverscan and not badAmp: 

1359 # Overscan correction on amp-by-amp basis. 

1360 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1361 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1362 if overscanResults is not None and \ 

1363 self.config.qa is not None and self.config.qa.saveStats is True: 

1364 if isinstance(overscanResults.overscanMean, float): 

1365 # Only serial overscan was run 

1366 mean = overscanResults.overscanMean 

1367 sigma = overscanResults.overscanSigma 

1368 residMean = overscanResults.residualMean 

1369 residSigma = overscanResults.residualSigma 

1370 else: 

1371 # Both serial and parallel overscan were 

1372 # run. Only report serial here. 

1373 mean = overscanResults.overscanMean[0] 

1374 sigma = overscanResults.overscanSigma[0] 

1375 residMean = overscanResults.residualMean[0] 

1376 residSigma = overscanResults.residualSigma[0] 

1377 

1378 self.metadata[f"FIT MEDIAN {amp.getName()}"] = mean 

1379 self.metadata[f"FIT STDEV {amp.getName()}"] = sigma 

1380 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1381 amp.getName(), mean, sigma) 

1382 

1383 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = residMean 

1384 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = residSigma 

1385 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1386 amp.getName(), residMean, residSigma) 

1387 

1388 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1389 else: 

1390 if badAmp: 

1391 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1392 overscanResults = None 

1393 

1394 overscans.append(overscanResults if overscanResults is not None else None) 

1395 else: 

1396 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1397 

1398 if self.config.doDeferredCharge: 

1399 self.log.info("Applying deferred charge/CTI correction.") 

1400 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1401 self.debugView(ccdExposure, "doDeferredCharge") 

1402 

1403 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1404 self.log.info("Applying crosstalk correction.") 

1405 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1406 crosstalkSources=crosstalkSources, camera=camera) 

1407 self.debugView(ccdExposure, "doCrosstalk") 

1408 

1409 if self.config.doAssembleCcd: 

1410 self.log.info("Assembling CCD from amplifiers.") 

1411 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1412 

1413 if self.config.expectWcs and not ccdExposure.getWcs(): 

1414 self.log.warning("No WCS found in input exposure.") 

1415 self.debugView(ccdExposure, "doAssembleCcd") 

1416 

1417 ossThumb = None 

1418 if self.config.qa.doThumbnailOss: 

1419 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1420 

1421 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1422 self.log.info("Applying bias correction.") 

1423 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1424 trimToFit=self.config.doTrimToMatchCalib) 

1425 self.debugView(ccdExposure, "doBias") 

1426 

1427 if self.config.doVariance: 

1428 for amp, overscanResults in zip(ccd, overscans): 

1429 if ccdExposure.getBBox().contains(amp.getBBox()): 

1430 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1431 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1432 if overscanResults is not None: 

1433 self.updateVariance(ampExposure, amp, 

1434 overscanImage=overscanResults.overscanImage, 

1435 ptcDataset=ptc) 

1436 else: 

1437 self.updateVariance(ampExposure, amp, 

1438 overscanImage=None, 

1439 ptcDataset=ptc) 

1440 if self.config.qa is not None and self.config.qa.saveStats is True: 

1441 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1442 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1443 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1444 qaStats.getValue(afwMath.MEDIAN) 

1445 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1446 qaStats.getValue(afwMath.STDEVCLIP) 

1447 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1448 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1449 qaStats.getValue(afwMath.STDEVCLIP)) 

1450 if self.config.maskNegativeVariance: 

1451 self.maskNegativeVariance(ccdExposure) 

1452 

1453 if self.doLinearize(ccd): 

1454 self.log.info("Applying linearizer.") 

1455 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1456 detector=ccd, log=self.log) 

1457 

1458 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1459 self.log.info("Applying crosstalk correction.") 

1460 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1461 crosstalkSources=crosstalkSources, isTrimmed=True) 

1462 self.debugView(ccdExposure, "doCrosstalk") 

1463 

1464 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1465 # widen trails, and do anything else the camera needs. Saturated and 

1466 # suspect pixels have already been masked. 

1467 if self.config.doDefect: 

1468 self.log.info("Masking defects.") 

1469 self.maskDefect(ccdExposure, defects) 

1470 

1471 if self.config.numEdgeSuspect > 0: 

1472 self.log.info("Masking edges as SUSPECT.") 

1473 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1474 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1475 

1476 if self.config.doNanMasking: 

1477 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1478 self.maskNan(ccdExposure) 

1479 

1480 if self.config.doWidenSaturationTrails: 

1481 self.log.info("Widening saturation trails.") 

1482 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1483 

1484 if self.config.doCameraSpecificMasking: 

1485 self.log.info("Masking regions for camera specific reasons.") 

1486 self.masking.run(ccdExposure) 

1487 

1488 if self.config.doBrighterFatter: 

1489 # We need to apply flats and darks before we can interpolate, and 

1490 # we need to interpolate before we do B-F, but we do B-F without 

1491 # the flats and darks applied so we can work in units of electrons 

1492 # or holes. This context manager applies and then removes the darks 

1493 # and flats. 

1494 # 

1495 # We also do not want to interpolate values here, so operate on 

1496 # temporary images so we can apply only the BF-correction and roll 

1497 # back the interpolation. 

1498 interpExp = ccdExposure.clone() 

1499 with self.flatContext(interpExp, flat, dark): 

1500 isrFunctions.interpolateFromMask( 

1501 maskedImage=interpExp.getMaskedImage(), 

1502 fwhm=self.config.fwhm, 

1503 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1504 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1505 ) 

1506 bfExp = interpExp.clone() 

1507 

1508 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1509 type(bfKernel), type(bfGains)) 

1510 if self.config.doFluxConservingBrighterFatterCorrection: 

1511 bfResults = isrFunctions.fluxConservingBrighterFatterCorrection( 

1512 bfExp, 

1513 bfKernel, 

1514 self.config.brighterFatterMaxIter, 

1515 self.config.brighterFatterThreshold, 

1516 self.config.brighterFatterApplyGain, 

1517 bfGains 

1518 ) 

1519 else: 

1520 bfResults = isrFunctions.brighterFatterCorrection( 

1521 bfExp, 

1522 bfKernel, 

1523 self.config.brighterFatterMaxIter, 

1524 self.config.brighterFatterThreshold, 

1525 self.config.brighterFatterApplyGain, 

1526 bfGains 

1527 ) 

1528 if bfResults[1] == self.config.brighterFatterMaxIter: 

1529 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1530 bfResults[0]) 

1531 else: 

1532 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1533 bfResults[1]) 

1534 image = ccdExposure.getMaskedImage().getImage() 

1535 bfCorr = bfExp.getMaskedImage().getImage() 

1536 bfCorr -= interpExp.getMaskedImage().getImage() 

1537 image += bfCorr 

1538 

1539 # Applying the brighter-fatter correction applies a 

1540 # convolution to the science image. At the edges this 

1541 # convolution may not have sufficient valid pixels to 

1542 # produce a valid correction. Mark pixels within the size 

1543 # of the brighter-fatter kernel as EDGE to warn of this 

1544 # fact. 

1545 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1546 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1547 maskPlane="EDGE") 

1548 

1549 if self.config.brighterFatterMaskGrowSize > 0: 

1550 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1551 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1552 isrFunctions.growMasks(ccdExposure.getMask(), 

1553 radius=self.config.brighterFatterMaskGrowSize, 

1554 maskNameList=maskPlane, 

1555 maskValue=maskPlane) 

1556 

1557 self.debugView(ccdExposure, "doBrighterFatter") 

1558 

1559 if self.config.doDark: 

1560 self.log.info("Applying dark correction.") 

1561 self.darkCorrection(ccdExposure, dark) 

1562 self.debugView(ccdExposure, "doDark") 

1563 

1564 if self.config.doFringe and not self.config.fringeAfterFlat: 

1565 self.log.info("Applying fringe correction before flat.") 

1566 self.fringe.run(ccdExposure, **fringes.getDict()) 

1567 self.debugView(ccdExposure, "doFringe") 

1568 

1569 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1570 self.log.info("Checking strayLight correction.") 

1571 self.strayLight.run(ccdExposure, strayLightData) 

1572 self.debugView(ccdExposure, "doStrayLight") 

1573 

1574 if self.config.doFlat: 

1575 self.log.info("Applying flat correction.") 

1576 self.flatCorrection(ccdExposure, flat) 

1577 self.debugView(ccdExposure, "doFlat") 

1578 

1579 if self.config.doApplyGains: 

1580 self.log.info("Applying gain correction instead of flat.") 

1581 if self.config.usePtcGains: 

1582 self.log.info("Using gains from the Photon Transfer Curve.") 

1583 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1584 ptcGains=ptc.gain) 

1585 else: 

1586 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1587 

1588 if self.config.doFringe and self.config.fringeAfterFlat: 

1589 self.log.info("Applying fringe correction after flat.") 

1590 self.fringe.run(ccdExposure, **fringes.getDict()) 

1591 

1592 if self.config.doVignette: 

1593 if self.config.doMaskVignettePolygon: 

1594 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1595 else: 

1596 self.log.info("Constructing and attaching vignette polygon.") 

1597 self.vignettePolygon = self.vignette.run( 

1598 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1599 vignetteValue=self.config.vignetteValue, log=self.log) 

1600 

1601 if self.config.doAttachTransmissionCurve: 

1602 self.log.info("Adding transmission curves.") 

1603 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1604 filterTransmission=filterTransmission, 

1605 sensorTransmission=sensorTransmission, 

1606 atmosphereTransmission=atmosphereTransmission) 

1607 

1608 flattenedThumb = None 

1609 if self.config.qa.doThumbnailFlattened: 

1610 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1611 

1612 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1613 self.log.info("Performing illumination correction.") 

1614 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1615 illumMaskedImage, illumScale=self.config.illumScale, 

1616 trimToFit=self.config.doTrimToMatchCalib) 

1617 

1618 preInterpExp = None 

1619 if self.config.doSaveInterpPixels: 

1620 preInterpExp = ccdExposure.clone() 

1621 

1622 # Reset and interpolate bad pixels. 

1623 # 

1624 # Large contiguous bad regions (which should have the BAD mask 

1625 # bit set) should have their values set to the image median. 

1626 # This group should include defects and bad amplifiers. As the 

1627 # area covered by these defects are large, there's little 

1628 # reason to expect that interpolation would provide a more 

1629 # useful value. 

1630 # 

1631 # Smaller defects can be safely interpolated after the larger 

1632 # regions have had their pixel values reset. This ensures 

1633 # that the remaining defects adjacent to bad amplifiers (as an 

1634 # example) do not attempt to interpolate extreme values. 

1635 if self.config.doSetBadRegions: 

1636 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1637 if badPixelCount > 0: 

1638 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1639 

1640 if self.config.doInterpolate: 

1641 self.log.info("Interpolating masked pixels.") 

1642 isrFunctions.interpolateFromMask( 

1643 maskedImage=ccdExposure.getMaskedImage(), 

1644 fwhm=self.config.fwhm, 

1645 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1646 maskNameList=list(self.config.maskListToInterpolate) 

1647 ) 

1648 

1649 self.roughZeroPoint(ccdExposure) 

1650 

1651 # correct for amp offsets within the CCD 

1652 if self.config.doAmpOffset: 

1653 self.log.info("Correcting amp offsets.") 

1654 self.ampOffset.run(ccdExposure) 

1655 

1656 if self.config.doMeasureBackground: 

1657 self.log.info("Measuring background level.") 

1658 self.measureBackground(ccdExposure, self.config.qa) 

1659 

1660 if self.config.qa is not None and self.config.qa.saveStats is True: 

1661 for amp in ccd: 

1662 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1663 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1664 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1665 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1666 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1667 qaStats.getValue(afwMath.STDEVCLIP) 

1668 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1669 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1670 qaStats.getValue(afwMath.STDEVCLIP)) 

1671 

1672 # Calculate standard image quality statistics 

1673 if self.config.doStandardStatistics: 

1674 metadata = ccdExposure.getMetadata() 

1675 for amp in ccd: 

1676 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1677 ampName = amp.getName() 

1678 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1679 ampExposure.getMaskedImage(), 

1680 [self.config.saturatedMaskName] 

1681 ) 

1682 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1683 ampExposure.getMaskedImage(), 

1684 ["BAD"] 

1685 ) 

1686 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1687 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1688 

1689 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1690 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1691 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1692 

1693 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1694 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1695 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1696 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1697 else: 

1698 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1699 

1700 # calculate additional statistics. 

1701 outputStatistics = None 

1702 if self.config.doCalculateStatistics: 

1703 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1704 ptc=ptc).results 

1705 

1706 self.debugView(ccdExposure, "postISRCCD") 

1707 

1708 return pipeBase.Struct( 

1709 exposure=ccdExposure, 

1710 ossThumb=ossThumb, 

1711 flattenedThumb=flattenedThumb, 

1712 

1713 preInterpExposure=preInterpExp, 

1714 outputExposure=ccdExposure, 

1715 outputOssThumbnail=ossThumb, 

1716 outputFlattenedThumbnail=flattenedThumb, 

1717 outputStatistics=outputStatistics, 

1718 ) 

1719 

1720 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1721 """Ensure that the data returned by Butler is a fully constructed exp. 

1722 

1723 ISR requires exposure-level image data for historical reasons, so if we 

1724 did not recieve that from Butler, construct it from what we have, 

1725 modifying the input in place. 

1726 

1727 Parameters 

1728 ---------- 

1729 inputExp : `lsst.afw.image` image-type. 

1730 The input data structure obtained from Butler. 

1731 Can be `lsst.afw.image.Exposure`, 

1732 `lsst.afw.image.DecoratedImageU`, 

1733 or `lsst.afw.image.ImageF` 

1734 camera : `lsst.afw.cameraGeom.camera`, optional 

1735 The camera associated with the image. Used to find the appropriate 

1736 detector if detector is not already set. 

1737 detectorNum : `int`, optional 

1738 The detector in the camera to attach, if the detector is not 

1739 already set. 

1740 

1741 Returns 

1742 ------- 

1743 inputExp : `lsst.afw.image.Exposure` 

1744 The re-constructed exposure, with appropriate detector parameters. 

1745 

1746 Raises 

1747 ------ 

1748 TypeError 

1749 Raised if the input data cannot be used to construct an exposure. 

1750 """ 

1751 if isinstance(inputExp, afwImage.DecoratedImageU): 

1752 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1753 elif isinstance(inputExp, afwImage.ImageF): 

1754 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1755 elif isinstance(inputExp, afwImage.MaskedImageF): 

1756 inputExp = afwImage.makeExposure(inputExp) 

1757 elif isinstance(inputExp, afwImage.Exposure): 

1758 pass 

1759 elif inputExp is None: 

1760 # Assume this will be caught by the setup if it is a problem. 

1761 return inputExp 

1762 else: 

1763 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1764 (type(inputExp), )) 

1765 

1766 if inputExp.getDetector() is None: 

1767 if camera is None or detectorNum is None: 

1768 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1769 'without a detector set.') 

1770 inputExp.setDetector(camera[detectorNum]) 

1771 

1772 return inputExp 

1773 

1774 @staticmethod 

1775 def extractCalibDate(calib): 

1776 """Extract common calibration metadata values that will be written to 

1777 output header. 

1778 

1779 Parameters 

1780 ---------- 

1781 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1782 Calibration to pull date information from. 

1783 

1784 Returns 

1785 ------- 

1786 dateString : `str` 

1787 Calibration creation date string to add to header. 

1788 """ 

1789 if hasattr(calib, "getMetadata"): 

1790 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1791 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1792 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1793 else: 

1794 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1795 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1796 else: 

1797 return "Unknown Unknown" 

1798 

1799 def compareCameraKeywords(self, exposureMetadata, calib, calibName): 

1800 """Compare header keywords to confirm camera states match. 

1801 

1802 Parameters 

1803 ---------- 

1804 exposureMetadata : `lsst.daf.base.PropertySet` 

1805 Header for the exposure being processed. 

1806 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1807 Calibration to be applied. 

1808 calibName : `str` 

1809 Calib type for log message. 

1810 """ 

1811 try: 

1812 calibMetadata = calib.getMetadata() 

1813 except AttributeError: 

1814 return 

1815 for keyword in self.config.cameraKeywordsToCompare: 

1816 if keyword in exposureMetadata and keyword in calibMetadata: 

1817 if exposureMetadata[keyword] != calibMetadata[keyword]: 

1818 if self.config.doRaiseOnCalibMismatch: 

1819 raise RuntimeError("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1820 calibName, keyword, 

1821 exposureMetadata[keyword], calibMetadata[keyword]) 

1822 else: 

1823 self.log.warning("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1824 calibName, keyword, 

1825 exposureMetadata[keyword], calibMetadata[keyword]) 

1826 else: 

1827 self.log.debug("Sequencer keyword %s not found.", keyword) 

1828 

1829 def convertIntToFloat(self, exposure): 

1830 """Convert exposure image from uint16 to float. 

1831 

1832 If the exposure does not need to be converted, the input is 

1833 immediately returned. For exposures that are converted to use 

1834 floating point pixels, the variance is set to unity and the 

1835 mask to zero. 

1836 

1837 Parameters 

1838 ---------- 

1839 exposure : `lsst.afw.image.Exposure` 

1840 The raw exposure to be converted. 

1841 

1842 Returns 

1843 ------- 

1844 newexposure : `lsst.afw.image.Exposure` 

1845 The input ``exposure``, converted to floating point pixels. 

1846 

1847 Raises 

1848 ------ 

1849 RuntimeError 

1850 Raised if the exposure type cannot be converted to float. 

1851 

1852 """ 

1853 if isinstance(exposure, afwImage.ExposureF): 

1854 # Nothing to be done 

1855 self.log.debug("Exposure already of type float.") 

1856 return exposure 

1857 if not hasattr(exposure, "convertF"): 

1858 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1859 

1860 newexposure = exposure.convertF() 

1861 newexposure.variance[:] = 1 

1862 newexposure.mask[:] = 0x0 

1863 

1864 return newexposure 

1865 

1866 def maskAmplifier(self, ccdExposure, amp, defects): 

1867 """Identify bad amplifiers, saturated and suspect pixels. 

1868 

1869 Parameters 

1870 ---------- 

1871 ccdExposure : `lsst.afw.image.Exposure` 

1872 Input exposure to be masked. 

1873 amp : `lsst.afw.cameraGeom.Amplifier` 

1874 Catalog of parameters defining the amplifier on this 

1875 exposure to mask. 

1876 defects : `lsst.ip.isr.Defects` 

1877 List of defects. Used to determine if the entire 

1878 amplifier is bad. 

1879 

1880 Returns 

1881 ------- 

1882 badAmp : `Bool` 

1883 If this is true, the entire amplifier area is covered by 

1884 defects and unusable. 

1885 

1886 """ 

1887 maskedImage = ccdExposure.getMaskedImage() 

1888 

1889 badAmp = False 

1890 

1891 # Check if entire amp region is defined as a defect 

1892 # NB: need to use amp.getBBox() for correct comparison with current 

1893 # defects definition. 

1894 if defects is not None: 

1895 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1896 

1897 # In the case of a bad amp, we will set mask to "BAD" 

1898 # (here use amp.getRawBBox() for correct association with pixels in 

1899 # current ccdExposure). 

1900 if badAmp: 

1901 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1902 afwImage.PARENT) 

1903 maskView = dataView.getMask() 

1904 maskView |= maskView.getPlaneBitMask("BAD") 

1905 del maskView 

1906 return badAmp 

1907 

1908 # Mask remaining defects after assembleCcd() to allow for defects that 

1909 # cross amplifier boundaries. Saturation and suspect pixels can be 

1910 # masked now, though. 

1911 limits = dict() 

1912 if self.config.doSaturation and not badAmp: 

1913 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1914 if self.config.doSuspect and not badAmp: 

1915 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1916 if math.isfinite(self.config.saturation): 

1917 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1918 

1919 for maskName, maskThreshold in limits.items(): 

1920 if not math.isnan(maskThreshold): 

1921 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1922 isrFunctions.makeThresholdMask( 

1923 maskedImage=dataView, 

1924 threshold=maskThreshold, 

1925 growFootprints=0, 

1926 maskName=maskName 

1927 ) 

1928 

1929 # Determine if we've fully masked this amplifier with SUSPECT and 

1930 # SAT pixels. 

1931 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1932 afwImage.PARENT) 

1933 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1934 self.config.suspectMaskName]) 

1935 if numpy.all(maskView.getArray() & maskVal > 0): 

1936 badAmp = True 

1937 maskView |= maskView.getPlaneBitMask("BAD") 

1938 

1939 return badAmp 

1940 

1941 def overscanCorrection(self, ccdExposure, amp): 

1942 """Apply overscan correction in place. 

1943 

1944 This method does initial pixel rejection of the overscan 

1945 region. The overscan can also be optionally segmented to 

1946 allow for discontinuous overscan responses to be fit 

1947 separately. The actual overscan subtraction is performed by 

1948 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

1949 after the amplifier is preprocessed. 

1950 

1951 Parameters 

1952 ---------- 

1953 ccdExposure : `lsst.afw.image.Exposure` 

1954 Exposure to have overscan correction performed. 

1955 amp : `lsst.afw.cameraGeom.Amplifer` 

1956 The amplifier to consider while correcting the overscan. 

1957 

1958 Returns 

1959 ------- 

1960 overscanResults : `lsst.pipe.base.Struct` 

1961 Result struct with components: 

1962 

1963 ``imageFit`` 

1964 Value or fit subtracted from the amplifier image data. 

1965 (scalar or `lsst.afw.image.Image`) 

1966 ``overscanFit`` 

1967 Value or fit subtracted from the overscan image data. 

1968 (scalar or `lsst.afw.image.Image`) 

1969 ``overscanImage`` 

1970 Image of the overscan region with the overscan 

1971 correction applied. This quantity is used to estimate 

1972 the amplifier read noise empirically. 

1973 (`lsst.afw.image.Image`) 

1974 ``edgeMask`` 

1975 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

1976 ``overscanMean`` 

1977 Median overscan fit value. (`float`) 

1978 ``overscanSigma`` 

1979 Clipped standard deviation of the overscan after 

1980 correction. (`float`) 

1981 

1982 Raises 

1983 ------ 

1984 RuntimeError 

1985 Raised if the ``amp`` does not contain raw pixel information. 

1986 

1987 See Also 

1988 -------- 

1989 lsst.ip.isr.overscan.OverscanTask 

1990 """ 

1991 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1992 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1993 return None 

1994 

1995 # Perform overscan correction on subregions. 

1996 overscanResults = self.overscan.run(ccdExposure, amp) 

1997 

1998 metadata = ccdExposure.getMetadata() 

1999 ampName = amp.getName() 

2000 

2001 keyBase = "LSST ISR OVERSCAN" 

2002 # Updated quantities 

2003 if isinstance(overscanResults.overscanMean, float): 

2004 # Serial overscan correction only: 

2005 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

2006 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

2007 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

2008 

2009 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

2010 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

2011 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

2012 elif isinstance(overscanResults.overscanMean, tuple): 

2013 # Both serial and parallel overscan have run: 

2014 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

2015 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

2016 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

2017 

2018 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

2019 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

2020 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

2021 

2022 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

2023 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

2024 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

2025 

2026 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

2027 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

2028 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

2029 else: 

2030 self.log.warning("Unexpected type for overscan values; none added to header.") 

2031 

2032 return overscanResults 

2033 

2034 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2035 """Set the variance plane using the gain and read noise 

2036 

2037 The read noise is calculated from the ``overscanImage`` if the 

2038 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2039 the value from the amplifier data is used. 

2040 

2041 Parameters 

2042 ---------- 

2043 ampExposure : `lsst.afw.image.Exposure` 

2044 Exposure to process. 

2045 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

2046 Amplifier detector data. 

2047 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2048 Image of overscan, required only for empirical read noise. 

2049 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2050 PTC dataset containing the gains and read noise. 

2051 

2052 Raises 

2053 ------ 

2054 RuntimeError 

2055 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2056 are ``True``, but ptcDataset is not provided. 

2057 

2058 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2059 ``overscanImage`` is ``None``. 

2060 

2061 See also 

2062 -------- 

2063 lsst.ip.isr.isrFunctions.updateVariance 

2064 """ 

2065 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2066 if self.config.usePtcGains: 

2067 if ptcDataset is None: 

2068 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2069 else: 

2070 gain = ptcDataset.gain[amp.getName()] 

2071 self.log.info("Using gain from Photon Transfer Curve.") 

2072 else: 

2073 gain = amp.getGain() 

2074 

2075 if math.isnan(gain): 

2076 gain = 1.0 

2077 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2078 elif gain <= 0: 

2079 patchedGain = 1.0 

2080 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2081 amp.getName(), gain, patchedGain) 

2082 gain = patchedGain 

2083 

2084 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2085 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(), 

2086 [self.config.saturatedMaskName, 

2087 self.config.suspectMaskName, 

2088 "BAD", "NO_DATA"]) 

2089 allPixels = ampExposure.getWidth() * ampExposure.getHeight() 

2090 if allPixels == badPixels: 

2091 # If the image is bad, do not raise. 

2092 self.log.info("Skipping empirical read noise for amp %s. No good pixels.", 

2093 amp.getName()) 

2094 else: 

2095 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2096 

2097 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2098 stats = afwMath.StatisticsControl() 

2099 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2100 readNoise = afwMath.makeStatistics(overscanImage.getImage(), 

2101 afwMath.STDEVCLIP, stats).getValue() 

2102 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2103 amp.getName(), readNoise) 

2104 elif self.config.usePtcReadNoise: 

2105 if ptcDataset is None: 

2106 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2107 else: 

2108 readNoise = ptcDataset.noise[amp.getName()] 

2109 self.log.info("Using read noise from Photon Transfer Curve.") 

2110 else: 

2111 readNoise = amp.getReadNoise() 

2112 

2113 metadata = ampExposure.getMetadata() 

2114 metadata[f'LSST GAIN {amp.getName()}'] = gain 

2115 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise 

2116 

2117 isrFunctions.updateVariance( 

2118 maskedImage=ampExposure.getMaskedImage(), 

2119 gain=gain, 

2120 readNoise=readNoise, 

2121 ) 

2122 

2123 def maskNegativeVariance(self, exposure): 

2124 """Identify and mask pixels with negative variance values. 

2125 

2126 Parameters 

2127 ---------- 

2128 exposure : `lsst.afw.image.Exposure` 

2129 Exposure to process. 

2130 

2131 See Also 

2132 -------- 

2133 lsst.ip.isr.isrFunctions.updateVariance 

2134 """ 

2135 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2136 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2137 exposure.mask.array[bad] |= maskPlane 

2138 

2139 def darkCorrection(self, exposure, darkExposure, invert=False): 

2140 """Apply dark correction in place. 

2141 

2142 Parameters 

2143 ---------- 

2144 exposure : `lsst.afw.image.Exposure` 

2145 Exposure to process. 

2146 darkExposure : `lsst.afw.image.Exposure` 

2147 Dark exposure of the same size as ``exposure``. 

2148 invert : `Bool`, optional 

2149 If True, re-add the dark to an already corrected image. 

2150 

2151 Raises 

2152 ------ 

2153 RuntimeError 

2154 Raised if either ``exposure`` or ``darkExposure`` do not 

2155 have their dark time defined. 

2156 

2157 See Also 

2158 -------- 

2159 lsst.ip.isr.isrFunctions.darkCorrection 

2160 """ 

2161 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2162 if math.isnan(expScale): 

2163 raise RuntimeError("Exposure darktime is NAN.") 

2164 if darkExposure.getInfo().getVisitInfo() is not None \ 

2165 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2166 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2167 else: 

2168 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2169 # so getDarkTime() does not exist. 

2170 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2171 darkScale = 1.0 

2172 

2173 isrFunctions.darkCorrection( 

2174 maskedImage=exposure.getMaskedImage(), 

2175 darkMaskedImage=darkExposure.getMaskedImage(), 

2176 expScale=expScale, 

2177 darkScale=darkScale, 

2178 invert=invert, 

2179 trimToFit=self.config.doTrimToMatchCalib 

2180 ) 

2181 

2182 def doLinearize(self, detector): 

2183 """Check if linearization is needed for the detector cameraGeom. 

2184 

2185 Checks config.doLinearize and the linearity type of the first 

2186 amplifier. 

2187 

2188 Parameters 

2189 ---------- 

2190 detector : `lsst.afw.cameraGeom.Detector` 

2191 Detector to get linearity type from. 

2192 

2193 Returns 

2194 ------- 

2195 doLinearize : `Bool` 

2196 If True, linearization should be performed. 

2197 """ 

2198 return self.config.doLinearize and \ 

2199 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2200 

2201 def flatCorrection(self, exposure, flatExposure, invert=False): 

2202 """Apply flat correction in place. 

2203 

2204 Parameters 

2205 ---------- 

2206 exposure : `lsst.afw.image.Exposure` 

2207 Exposure to process. 

2208 flatExposure : `lsst.afw.image.Exposure` 

2209 Flat exposure of the same size as ``exposure``. 

2210 invert : `Bool`, optional 

2211 If True, unflatten an already flattened image. 

2212 

2213 See Also 

2214 -------- 

2215 lsst.ip.isr.isrFunctions.flatCorrection 

2216 """ 

2217 isrFunctions.flatCorrection( 

2218 maskedImage=exposure.getMaskedImage(), 

2219 flatMaskedImage=flatExposure.getMaskedImage(), 

2220 scalingType=self.config.flatScalingType, 

2221 userScale=self.config.flatUserScale, 

2222 invert=invert, 

2223 trimToFit=self.config.doTrimToMatchCalib 

2224 ) 

2225 

2226 def saturationDetection(self, exposure, amp): 

2227 """Detect and mask saturated pixels in config.saturatedMaskName. 

2228 

2229 Parameters 

2230 ---------- 

2231 exposure : `lsst.afw.image.Exposure` 

2232 Exposure to process. Only the amplifier DataSec is processed. 

2233 amp : `lsst.afw.cameraGeom.Amplifier` 

2234 Amplifier detector data. 

2235 

2236 See Also 

2237 -------- 

2238 lsst.ip.isr.isrFunctions.makeThresholdMask 

2239 """ 

2240 if not math.isnan(amp.getSaturation()): 

2241 maskedImage = exposure.getMaskedImage() 

2242 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2243 isrFunctions.makeThresholdMask( 

2244 maskedImage=dataView, 

2245 threshold=amp.getSaturation(), 

2246 growFootprints=0, 

2247 maskName=self.config.saturatedMaskName, 

2248 ) 

2249 

2250 def saturationInterpolation(self, exposure): 

2251 """Interpolate over saturated pixels, in place. 

2252 

2253 This method should be called after `saturationDetection`, to 

2254 ensure that the saturated pixels have been identified in the 

2255 SAT mask. It should also be called after `assembleCcd`, since 

2256 saturated regions may cross amplifier boundaries. 

2257 

2258 Parameters 

2259 ---------- 

2260 exposure : `lsst.afw.image.Exposure` 

2261 Exposure to process. 

2262 

2263 See Also 

2264 -------- 

2265 lsst.ip.isr.isrTask.saturationDetection 

2266 lsst.ip.isr.isrFunctions.interpolateFromMask 

2267 """ 

2268 isrFunctions.interpolateFromMask( 

2269 maskedImage=exposure.getMaskedImage(), 

2270 fwhm=self.config.fwhm, 

2271 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2272 maskNameList=list(self.config.saturatedMaskName), 

2273 ) 

2274 

2275 def suspectDetection(self, exposure, amp): 

2276 """Detect and mask suspect pixels in config.suspectMaskName. 

2277 

2278 Parameters 

2279 ---------- 

2280 exposure : `lsst.afw.image.Exposure` 

2281 Exposure to process. Only the amplifier DataSec is processed. 

2282 amp : `lsst.afw.cameraGeom.Amplifier` 

2283 Amplifier detector data. 

2284 

2285 See Also 

2286 -------- 

2287 lsst.ip.isr.isrFunctions.makeThresholdMask 

2288 

2289 Notes 

2290 ----- 

2291 Suspect pixels are pixels whose value is greater than 

2292 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2293 affected by unknown systematics; for example if non-linearity 

2294 corrections above a certain level are unstable then that would be a 

2295 useful value for suspectLevel. A value of `nan` indicates that no such 

2296 level exists and no pixels are to be masked as suspicious. 

2297 """ 

2298 suspectLevel = amp.getSuspectLevel() 

2299 if math.isnan(suspectLevel): 

2300 return 

2301 

2302 maskedImage = exposure.getMaskedImage() 

2303 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2304 isrFunctions.makeThresholdMask( 

2305 maskedImage=dataView, 

2306 threshold=suspectLevel, 

2307 growFootprints=0, 

2308 maskName=self.config.suspectMaskName, 

2309 ) 

2310 

2311 def maskDefect(self, exposure, defectBaseList): 

2312 """Mask defects using mask plane "BAD", in place. 

2313 

2314 Parameters 

2315 ---------- 

2316 exposure : `lsst.afw.image.Exposure` 

2317 Exposure to process. 

2318 defectBaseList : defect-type 

2319 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2320 or `list` of `lsst.afw.image.DefectBase`. 

2321 

2322 Notes 

2323 ----- 

2324 Call this after CCD assembly, since defects may cross amplifier 

2325 boundaries. 

2326 """ 

2327 maskedImage = exposure.getMaskedImage() 

2328 if not isinstance(defectBaseList, Defects): 

2329 # Promotes DefectBase to Defect 

2330 defectList = Defects(defectBaseList) 

2331 else: 

2332 defectList = defectBaseList 

2333 defectList.maskPixels(maskedImage, maskName="BAD") 

2334 

2335 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2336 """Mask edge pixels with applicable mask plane. 

2337 

2338 Parameters 

2339 ---------- 

2340 exposure : `lsst.afw.image.Exposure` 

2341 Exposure to process. 

2342 numEdgePixels : `int`, optional 

2343 Number of edge pixels to mask. 

2344 maskPlane : `str`, optional 

2345 Mask plane name to use. 

2346 level : `str`, optional 

2347 Level at which to mask edges. 

2348 """ 

2349 maskedImage = exposure.getMaskedImage() 

2350 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2351 

2352 if numEdgePixels > 0: 

2353 if level == 'DETECTOR': 

2354 boxes = [maskedImage.getBBox()] 

2355 elif level == 'AMP': 

2356 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2357 

2358 for box in boxes: 

2359 # This makes a bbox numEdgeSuspect pixels smaller than the 

2360 # image on each side 

2361 subImage = maskedImage[box] 

2362 box.grow(-numEdgePixels) 

2363 # Mask pixels outside box 

2364 SourceDetectionTask.setEdgeBits( 

2365 subImage, 

2366 box, 

2367 maskBitMask) 

2368 

2369 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2370 """Mask and interpolate defects using mask plane "BAD", in place. 

2371 

2372 Parameters 

2373 ---------- 

2374 exposure : `lsst.afw.image.Exposure` 

2375 Exposure to process. 

2376 defectBaseList : defects-like 

2377 List of defects to mask and interpolate. Can be 

2378 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2379 

2380 See Also 

2381 -------- 

2382 lsst.ip.isr.isrTask.maskDefect 

2383 """ 

2384 self.maskDefect(exposure, defectBaseList) 

2385 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2386 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2387 isrFunctions.interpolateFromMask( 

2388 maskedImage=exposure.getMaskedImage(), 

2389 fwhm=self.config.fwhm, 

2390 growSaturatedFootprints=0, 

2391 maskNameList=["BAD"], 

2392 ) 

2393 

2394 def maskNan(self, exposure): 

2395 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2396 

2397 Parameters 

2398 ---------- 

2399 exposure : `lsst.afw.image.Exposure` 

2400 Exposure to process. 

2401 

2402 Notes 

2403 ----- 

2404 We mask over all non-finite values (NaN, inf), including those 

2405 that are masked with other bits (because those may or may not be 

2406 interpolated over later, and we want to remove all NaN/infs). 

2407 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2408 preserve the historical name. 

2409 """ 

2410 maskedImage = exposure.getMaskedImage() 

2411 

2412 # Find and mask NaNs 

2413 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2414 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2415 numNans = maskNans(maskedImage, maskVal) 

2416 self.metadata["NUMNANS"] = numNans 

2417 if numNans > 0: 

2418 self.log.warning("There were %d unmasked NaNs.", numNans) 

2419 

2420 def maskAndInterpolateNan(self, exposure): 

2421 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2422 in place. 

2423 

2424 Parameters 

2425 ---------- 

2426 exposure : `lsst.afw.image.Exposure` 

2427 Exposure to process. 

2428 

2429 See Also 

2430 -------- 

2431 lsst.ip.isr.isrTask.maskNan 

2432 """ 

2433 self.maskNan(exposure) 

2434 isrFunctions.interpolateFromMask( 

2435 maskedImage=exposure.getMaskedImage(), 

2436 fwhm=self.config.fwhm, 

2437 growSaturatedFootprints=0, 

2438 maskNameList=["UNMASKEDNAN"], 

2439 ) 

2440 

2441 def measureBackground(self, exposure, IsrQaConfig=None): 

2442 """Measure the image background in subgrids, for quality control. 

2443 

2444 Parameters 

2445 ---------- 

2446 exposure : `lsst.afw.image.Exposure` 

2447 Exposure to process. 

2448 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2449 Configuration object containing parameters on which background 

2450 statistics and subgrids to use. 

2451 """ 

2452 if IsrQaConfig is not None: 

2453 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2454 IsrQaConfig.flatness.nIter) 

2455 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2456 statsControl.setAndMask(maskVal) 

2457 maskedImage = exposure.getMaskedImage() 

2458 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2459 skyLevel = stats.getValue(afwMath.MEDIAN) 

2460 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2461 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2462 metadata = exposure.getMetadata() 

2463 metadata["SKYLEVEL"] = skyLevel 

2464 metadata["SKYSIGMA"] = skySigma 

2465 

2466 # calcluating flatlevel over the subgrids 

2467 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2468 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2469 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2470 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2471 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2472 skyLevels = numpy.zeros((nX, nY)) 

2473 

2474 for j in range(nY): 

2475 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2476 for i in range(nX): 

2477 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2478 

2479 xLLC = xc - meshXHalf 

2480 yLLC = yc - meshYHalf 

2481 xURC = xc + meshXHalf - 1 

2482 yURC = yc + meshYHalf - 1 

2483 

2484 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2485 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2486 

2487 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2488 

2489 good = numpy.where(numpy.isfinite(skyLevels)) 

2490 skyMedian = numpy.median(skyLevels[good]) 

2491 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2492 flatness_rms = numpy.std(flatness) 

2493 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2494 

2495 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2496 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2497 nX, nY, flatness_pp, flatness_rms) 

2498 

2499 metadata["FLATNESS_PP"] = float(flatness_pp) 

2500 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2501 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2502 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2503 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2504 

2505 def roughZeroPoint(self, exposure): 

2506 """Set an approximate magnitude zero point for the exposure. 

2507 

2508 Parameters 

2509 ---------- 

2510 exposure : `lsst.afw.image.Exposure` 

2511 Exposure to process. 

2512 """ 

2513 filterLabel = exposure.getFilter() 

2514 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2515 

2516 if physicalFilter in self.config.fluxMag0T1: 

2517 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2518 else: 

2519 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2520 fluxMag0 = self.config.defaultFluxMag0T1 

2521 

2522 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2523 if not expTime > 0: # handle NaN as well as <= 0 

2524 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2525 return 

2526 

2527 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2528 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2529 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2530 

2531 @contextmanager 

2532 def flatContext(self, exp, flat, dark=None): 

2533 """Context manager that applies and removes flats and darks, 

2534 if the task is configured to apply them. 

2535 

2536 Parameters 

2537 ---------- 

2538 exp : `lsst.afw.image.Exposure` 

2539 Exposure to process. 

2540 flat : `lsst.afw.image.Exposure` 

2541 Flat exposure the same size as ``exp``. 

2542 dark : `lsst.afw.image.Exposure`, optional 

2543 Dark exposure the same size as ``exp``. 

2544 

2545 Yields 

2546 ------ 

2547 exp : `lsst.afw.image.Exposure` 

2548 The flat and dark corrected exposure. 

2549 """ 

2550 if self.config.doDark and dark is not None: 

2551 self.darkCorrection(exp, dark) 

2552 if self.config.doFlat: 

2553 self.flatCorrection(exp, flat) 

2554 try: 

2555 yield exp 

2556 finally: 

2557 if self.config.doFlat: 

2558 self.flatCorrection(exp, flat, invert=True) 

2559 if self.config.doDark and dark is not None: 

2560 self.darkCorrection(exp, dark, invert=True) 

2561 

2562 def debugView(self, exposure, stepname): 

2563 """Utility function to examine ISR exposure at different stages. 

2564 

2565 Parameters 

2566 ---------- 

2567 exposure : `lsst.afw.image.Exposure` 

2568 Exposure to view. 

2569 stepname : `str` 

2570 State of processing to view. 

2571 """ 

2572 frame = getDebugFrame(self._display, stepname) 

2573 if frame: 

2574 display = getDisplay(frame) 

2575 display.scale('asinh', 'zscale') 

2576 display.mtv(exposure) 

2577 prompt = "Press Enter to continue [c]... " 

2578 while True: 

2579 ans = input(prompt).lower() 

2580 if ans in ("", "c",): 

2581 break 

2582 

2583 

2584class FakeAmp(object): 

2585 """A Detector-like object that supports returning gain and saturation level 

2586 

2587 This is used when the input exposure does not have a detector. 

2588 

2589 Parameters 

2590 ---------- 

2591 exposure : `lsst.afw.image.Exposure` 

2592 Exposure to generate a fake amplifier for. 

2593 config : `lsst.ip.isr.isrTaskConfig` 

2594 Configuration to apply to the fake amplifier. 

2595 """ 

2596 

2597 def __init__(self, exposure, config): 

2598 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2599 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2600 self._gain = config.gain 

2601 self._readNoise = config.readNoise 

2602 self._saturation = config.saturation 

2603 

2604 def getBBox(self): 

2605 return self._bbox 

2606 

2607 def getRawBBox(self): 

2608 return self._bbox 

2609 

2610 def getRawHorizontalOverscanBBox(self): 

2611 return self._RawHorizontalOverscanBBox 

2612 

2613 def getGain(self): 

2614 return self._gain 

2615 

2616 def getReadNoise(self): 

2617 return self._readNoise 

2618 

2619 def getSaturation(self): 

2620 return self._saturation 

2621 

2622 def getSuspectLevel(self): 

2623 return float("NaN")