Coverage for python/lsst/ip/isr/isrTask.py: 16%

895 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-12 02:25 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from lsst.daf.butler import DimensionGraph 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

75 Data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

95 

96 

97class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

98 dimensions={"instrument", "exposure", "detector"}, 

99 defaultTemplates={}): 

100 ccdExposure = cT.Input( 

101 name="raw", 

102 doc="Input exposure to process.", 

103 storageClass="Exposure", 

104 dimensions=["instrument", "exposure", "detector"], 

105 ) 

106 camera = cT.PrerequisiteInput( 

107 name="camera", 

108 storageClass="Camera", 

109 doc="Input camera to construct complete exposures.", 

110 dimensions=["instrument"], 

111 isCalibration=True, 

112 ) 

113 

114 crosstalk = cT.PrerequisiteInput( 

115 name="crosstalk", 

116 doc="Input crosstalk object", 

117 storageClass="CrosstalkCalib", 

118 dimensions=["instrument", "detector"], 

119 isCalibration=True, 

120 minimum=0, # can fall back to cameraGeom 

121 ) 

122 crosstalkSources = cT.PrerequisiteInput( 

123 name="isrOverscanCorrected", 

124 doc="Overscan corrected input images.", 

125 storageClass="Exposure", 

126 dimensions=["instrument", "exposure", "detector"], 

127 deferLoad=True, 

128 multiple=True, 

129 lookupFunction=crosstalkSourceLookup, 

130 minimum=0, # not needed for all instruments, no config to control this 

131 ) 

132 bias = cT.PrerequisiteInput( 

133 name="bias", 

134 doc="Input bias calibration.", 

135 storageClass="ExposureF", 

136 dimensions=["instrument", "detector"], 

137 isCalibration=True, 

138 ) 

139 dark = cT.PrerequisiteInput( 

140 name='dark', 

141 doc="Input dark calibration.", 

142 storageClass="ExposureF", 

143 dimensions=["instrument", "detector"], 

144 isCalibration=True, 

145 ) 

146 flat = cT.PrerequisiteInput( 

147 name="flat", 

148 doc="Input flat calibration.", 

149 storageClass="ExposureF", 

150 dimensions=["instrument", "physical_filter", "detector"], 

151 isCalibration=True, 

152 ) 

153 ptc = cT.PrerequisiteInput( 

154 name="ptc", 

155 doc="Input Photon Transfer Curve dataset", 

156 storageClass="PhotonTransferCurveDataset", 

157 dimensions=["instrument", "detector"], 

158 isCalibration=True, 

159 ) 

160 fringes = cT.PrerequisiteInput( 

161 name="fringe", 

162 doc="Input fringe calibration.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "physical_filter", "detector"], 

165 isCalibration=True, 

166 minimum=0, # only needed for some bands, even when enabled 

167 ) 

168 strayLightData = cT.PrerequisiteInput( 

169 name='yBackground', 

170 doc="Input stray light calibration.", 

171 storageClass="StrayLightData", 

172 dimensions=["instrument", "physical_filter", "detector"], 

173 deferLoad=True, 

174 isCalibration=True, 

175 minimum=0, # only needed for some bands, even when enabled 

176 ) 

177 bfKernel = cT.PrerequisiteInput( 

178 name='bfKernel', 

179 doc="Input brighter-fatter kernel.", 

180 storageClass="NumpyArray", 

181 dimensions=["instrument"], 

182 isCalibration=True, 

183 minimum=0, # can use either bfKernel or newBFKernel 

184 ) 

185 newBFKernel = cT.PrerequisiteInput( 

186 name='brighterFatterKernel', 

187 doc="Newer complete kernel + gain solutions.", 

188 storageClass="BrighterFatterKernel", 

189 dimensions=["instrument", "detector"], 

190 isCalibration=True, 

191 minimum=0, # can use either bfKernel or newBFKernel 

192 ) 

193 defects = cT.PrerequisiteInput( 

194 name='defects', 

195 doc="Input defect tables.", 

196 storageClass="Defects", 

197 dimensions=["instrument", "detector"], 

198 isCalibration=True, 

199 ) 

200 linearizer = cT.PrerequisiteInput( 

201 name='linearizer', 

202 storageClass="Linearizer", 

203 doc="Linearity correction calibration.", 

204 dimensions=["instrument", "detector"], 

205 isCalibration=True, 

206 minimum=0, # can fall back to cameraGeom 

207 ) 

208 opticsTransmission = cT.PrerequisiteInput( 

209 name="transmission_optics", 

210 storageClass="TransmissionCurve", 

211 doc="Transmission curve due to the optics.", 

212 dimensions=["instrument"], 

213 isCalibration=True, 

214 ) 

215 filterTransmission = cT.PrerequisiteInput( 

216 name="transmission_filter", 

217 storageClass="TransmissionCurve", 

218 doc="Transmission curve due to the filter.", 

219 dimensions=["instrument", "physical_filter"], 

220 isCalibration=True, 

221 ) 

222 sensorTransmission = cT.PrerequisiteInput( 

223 name="transmission_sensor", 

224 storageClass="TransmissionCurve", 

225 doc="Transmission curve due to the sensor.", 

226 dimensions=["instrument", "detector"], 

227 isCalibration=True, 

228 ) 

229 atmosphereTransmission = cT.PrerequisiteInput( 

230 name="transmission_atmosphere", 

231 storageClass="TransmissionCurve", 

232 doc="Transmission curve due to the atmosphere.", 

233 dimensions=["instrument"], 

234 isCalibration=True, 

235 ) 

236 illumMaskedImage = cT.PrerequisiteInput( 

237 name="illum", 

238 doc="Input illumination correction.", 

239 storageClass="MaskedImageF", 

240 dimensions=["instrument", "physical_filter", "detector"], 

241 isCalibration=True, 

242 ) 

243 deferredChargeCalib = cT.PrerequisiteInput( 

244 name="cpCtiCalib", 

245 doc="Deferred charge/CTI correction dataset.", 

246 storageClass="IsrCalib", 

247 dimensions=["instrument", "detector"], 

248 isCalibration=True, 

249 ) 

250 

251 outputExposure = cT.Output( 

252 name='postISRCCD', 

253 doc="Output ISR processed exposure.", 

254 storageClass="Exposure", 

255 dimensions=["instrument", "exposure", "detector"], 

256 ) 

257 preInterpExposure = cT.Output( 

258 name='preInterpISRCCD', 

259 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

260 storageClass="ExposureF", 

261 dimensions=["instrument", "exposure", "detector"], 

262 ) 

263 outputOssThumbnail = cT.Output( 

264 name="OssThumb", 

265 doc="Output Overscan-subtracted thumbnail image.", 

266 storageClass="Thumbnail", 

267 dimensions=["instrument", "exposure", "detector"], 

268 ) 

269 outputFlattenedThumbnail = cT.Output( 

270 name="FlattenedThumb", 

271 doc="Output flat-corrected thumbnail image.", 

272 storageClass="Thumbnail", 

273 dimensions=["instrument", "exposure", "detector"], 

274 ) 

275 outputStatistics = cT.Output( 

276 name="isrStatistics", 

277 doc="Output of additional statistics table.", 

278 storageClass="StructuredDataDict", 

279 dimensions=["instrument", "exposure", "detector"], 

280 ) 

281 

282 def __init__(self, *, config=None): 

283 super().__init__(config=config) 

284 

285 if config.doBias is not True: 

286 self.prerequisiteInputs.remove("bias") 

287 if config.doLinearize is not True: 

288 self.prerequisiteInputs.remove("linearizer") 

289 if config.doCrosstalk is not True: 

290 self.prerequisiteInputs.remove("crosstalkSources") 

291 self.prerequisiteInputs.remove("crosstalk") 

292 if config.doBrighterFatter is not True: 

293 self.prerequisiteInputs.remove("bfKernel") 

294 self.prerequisiteInputs.remove("newBFKernel") 

295 if config.doDefect is not True: 

296 self.prerequisiteInputs.remove("defects") 

297 if config.doDark is not True: 

298 self.prerequisiteInputs.remove("dark") 

299 if config.doFlat is not True: 

300 self.prerequisiteInputs.remove("flat") 

301 if config.doFringe is not True: 

302 self.prerequisiteInputs.remove("fringes") 

303 if config.doStrayLight is not True: 

304 self.prerequisiteInputs.remove("strayLightData") 

305 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

306 self.prerequisiteInputs.remove("ptc") 

307 if config.doAttachTransmissionCurve is not True: 

308 self.prerequisiteInputs.remove("opticsTransmission") 

309 self.prerequisiteInputs.remove("filterTransmission") 

310 self.prerequisiteInputs.remove("sensorTransmission") 

311 self.prerequisiteInputs.remove("atmosphereTransmission") 

312 else: 

313 if config.doUseOpticsTransmission is not True: 

314 self.prerequisiteInputs.remove("opticsTransmission") 

315 if config.doUseFilterTransmission is not True: 

316 self.prerequisiteInputs.remove("filterTransmission") 

317 if config.doUseSensorTransmission is not True: 

318 self.prerequisiteInputs.remove("sensorTransmission") 

319 if config.doUseAtmosphereTransmission is not True: 

320 self.prerequisiteInputs.remove("atmosphereTransmission") 

321 if config.doIlluminationCorrection is not True: 

322 self.prerequisiteInputs.remove("illumMaskedImage") 

323 if config.doDeferredCharge is not True: 

324 self.prerequisiteInputs.remove("deferredChargeCalib") 

325 

326 if config.doWrite is not True: 

327 self.outputs.remove("outputExposure") 

328 self.outputs.remove("preInterpExposure") 

329 self.outputs.remove("outputFlattenedThumbnail") 

330 self.outputs.remove("outputOssThumbnail") 

331 self.outputs.remove("outputStatistics") 

332 

333 if config.doSaveInterpPixels is not True: 

334 self.outputs.remove("preInterpExposure") 

335 if config.qa.doThumbnailOss is not True: 

336 self.outputs.remove("outputOssThumbnail") 

337 if config.qa.doThumbnailFlattened is not True: 

338 self.outputs.remove("outputFlattenedThumbnail") 

339 if config.doCalculateStatistics is not True: 

340 self.outputs.remove("outputStatistics") 

341 

342 

343class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

344 pipelineConnections=IsrTaskConnections): 

345 """Configuration parameters for IsrTask. 

346 

347 Items are grouped in the order in which they are executed by the task. 

348 """ 

349 datasetType = pexConfig.Field( 

350 dtype=str, 

351 doc="Dataset type for input data; users will typically leave this alone, " 

352 "but camera-specific ISR tasks will override it", 

353 default="raw", 

354 ) 

355 

356 fallbackFilterName = pexConfig.Field( 

357 dtype=str, 

358 doc="Fallback default filter name for calibrations.", 

359 optional=True 

360 ) 

361 useFallbackDate = pexConfig.Field( 

362 dtype=bool, 

363 doc="Pass observation date when using fallback filter.", 

364 default=False, 

365 ) 

366 expectWcs = pexConfig.Field( 

367 dtype=bool, 

368 default=True, 

369 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

370 ) 

371 fwhm = pexConfig.Field( 

372 dtype=float, 

373 doc="FWHM of PSF in arcseconds.", 

374 default=1.0, 

375 ) 

376 qa = pexConfig.ConfigField( 

377 dtype=isrQa.IsrQaConfig, 

378 doc="QA related configuration options.", 

379 ) 

380 doHeaderProvenance = pexConfig.Field( 

381 dtype=bool, 

382 default=True, 

383 doc="Write calibration identifiers into output exposure header?", 

384 ) 

385 

386 # Calib checking configuration: 

387 doRaiseOnCalibMismatch = pexConfig.Field( 

388 dtype=bool, 

389 default=False, 

390 doc="Should IsrTask halt if exposure and calibration header values do not match?", 

391 ) 

392 cameraKeywordsToCompare = pexConfig.ListField( 

393 dtype=str, 

394 doc="List of header keywords to compare between exposure and calibrations.", 

395 default=[], 

396 ) 

397 

398 # Image conversion configuration 

399 doConvertIntToFloat = pexConfig.Field( 

400 dtype=bool, 

401 doc="Convert integer raw images to floating point values?", 

402 default=True, 

403 ) 

404 

405 # Saturated pixel handling. 

406 doSaturation = pexConfig.Field( 

407 dtype=bool, 

408 doc="Mask saturated pixels? NB: this is totally independent of the" 

409 " interpolation option - this is ONLY setting the bits in the mask." 

410 " To have them interpolated make sure doSaturationInterpolation=True", 

411 default=True, 

412 ) 

413 saturatedMaskName = pexConfig.Field( 

414 dtype=str, 

415 doc="Name of mask plane to use in saturation detection and interpolation", 

416 default="SAT", 

417 ) 

418 saturation = pexConfig.Field( 

419 dtype=float, 

420 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

421 default=float("NaN"), 

422 ) 

423 growSaturationFootprintSize = pexConfig.Field( 

424 dtype=int, 

425 doc="Number of pixels by which to grow the saturation footprints", 

426 default=1, 

427 ) 

428 

429 # Suspect pixel handling. 

430 doSuspect = pexConfig.Field( 

431 dtype=bool, 

432 doc="Mask suspect pixels?", 

433 default=False, 

434 ) 

435 suspectMaskName = pexConfig.Field( 

436 dtype=str, 

437 doc="Name of mask plane to use for suspect pixels", 

438 default="SUSPECT", 

439 ) 

440 numEdgeSuspect = pexConfig.Field( 

441 dtype=int, 

442 doc="Number of edge pixels to be flagged as untrustworthy.", 

443 default=0, 

444 ) 

445 edgeMaskLevel = pexConfig.ChoiceField( 

446 dtype=str, 

447 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

448 default="DETECTOR", 

449 allowed={ 

450 'DETECTOR': 'Mask only the edges of the full detector.', 

451 'AMP': 'Mask edges of each amplifier.', 

452 }, 

453 ) 

454 

455 # Initial masking options. 

456 doSetBadRegions = pexConfig.Field( 

457 dtype=bool, 

458 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

459 default=True, 

460 ) 

461 badStatistic = pexConfig.ChoiceField( 

462 dtype=str, 

463 doc="How to estimate the average value for BAD regions.", 

464 default='MEANCLIP', 

465 allowed={ 

466 "MEANCLIP": "Correct using the (clipped) mean of good data", 

467 "MEDIAN": "Correct using the median of the good data", 

468 }, 

469 ) 

470 

471 # Overscan subtraction configuration. 

472 doOverscan = pexConfig.Field( 

473 dtype=bool, 

474 doc="Do overscan subtraction?", 

475 default=True, 

476 ) 

477 overscan = pexConfig.ConfigurableField( 

478 target=OverscanCorrectionTask, 

479 doc="Overscan subtraction task for image segments.", 

480 ) 

481 

482 # Amplifier to CCD assembly configuration 

483 doAssembleCcd = pexConfig.Field( 

484 dtype=bool, 

485 default=True, 

486 doc="Assemble amp-level exposures into a ccd-level exposure?" 

487 ) 

488 assembleCcd = pexConfig.ConfigurableField( 

489 target=AssembleCcdTask, 

490 doc="CCD assembly task", 

491 ) 

492 

493 # General calibration configuration. 

494 doAssembleIsrExposures = pexConfig.Field( 

495 dtype=bool, 

496 default=False, 

497 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

498 ) 

499 doTrimToMatchCalib = pexConfig.Field( 

500 dtype=bool, 

501 default=False, 

502 doc="Trim raw data to match calibration bounding boxes?" 

503 ) 

504 

505 # Bias subtraction. 

506 doBias = pexConfig.Field( 

507 dtype=bool, 

508 doc="Apply bias frame correction?", 

509 default=True, 

510 ) 

511 biasDataProductName = pexConfig.Field( 

512 dtype=str, 

513 doc="Name of the bias data product", 

514 default="bias", 

515 ) 

516 doBiasBeforeOverscan = pexConfig.Field( 

517 dtype=bool, 

518 doc="Reverse order of overscan and bias correction.", 

519 default=False 

520 ) 

521 

522 # Deferred charge correction. 

523 doDeferredCharge = pexConfig.Field( 

524 dtype=bool, 

525 doc="Apply deferred charge correction?", 

526 default=False, 

527 ) 

528 deferredChargeCorrection = pexConfig.ConfigurableField( 

529 target=DeferredChargeTask, 

530 doc="Deferred charge correction task.", 

531 ) 

532 

533 # Variance construction 

534 doVariance = pexConfig.Field( 

535 dtype=bool, 

536 doc="Calculate variance?", 

537 default=True 

538 ) 

539 gain = pexConfig.Field( 

540 dtype=float, 

541 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

542 default=float("NaN"), 

543 ) 

544 readNoise = pexConfig.Field( 

545 dtype=float, 

546 doc="The read noise to use if no Detector is present in the Exposure", 

547 default=0.0, 

548 ) 

549 doEmpiricalReadNoise = pexConfig.Field( 

550 dtype=bool, 

551 default=False, 

552 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

553 ) 

554 usePtcReadNoise = pexConfig.Field( 

555 dtype=bool, 

556 default=False, 

557 doc="Use readnoise values from the Photon Transfer Curve?" 

558 ) 

559 maskNegativeVariance = pexConfig.Field( 

560 dtype=bool, 

561 default=True, 

562 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

563 "in the measurement of the overscan at an edge due to the data falling off faster " 

564 "than the overscan model can account for it." 

565 ) 

566 negativeVarianceMaskName = pexConfig.Field( 

567 dtype=str, 

568 default="BAD", 

569 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

570 ) 

571 # Linearization. 

572 doLinearize = pexConfig.Field( 

573 dtype=bool, 

574 doc="Correct for nonlinearity of the detector's response?", 

575 default=True, 

576 ) 

577 

578 # Crosstalk. 

579 doCrosstalk = pexConfig.Field( 

580 dtype=bool, 

581 doc="Apply intra-CCD crosstalk correction?", 

582 default=False, 

583 ) 

584 doCrosstalkBeforeAssemble = pexConfig.Field( 

585 dtype=bool, 

586 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

587 default=False, 

588 ) 

589 crosstalk = pexConfig.ConfigurableField( 

590 target=CrosstalkTask, 

591 doc="Intra-CCD crosstalk correction", 

592 ) 

593 

594 # Masking options. 

595 doDefect = pexConfig.Field( 

596 dtype=bool, 

597 doc="Apply correction for CCD defects, e.g. hot pixels?", 

598 default=True, 

599 ) 

600 doNanMasking = pexConfig.Field( 

601 dtype=bool, 

602 doc="Mask non-finite (NAN, inf) pixels?", 

603 default=True, 

604 ) 

605 doWidenSaturationTrails = pexConfig.Field( 

606 dtype=bool, 

607 doc="Widen bleed trails based on their width?", 

608 default=True 

609 ) 

610 

611 # Brighter-Fatter correction. 

612 doBrighterFatter = pexConfig.Field( 

613 dtype=bool, 

614 default=False, 

615 doc="Apply the brighter-fatter correction?" 

616 ) 

617 doFluxConservingBrighterFatterCorrection = pexConfig.Field( 

618 dtype=bool, 

619 default=False, 

620 doc="Apply the flux-conserving BFE correction by Miller et al.?" 

621 ) 

622 brighterFatterLevel = pexConfig.ChoiceField( 

623 dtype=str, 

624 default="DETECTOR", 

625 doc="The level at which to correct for brighter-fatter.", 

626 allowed={ 

627 "AMP": "Every amplifier treated separately.", 

628 "DETECTOR": "One kernel per detector", 

629 } 

630 ) 

631 brighterFatterMaxIter = pexConfig.Field( 

632 dtype=int, 

633 default=10, 

634 doc="Maximum number of iterations for the brighter-fatter correction" 

635 ) 

636 brighterFatterThreshold = pexConfig.Field( 

637 dtype=float, 

638 default=1000, 

639 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

640 "absolute value of the difference between the current corrected image and the one " 

641 "from the previous iteration summed over all the pixels." 

642 ) 

643 brighterFatterApplyGain = pexConfig.Field( 

644 dtype=bool, 

645 default=True, 

646 doc="Should the gain be applied when applying the brighter-fatter correction?" 

647 ) 

648 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

649 dtype=str, 

650 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

651 "correction.", 

652 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

653 ) 

654 brighterFatterMaskGrowSize = pexConfig.Field( 

655 dtype=int, 

656 default=0, 

657 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

658 "when brighter-fatter correction is applied." 

659 ) 

660 

661 # Dark subtraction. 

662 doDark = pexConfig.Field( 

663 dtype=bool, 

664 doc="Apply dark frame correction?", 

665 default=True, 

666 ) 

667 darkDataProductName = pexConfig.Field( 

668 dtype=str, 

669 doc="Name of the dark data product", 

670 default="dark", 

671 ) 

672 

673 # Camera-specific stray light removal. 

674 doStrayLight = pexConfig.Field( 

675 dtype=bool, 

676 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

677 default=False, 

678 ) 

679 strayLight = pexConfig.ConfigurableField( 

680 target=StrayLightTask, 

681 doc="y-band stray light correction" 

682 ) 

683 

684 # Flat correction. 

685 doFlat = pexConfig.Field( 

686 dtype=bool, 

687 doc="Apply flat field correction?", 

688 default=True, 

689 ) 

690 flatDataProductName = pexConfig.Field( 

691 dtype=str, 

692 doc="Name of the flat data product", 

693 default="flat", 

694 ) 

695 flatScalingType = pexConfig.ChoiceField( 

696 dtype=str, 

697 doc="The method for scaling the flat on the fly.", 

698 default='USER', 

699 allowed={ 

700 "USER": "Scale by flatUserScale", 

701 "MEAN": "Scale by the inverse of the mean", 

702 "MEDIAN": "Scale by the inverse of the median", 

703 }, 

704 ) 

705 flatUserScale = pexConfig.Field( 

706 dtype=float, 

707 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

708 default=1.0, 

709 ) 

710 doTweakFlat = pexConfig.Field( 

711 dtype=bool, 

712 doc="Tweak flats to match observed amplifier ratios?", 

713 default=False 

714 ) 

715 

716 # Amplifier normalization based on gains instead of using flats 

717 # configuration. 

718 doApplyGains = pexConfig.Field( 

719 dtype=bool, 

720 doc="Correct the amplifiers for their gains instead of applying flat correction", 

721 default=False, 

722 ) 

723 usePtcGains = pexConfig.Field( 

724 dtype=bool, 

725 doc="Use the gain values from the Photon Transfer Curve?", 

726 default=False, 

727 ) 

728 normalizeGains = pexConfig.Field( 

729 dtype=bool, 

730 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

731 default=False, 

732 ) 

733 

734 # Fringe correction. 

735 doFringe = pexConfig.Field( 

736 dtype=bool, 

737 doc="Apply fringe correction?", 

738 default=True, 

739 ) 

740 fringe = pexConfig.ConfigurableField( 

741 target=FringeTask, 

742 doc="Fringe subtraction task", 

743 ) 

744 fringeAfterFlat = pexConfig.Field( 

745 dtype=bool, 

746 doc="Do fringe subtraction after flat-fielding?", 

747 default=True, 

748 ) 

749 

750 # Amp offset correction. 

751 doAmpOffset = pexConfig.Field( 

752 doc="Calculate and apply amp offset corrections?", 

753 dtype=bool, 

754 default=False, 

755 ) 

756 ampOffset = pexConfig.ConfigurableField( 

757 doc="Amp offset correction task.", 

758 target=AmpOffsetTask, 

759 ) 

760 

761 # Initial CCD-level background statistics options. 

762 doMeasureBackground = pexConfig.Field( 

763 dtype=bool, 

764 doc="Measure the background level on the reduced image?", 

765 default=False, 

766 ) 

767 

768 # Camera-specific masking configuration. 

769 doCameraSpecificMasking = pexConfig.Field( 

770 dtype=bool, 

771 doc="Mask camera-specific bad regions?", 

772 default=False, 

773 ) 

774 masking = pexConfig.ConfigurableField( 

775 target=MaskingTask, 

776 doc="Masking task." 

777 ) 

778 

779 # Interpolation options. 

780 doInterpolate = pexConfig.Field( 

781 dtype=bool, 

782 doc="Interpolate masked pixels?", 

783 default=True, 

784 ) 

785 doSaturationInterpolation = pexConfig.Field( 

786 dtype=bool, 

787 doc="Perform interpolation over pixels masked as saturated?" 

788 " NB: This is independent of doSaturation; if that is False this plane" 

789 " will likely be blank, resulting in a no-op here.", 

790 default=True, 

791 ) 

792 doNanInterpolation = pexConfig.Field( 

793 dtype=bool, 

794 doc="Perform interpolation over pixels masked as NaN?" 

795 " NB: This is independent of doNanMasking; if that is False this plane" 

796 " will likely be blank, resulting in a no-op here.", 

797 default=True, 

798 ) 

799 doNanInterpAfterFlat = pexConfig.Field( 

800 dtype=bool, 

801 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

802 "also have to interpolate them before flat-fielding."), 

803 default=False, 

804 ) 

805 maskListToInterpolate = pexConfig.ListField( 

806 dtype=str, 

807 doc="List of mask planes that should be interpolated.", 

808 default=['SAT', 'BAD'], 

809 ) 

810 doSaveInterpPixels = pexConfig.Field( 

811 dtype=bool, 

812 doc="Save a copy of the pre-interpolated pixel values?", 

813 default=False, 

814 ) 

815 

816 # Default photometric calibration options. 

817 fluxMag0T1 = pexConfig.DictField( 

818 keytype=str, 

819 itemtype=float, 

820 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

821 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

822 )) 

823 ) 

824 defaultFluxMag0T1 = pexConfig.Field( 

825 dtype=float, 

826 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

827 default=pow(10.0, 0.4*28.0) 

828 ) 

829 

830 # Vignette correction configuration. 

831 doVignette = pexConfig.Field( 

832 dtype=bool, 

833 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

834 "according to vignetting parameters?"), 

835 default=False, 

836 ) 

837 doMaskVignettePolygon = pexConfig.Field( 

838 dtype=bool, 

839 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

840 "is False"), 

841 default=True, 

842 ) 

843 vignetteValue = pexConfig.Field( 

844 dtype=float, 

845 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

846 optional=True, 

847 default=None, 

848 ) 

849 vignette = pexConfig.ConfigurableField( 

850 target=VignetteTask, 

851 doc="Vignetting task.", 

852 ) 

853 

854 # Transmission curve configuration. 

855 doAttachTransmissionCurve = pexConfig.Field( 

856 dtype=bool, 

857 default=False, 

858 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

859 ) 

860 doUseOpticsTransmission = pexConfig.Field( 

861 dtype=bool, 

862 default=True, 

863 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

864 ) 

865 doUseFilterTransmission = pexConfig.Field( 

866 dtype=bool, 

867 default=True, 

868 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

869 ) 

870 doUseSensorTransmission = pexConfig.Field( 

871 dtype=bool, 

872 default=True, 

873 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

874 ) 

875 doUseAtmosphereTransmission = pexConfig.Field( 

876 dtype=bool, 

877 default=True, 

878 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

879 ) 

880 

881 # Illumination correction. 

882 doIlluminationCorrection = pexConfig.Field( 

883 dtype=bool, 

884 default=False, 

885 doc="Perform illumination correction?" 

886 ) 

887 illuminationCorrectionDataProductName = pexConfig.Field( 

888 dtype=str, 

889 doc="Name of the illumination correction data product.", 

890 default="illumcor", 

891 ) 

892 illumScale = pexConfig.Field( 

893 dtype=float, 

894 doc="Scale factor for the illumination correction.", 

895 default=1.0, 

896 ) 

897 illumFilters = pexConfig.ListField( 

898 dtype=str, 

899 default=[], 

900 doc="Only perform illumination correction for these filters." 

901 ) 

902 

903 # Calculate image quality statistics? 

904 doStandardStatistics = pexConfig.Field( 

905 dtype=bool, 

906 doc="Should standard image quality statistics be calculated?", 

907 default=True, 

908 ) 

909 # Calculate additional statistics? 

910 doCalculateStatistics = pexConfig.Field( 

911 dtype=bool, 

912 doc="Should additional ISR statistics be calculated?", 

913 default=False, 

914 ) 

915 isrStats = pexConfig.ConfigurableField( 

916 target=IsrStatisticsTask, 

917 doc="Task to calculate additional statistics.", 

918 ) 

919 

920 # Write the outputs to disk. If ISR is run as a subtask, this may not 

921 # be needed. 

922 doWrite = pexConfig.Field( 

923 dtype=bool, 

924 doc="Persist postISRCCD?", 

925 default=True, 

926 ) 

927 

928 def validate(self): 

929 super().validate() 

930 if self.doFlat and self.doApplyGains: 

931 raise ValueError("You may not specify both doFlat and doApplyGains") 

932 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

933 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

934 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

935 self.maskListToInterpolate.append(self.saturatedMaskName) 

936 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

937 self.maskListToInterpolate.remove(self.saturatedMaskName) 

938 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

939 self.maskListToInterpolate.append("UNMASKEDNAN") 

940 

941 

942class IsrTask(pipeBase.PipelineTask): 

943 """Apply common instrument signature correction algorithms to a raw frame. 

944 

945 The process for correcting imaging data is very similar from 

946 camera to camera. This task provides a vanilla implementation of 

947 doing these corrections, including the ability to turn certain 

948 corrections off if they are not needed. The inputs to the primary 

949 method, `run()`, are a raw exposure to be corrected and the 

950 calibration data products. The raw input is a single chip sized 

951 mosaic of all amps including overscans and other non-science 

952 pixels. 

953 

954 The __init__ method sets up the subtasks for ISR processing, using 

955 the defaults from `lsst.ip.isr`. 

956 

957 Parameters 

958 ---------- 

959 args : `list` 

960 Positional arguments passed to the Task constructor. 

961 None used at this time. 

962 kwargs : `dict`, optional 

963 Keyword arguments passed on to the Task constructor. 

964 None used at this time. 

965 """ 

966 ConfigClass = IsrTaskConfig 

967 _DefaultName = "isr" 

968 

969 def __init__(self, **kwargs): 

970 super().__init__(**kwargs) 

971 self.makeSubtask("assembleCcd") 

972 self.makeSubtask("crosstalk") 

973 self.makeSubtask("strayLight") 

974 self.makeSubtask("fringe") 

975 self.makeSubtask("masking") 

976 self.makeSubtask("overscan") 

977 self.makeSubtask("vignette") 

978 self.makeSubtask("ampOffset") 

979 self.makeSubtask("deferredChargeCorrection") 

980 self.makeSubtask("isrStats") 

981 

982 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

983 inputs = butlerQC.get(inputRefs) 

984 

985 try: 

986 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

987 except Exception as e: 

988 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

989 (inputRefs, e)) 

990 

991 detector = inputs['ccdExposure'].getDetector() 

992 

993 if self.config.doCrosstalk is True: 

994 # Crosstalk sources need to be defined by the pipeline 

995 # yaml if they exist. 

996 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

997 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

998 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

999 else: 

1000 coeffVector = (self.config.crosstalk.crosstalkValues 

1001 if self.config.crosstalk.useConfigCoefficients else None) 

1002 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1003 inputs['crosstalk'] = crosstalkCalib 

1004 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1005 if 'crosstalkSources' not in inputs: 

1006 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1007 

1008 if self.doLinearize(detector) is True: 

1009 if 'linearizer' in inputs: 

1010 if isinstance(inputs['linearizer'], dict): 

1011 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1012 linearizer.fromYaml(inputs['linearizer']) 

1013 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1014 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1015 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1016 detector=detector, 

1017 log=self.log) 

1018 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1019 else: 

1020 linearizer = inputs['linearizer'] 

1021 linearizer.log = self.log 

1022 inputs['linearizer'] = linearizer 

1023 else: 

1024 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1025 self.log.warning("Constructing linearizer from cameraGeom information.") 

1026 

1027 if self.config.doDefect is True: 

1028 if "defects" in inputs and inputs['defects'] is not None: 

1029 # defects is loaded as a BaseCatalog with columns 

1030 # x0, y0, width, height. Masking expects a list of defects 

1031 # defined by their bounding box 

1032 if not isinstance(inputs["defects"], Defects): 

1033 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1034 

1035 # Load the correct style of brighter-fatter kernel, and repack 

1036 # the information as a numpy array. 

1037 if self.config.doBrighterFatter: 

1038 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1039 if brighterFatterKernel is None: 

1040 brighterFatterKernel = inputs.get('bfKernel', None) 

1041 

1042 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1043 # This is a ISR calib kernel 

1044 detName = detector.getName() 

1045 level = brighterFatterKernel.level 

1046 

1047 # This is expected to be a dictionary of amp-wise gains. 

1048 inputs['bfGains'] = brighterFatterKernel.gain 

1049 if self.config.brighterFatterLevel == 'DETECTOR': 

1050 if level == 'DETECTOR': 

1051 if detName in brighterFatterKernel.detKernels: 

1052 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1053 else: 

1054 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1055 elif level == 'AMP': 

1056 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1057 "fatter kernels.") 

1058 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1059 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1060 elif self.config.brighterFatterLevel == 'AMP': 

1061 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1062 

1063 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1064 expId = inputs['ccdExposure'].info.id 

1065 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1066 expId=expId, 

1067 assembler=self.assembleCcd 

1068 if self.config.doAssembleIsrExposures else None) 

1069 else: 

1070 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1071 

1072 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1073 if 'strayLightData' not in inputs: 

1074 inputs['strayLightData'] = None 

1075 

1076 if self.config.doHeaderProvenance: 

1077 # Add calibration provenanace info to header. 

1078 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1079 for inputName in sorted(inputs.keys()): 

1080 reference = getattr(inputRefs, inputName, None) 

1081 if reference is not None and hasattr(reference, "run"): 

1082 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1083 runValue = reference.run 

1084 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1085 idValue = str(reference.id) 

1086 

1087 exposureMetadata[runKey] = runValue 

1088 exposureMetadata[idKey] = idValue 

1089 

1090 outputs = self.run(**inputs) 

1091 butlerQC.put(outputs, outputRefs) 

1092 

1093 @timeMethod 

1094 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1095 crosstalk=None, crosstalkSources=None, 

1096 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1097 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1098 sensorTransmission=None, atmosphereTransmission=None, 

1099 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1100 deferredChargeCalib=None, 

1101 ): 

1102 """Perform instrument signature removal on an exposure. 

1103 

1104 Steps included in the ISR processing, in order performed, are: 

1105 

1106 - saturation and suspect pixel masking 

1107 - overscan subtraction 

1108 - CCD assembly of individual amplifiers 

1109 - bias subtraction 

1110 - variance image construction 

1111 - linearization of non-linear response 

1112 - crosstalk masking 

1113 - brighter-fatter correction 

1114 - dark subtraction 

1115 - fringe correction 

1116 - stray light subtraction 

1117 - flat correction 

1118 - masking of known defects and camera specific features 

1119 - vignette calculation 

1120 - appending transmission curve and distortion model 

1121 

1122 Parameters 

1123 ---------- 

1124 ccdExposure : `lsst.afw.image.Exposure` 

1125 The raw exposure that is to be run through ISR. The 

1126 exposure is modified by this method. 

1127 camera : `lsst.afw.cameraGeom.Camera`, optional 

1128 The camera geometry for this exposure. Required if 

1129 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1130 ``flat`` does not have an associated detector. 

1131 bias : `lsst.afw.image.Exposure`, optional 

1132 Bias calibration frame. 

1133 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1134 Functor for linearization. 

1135 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1136 Calibration for crosstalk. 

1137 crosstalkSources : `list`, optional 

1138 List of possible crosstalk sources. 

1139 dark : `lsst.afw.image.Exposure`, optional 

1140 Dark calibration frame. 

1141 flat : `lsst.afw.image.Exposure`, optional 

1142 Flat calibration frame. 

1143 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1144 Photon transfer curve dataset, with, e.g., gains 

1145 and read noise. 

1146 bfKernel : `numpy.ndarray`, optional 

1147 Brighter-fatter kernel. 

1148 bfGains : `dict` of `float`, optional 

1149 Gains used to override the detector's nominal gains for the 

1150 brighter-fatter correction. A dict keyed by amplifier name for 

1151 the detector in question. 

1152 defects : `lsst.ip.isr.Defects`, optional 

1153 List of defects. 

1154 fringes : `lsst.pipe.base.Struct`, optional 

1155 Struct containing the fringe correction data, with 

1156 elements: 

1157 

1158 ``fringes`` 

1159 fringe calibration frame (`lsst.afw.image.Exposure`) 

1160 ``seed`` 

1161 random seed derived from the ``ccdExposureId`` for random 

1162 number generator (`numpy.uint32`) 

1163 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1164 A ``TransmissionCurve`` that represents the throughput of the, 

1165 optics, to be evaluated in focal-plane coordinates. 

1166 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1167 A ``TransmissionCurve`` that represents the throughput of the 

1168 filter itself, to be evaluated in focal-plane coordinates. 

1169 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1170 A ``TransmissionCurve`` that represents the throughput of the 

1171 sensor itself, to be evaluated in post-assembly trimmed detector 

1172 coordinates. 

1173 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1174 A ``TransmissionCurve`` that represents the throughput of the 

1175 atmosphere, assumed to be spatially constant. 

1176 detectorNum : `int`, optional 

1177 The integer number for the detector to process. 

1178 strayLightData : `object`, optional 

1179 Opaque object containing calibration information for stray-light 

1180 correction. If `None`, no correction will be performed. 

1181 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1182 Illumination correction image. 

1183 

1184 Returns 

1185 ------- 

1186 result : `lsst.pipe.base.Struct` 

1187 Result struct with component: 

1188 

1189 ``exposure`` 

1190 The fully ISR corrected exposure. 

1191 (`lsst.afw.image.Exposure`) 

1192 ``outputExposure`` 

1193 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1194 ``ossThumb`` 

1195 Thumbnail image of the exposure after overscan subtraction. 

1196 (`numpy.ndarray`) 

1197 ``flattenedThumb`` 

1198 Thumbnail image of the exposure after flat-field correction. 

1199 (`numpy.ndarray`) 

1200 ``outputStatistics`` 

1201 Values of the additional statistics calculated. 

1202 

1203 Raises 

1204 ------ 

1205 RuntimeError 

1206 Raised if a configuration option is set to `True`, but the 

1207 required calibration data has not been specified. 

1208 

1209 Notes 

1210 ----- 

1211 The current processed exposure can be viewed by setting the 

1212 appropriate `lsstDebug` entries in the ``debug.display`` 

1213 dictionary. The names of these entries correspond to some of 

1214 the `IsrTaskConfig` Boolean options, with the value denoting the 

1215 frame to use. The exposure is shown inside the matching 

1216 option check and after the processing of that step has 

1217 finished. The steps with debug points are: 

1218 

1219 * doAssembleCcd 

1220 * doBias 

1221 * doCrosstalk 

1222 * doBrighterFatter 

1223 * doDark 

1224 * doFringe 

1225 * doStrayLight 

1226 * doFlat 

1227 

1228 In addition, setting the ``postISRCCD`` entry displays the 

1229 exposure after all ISR processing has finished. 

1230 """ 

1231 

1232 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1233 bias = self.ensureExposure(bias, camera, detectorNum) 

1234 dark = self.ensureExposure(dark, camera, detectorNum) 

1235 flat = self.ensureExposure(flat, camera, detectorNum) 

1236 

1237 ccd = ccdExposure.getDetector() 

1238 filterLabel = ccdExposure.getFilter() 

1239 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1240 

1241 if not ccd: 

1242 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1243 ccd = [FakeAmp(ccdExposure, self.config)] 

1244 

1245 # Validate Input 

1246 if self.config.doBias and bias is None: 

1247 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1248 if self.doLinearize(ccd) and linearizer is None: 

1249 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1250 if self.config.doBrighterFatter and bfKernel is None: 

1251 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1252 if self.config.doDark and dark is None: 

1253 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1254 if self.config.doFlat and flat is None: 

1255 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1256 if self.config.doDefect and defects is None: 

1257 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1258 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1259 and fringes.fringes is None): 

1260 # The `fringes` object needs to be a pipeBase.Struct, as 

1261 # we use it as a `dict` for the parameters of 

1262 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1263 # not be `None` if `doFringe=True`. Otherwise, raise. 

1264 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1265 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1266 and illumMaskedImage is None): 

1267 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1268 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1269 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1270 

1271 if self.config.doHeaderProvenance: 

1272 # Inputs have been validated, so we can add their date 

1273 # information to the output header. 

1274 exposureMetadata = ccdExposure.getMetadata() 

1275 if self.config.doBias: 

1276 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias) 

1277 self.compareCameraKeywords(exposureMetadata, bias, "bias") 

1278 if self.config.doBrighterFatter: 

1279 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel) 

1280 self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter") 

1281 if self.config.doCrosstalk: 

1282 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk) 

1283 self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk") 

1284 if self.config.doDark: 

1285 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark) 

1286 self.compareCameraKeywords(exposureMetadata, dark, "dark") 

1287 if self.config.doDefect: 

1288 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects) 

1289 self.compareCameraKeywords(exposureMetadata, defects, "defects") 

1290 if self.config.doDeferredCharge: 

1291 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib) 

1292 self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI") 

1293 if self.config.doFlat: 

1294 exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat) 

1295 self.compareCameraKeywords(exposureMetadata, flat, "flat") 

1296 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1297 exposureMetadata["LSST CALIB DATE FRINGE"] = self.extractCalibDate(fringes.fringes) 

1298 self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe") 

1299 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1300 exposureMetadata["LSST CALIB DATE ILLUMINATION"] = self.extractCalibDate(illumMaskedImage) 

1301 self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination") 

1302 if self.doLinearize(ccd): 

1303 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer) 

1304 self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer") 

1305 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1306 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc) 

1307 self.compareCameraKeywords(exposureMetadata, ptc, "PTC") 

1308 if self.config.doStrayLight: 

1309 exposureMetadata["LSST CALIB DATE STRAYLIGHT"] = self.extractCalibDate(strayLightData) 

1310 self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight") 

1311 if self.config.doAttachTransmissionCurve: 

1312 exposureMetadata["LSST CALIB DATE OPTICS_TR"] = self.extractCalibDate(opticsTransmission) 

1313 exposureMetadata["LSST CALIB DATE FILTER_TR"] = self.extractCalibDate(filterTransmission) 

1314 exposureMetadata["LSST CALIB DATE SENSOR_TR"] = self.extractCalibDate(sensorTransmission) 

1315 exposureMetadata["LSST CALIB DATE ATMOSP_TR"] = self.extractCalibDate(atmosphereTransmission) 

1316 

1317 # Begin ISR processing. 

1318 if self.config.doConvertIntToFloat: 

1319 self.log.info("Converting exposure to floating point values.") 

1320 ccdExposure = self.convertIntToFloat(ccdExposure) 

1321 

1322 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1323 self.log.info("Applying bias correction.") 

1324 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1325 trimToFit=self.config.doTrimToMatchCalib) 

1326 self.debugView(ccdExposure, "doBias") 

1327 

1328 # Amplifier level processing. 

1329 overscans = [] 

1330 

1331 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1332 # This will attempt to mask bleed pixels across all amplifiers. 

1333 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1334 

1335 for amp in ccd: 

1336 # if ccdExposure is one amp, 

1337 # check for coverage to prevent performing ops multiple times 

1338 if ccdExposure.getBBox().contains(amp.getBBox()): 

1339 # Check for fully masked bad amplifiers, 

1340 # and generate masks for SUSPECT and SATURATED values. 

1341 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1342 

1343 if self.config.doOverscan and not badAmp: 

1344 # Overscan correction on amp-by-amp basis. 

1345 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1346 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1347 if overscanResults is not None and \ 

1348 self.config.qa is not None and self.config.qa.saveStats is True: 

1349 if isinstance(overscanResults.overscanMean, float): 

1350 # Only serial overscan was run 

1351 mean = overscanResults.overscanMean 

1352 sigma = overscanResults.overscanSigma 

1353 residMean = overscanResults.residualMean 

1354 residSigma = overscanResults.residualSigma 

1355 else: 

1356 # Both serial and parallel overscan were 

1357 # run. Only report serial here. 

1358 mean = overscanResults.overscanMean[0] 

1359 sigma = overscanResults.overscanSigma[0] 

1360 residMean = overscanResults.residualMean[0] 

1361 residSigma = overscanResults.residualSigma[0] 

1362 

1363 self.metadata[f"FIT MEDIAN {amp.getName()}"] = mean 

1364 self.metadata[f"FIT STDEV {amp.getName()}"] = sigma 

1365 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1366 amp.getName(), mean, sigma) 

1367 

1368 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = residMean 

1369 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = residSigma 

1370 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1371 amp.getName(), residMean, residSigma) 

1372 

1373 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1374 else: 

1375 if badAmp: 

1376 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1377 overscanResults = None 

1378 

1379 overscans.append(overscanResults if overscanResults is not None else None) 

1380 else: 

1381 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1382 

1383 if self.config.doDeferredCharge: 

1384 self.log.info("Applying deferred charge/CTI correction.") 

1385 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1386 self.debugView(ccdExposure, "doDeferredCharge") 

1387 

1388 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1389 self.log.info("Applying crosstalk correction.") 

1390 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1391 crosstalkSources=crosstalkSources, camera=camera) 

1392 self.debugView(ccdExposure, "doCrosstalk") 

1393 

1394 if self.config.doAssembleCcd: 

1395 self.log.info("Assembling CCD from amplifiers.") 

1396 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1397 

1398 if self.config.expectWcs and not ccdExposure.getWcs(): 

1399 self.log.warning("No WCS found in input exposure.") 

1400 self.debugView(ccdExposure, "doAssembleCcd") 

1401 

1402 ossThumb = None 

1403 if self.config.qa.doThumbnailOss: 

1404 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1405 

1406 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1407 self.log.info("Applying bias correction.") 

1408 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1409 trimToFit=self.config.doTrimToMatchCalib) 

1410 self.debugView(ccdExposure, "doBias") 

1411 

1412 if self.config.doVariance: 

1413 for amp, overscanResults in zip(ccd, overscans): 

1414 if ccdExposure.getBBox().contains(amp.getBBox()): 

1415 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1416 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1417 if overscanResults is not None: 

1418 self.updateVariance(ampExposure, amp, 

1419 overscanImage=overscanResults.overscanImage, 

1420 ptcDataset=ptc) 

1421 else: 

1422 self.updateVariance(ampExposure, amp, 

1423 overscanImage=None, 

1424 ptcDataset=ptc) 

1425 if self.config.qa is not None and self.config.qa.saveStats is True: 

1426 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1427 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1428 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1429 qaStats.getValue(afwMath.MEDIAN) 

1430 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1431 qaStats.getValue(afwMath.STDEVCLIP) 

1432 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1433 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1434 qaStats.getValue(afwMath.STDEVCLIP)) 

1435 if self.config.maskNegativeVariance: 

1436 self.maskNegativeVariance(ccdExposure) 

1437 

1438 if self.doLinearize(ccd): 

1439 self.log.info("Applying linearizer.") 

1440 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1441 detector=ccd, log=self.log) 

1442 

1443 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1444 self.log.info("Applying crosstalk correction.") 

1445 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1446 crosstalkSources=crosstalkSources, isTrimmed=True) 

1447 self.debugView(ccdExposure, "doCrosstalk") 

1448 

1449 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1450 # widen trails, and do anything else the camera needs. Saturated and 

1451 # suspect pixels have already been masked. 

1452 if self.config.doDefect: 

1453 self.log.info("Masking defects.") 

1454 self.maskDefect(ccdExposure, defects) 

1455 

1456 if self.config.numEdgeSuspect > 0: 

1457 self.log.info("Masking edges as SUSPECT.") 

1458 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1459 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1460 

1461 if self.config.doNanMasking: 

1462 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1463 self.maskNan(ccdExposure) 

1464 

1465 if self.config.doWidenSaturationTrails: 

1466 self.log.info("Widening saturation trails.") 

1467 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1468 

1469 if self.config.doCameraSpecificMasking: 

1470 self.log.info("Masking regions for camera specific reasons.") 

1471 self.masking.run(ccdExposure) 

1472 

1473 if self.config.doBrighterFatter: 

1474 # We need to apply flats and darks before we can interpolate, and 

1475 # we need to interpolate before we do B-F, but we do B-F without 

1476 # the flats and darks applied so we can work in units of electrons 

1477 # or holes. This context manager applies and then removes the darks 

1478 # and flats. 

1479 # 

1480 # We also do not want to interpolate values here, so operate on 

1481 # temporary images so we can apply only the BF-correction and roll 

1482 # back the interpolation. 

1483 interpExp = ccdExposure.clone() 

1484 with self.flatContext(interpExp, flat, dark): 

1485 isrFunctions.interpolateFromMask( 

1486 maskedImage=interpExp.getMaskedImage(), 

1487 fwhm=self.config.fwhm, 

1488 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1489 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1490 ) 

1491 bfExp = interpExp.clone() 

1492 

1493 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1494 type(bfKernel), type(bfGains)) 

1495 if self.config.doFluxConservingBrighterFatterCorrection: 

1496 bfResults = isrFunctions.fluxConservingBrighterFatterCorrection( 

1497 bfExp, 

1498 bfKernel, 

1499 self.config.brighterFatterMaxIter, 

1500 self.config.brighterFatterThreshold, 

1501 self.config.brighterFatterApplyGain, 

1502 bfGains 

1503 ) 

1504 else: 

1505 bfResults = isrFunctions.brighterFatterCorrection( 

1506 bfExp, 

1507 bfKernel, 

1508 self.config.brighterFatterMaxIter, 

1509 self.config.brighterFatterThreshold, 

1510 self.config.brighterFatterApplyGain, 

1511 bfGains 

1512 ) 

1513 if bfResults[1] == self.config.brighterFatterMaxIter: 

1514 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1515 bfResults[0]) 

1516 else: 

1517 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1518 bfResults[1]) 

1519 image = ccdExposure.getMaskedImage().getImage() 

1520 bfCorr = bfExp.getMaskedImage().getImage() 

1521 bfCorr -= interpExp.getMaskedImage().getImage() 

1522 image += bfCorr 

1523 

1524 # Applying the brighter-fatter correction applies a 

1525 # convolution to the science image. At the edges this 

1526 # convolution may not have sufficient valid pixels to 

1527 # produce a valid correction. Mark pixels within the size 

1528 # of the brighter-fatter kernel as EDGE to warn of this 

1529 # fact. 

1530 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1531 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1532 maskPlane="EDGE") 

1533 

1534 if self.config.brighterFatterMaskGrowSize > 0: 

1535 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1536 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1537 isrFunctions.growMasks(ccdExposure.getMask(), 

1538 radius=self.config.brighterFatterMaskGrowSize, 

1539 maskNameList=maskPlane, 

1540 maskValue=maskPlane) 

1541 

1542 self.debugView(ccdExposure, "doBrighterFatter") 

1543 

1544 if self.config.doDark: 

1545 self.log.info("Applying dark correction.") 

1546 self.darkCorrection(ccdExposure, dark) 

1547 self.debugView(ccdExposure, "doDark") 

1548 

1549 if self.config.doFringe and not self.config.fringeAfterFlat: 

1550 self.log.info("Applying fringe correction before flat.") 

1551 self.fringe.run(ccdExposure, **fringes.getDict()) 

1552 self.debugView(ccdExposure, "doFringe") 

1553 

1554 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1555 self.log.info("Checking strayLight correction.") 

1556 self.strayLight.run(ccdExposure, strayLightData) 

1557 self.debugView(ccdExposure, "doStrayLight") 

1558 

1559 if self.config.doFlat: 

1560 self.log.info("Applying flat correction.") 

1561 self.flatCorrection(ccdExposure, flat) 

1562 self.debugView(ccdExposure, "doFlat") 

1563 

1564 if self.config.doApplyGains: 

1565 self.log.info("Applying gain correction instead of flat.") 

1566 if self.config.usePtcGains: 

1567 self.log.info("Using gains from the Photon Transfer Curve.") 

1568 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1569 ptcGains=ptc.gain) 

1570 else: 

1571 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1572 

1573 if self.config.doFringe and self.config.fringeAfterFlat: 

1574 self.log.info("Applying fringe correction after flat.") 

1575 self.fringe.run(ccdExposure, **fringes.getDict()) 

1576 

1577 if self.config.doVignette: 

1578 if self.config.doMaskVignettePolygon: 

1579 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1580 else: 

1581 self.log.info("Constructing and attaching vignette polygon.") 

1582 self.vignettePolygon = self.vignette.run( 

1583 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1584 vignetteValue=self.config.vignetteValue, log=self.log) 

1585 

1586 if self.config.doAttachTransmissionCurve: 

1587 self.log.info("Adding transmission curves.") 

1588 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1589 filterTransmission=filterTransmission, 

1590 sensorTransmission=sensorTransmission, 

1591 atmosphereTransmission=atmosphereTransmission) 

1592 

1593 flattenedThumb = None 

1594 if self.config.qa.doThumbnailFlattened: 

1595 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1596 

1597 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1598 self.log.info("Performing illumination correction.") 

1599 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1600 illumMaskedImage, illumScale=self.config.illumScale, 

1601 trimToFit=self.config.doTrimToMatchCalib) 

1602 

1603 preInterpExp = None 

1604 if self.config.doSaveInterpPixels: 

1605 preInterpExp = ccdExposure.clone() 

1606 

1607 # Reset and interpolate bad pixels. 

1608 # 

1609 # Large contiguous bad regions (which should have the BAD mask 

1610 # bit set) should have their values set to the image median. 

1611 # This group should include defects and bad amplifiers. As the 

1612 # area covered by these defects are large, there's little 

1613 # reason to expect that interpolation would provide a more 

1614 # useful value. 

1615 # 

1616 # Smaller defects can be safely interpolated after the larger 

1617 # regions have had their pixel values reset. This ensures 

1618 # that the remaining defects adjacent to bad amplifiers (as an 

1619 # example) do not attempt to interpolate extreme values. 

1620 if self.config.doSetBadRegions: 

1621 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1622 if badPixelCount > 0: 

1623 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1624 

1625 if self.config.doInterpolate: 

1626 self.log.info("Interpolating masked pixels.") 

1627 isrFunctions.interpolateFromMask( 

1628 maskedImage=ccdExposure.getMaskedImage(), 

1629 fwhm=self.config.fwhm, 

1630 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1631 maskNameList=list(self.config.maskListToInterpolate) 

1632 ) 

1633 

1634 self.roughZeroPoint(ccdExposure) 

1635 

1636 # correct for amp offsets within the CCD 

1637 if self.config.doAmpOffset: 

1638 self.log.info("Correcting amp offsets.") 

1639 self.ampOffset.run(ccdExposure) 

1640 

1641 if self.config.doMeasureBackground: 

1642 self.log.info("Measuring background level.") 

1643 self.measureBackground(ccdExposure, self.config.qa) 

1644 

1645 if self.config.qa is not None and self.config.qa.saveStats is True: 

1646 for amp in ccd: 

1647 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1648 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1649 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1650 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1651 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1652 qaStats.getValue(afwMath.STDEVCLIP) 

1653 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1654 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1655 qaStats.getValue(afwMath.STDEVCLIP)) 

1656 

1657 # Calculate standard image quality statistics 

1658 if self.config.doStandardStatistics: 

1659 metadata = ccdExposure.getMetadata() 

1660 for amp in ccd: 

1661 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1662 ampName = amp.getName() 

1663 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1664 ampExposure.getMaskedImage(), 

1665 [self.config.saturatedMaskName] 

1666 ) 

1667 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1668 ampExposure.getMaskedImage(), 

1669 ["BAD"] 

1670 ) 

1671 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1672 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1673 

1674 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1675 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1676 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1677 

1678 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1679 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1680 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1681 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1682 else: 

1683 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1684 

1685 # calculate additional statistics. 

1686 outputStatistics = None 

1687 if self.config.doCalculateStatistics: 

1688 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1689 ptc=ptc).results 

1690 

1691 self.debugView(ccdExposure, "postISRCCD") 

1692 

1693 return pipeBase.Struct( 

1694 exposure=ccdExposure, 

1695 ossThumb=ossThumb, 

1696 flattenedThumb=flattenedThumb, 

1697 

1698 preInterpExposure=preInterpExp, 

1699 outputExposure=ccdExposure, 

1700 outputOssThumbnail=ossThumb, 

1701 outputFlattenedThumbnail=flattenedThumb, 

1702 outputStatistics=outputStatistics, 

1703 ) 

1704 

1705 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1706 """Ensure that the data returned by Butler is a fully constructed exp. 

1707 

1708 ISR requires exposure-level image data for historical reasons, so if we 

1709 did not recieve that from Butler, construct it from what we have, 

1710 modifying the input in place. 

1711 

1712 Parameters 

1713 ---------- 

1714 inputExp : `lsst.afw.image` image-type. 

1715 The input data structure obtained from Butler. 

1716 Can be `lsst.afw.image.Exposure`, 

1717 `lsst.afw.image.DecoratedImageU`, 

1718 or `lsst.afw.image.ImageF` 

1719 camera : `lsst.afw.cameraGeom.camera`, optional 

1720 The camera associated with the image. Used to find the appropriate 

1721 detector if detector is not already set. 

1722 detectorNum : `int`, optional 

1723 The detector in the camera to attach, if the detector is not 

1724 already set. 

1725 

1726 Returns 

1727 ------- 

1728 inputExp : `lsst.afw.image.Exposure` 

1729 The re-constructed exposure, with appropriate detector parameters. 

1730 

1731 Raises 

1732 ------ 

1733 TypeError 

1734 Raised if the input data cannot be used to construct an exposure. 

1735 """ 

1736 if isinstance(inputExp, afwImage.DecoratedImageU): 

1737 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1738 elif isinstance(inputExp, afwImage.ImageF): 

1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1740 elif isinstance(inputExp, afwImage.MaskedImageF): 

1741 inputExp = afwImage.makeExposure(inputExp) 

1742 elif isinstance(inputExp, afwImage.Exposure): 

1743 pass 

1744 elif inputExp is None: 

1745 # Assume this will be caught by the setup if it is a problem. 

1746 return inputExp 

1747 else: 

1748 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1749 (type(inputExp), )) 

1750 

1751 if inputExp.getDetector() is None: 

1752 if camera is None or detectorNum is None: 

1753 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1754 'without a detector set.') 

1755 inputExp.setDetector(camera[detectorNum]) 

1756 

1757 return inputExp 

1758 

1759 @staticmethod 

1760 def extractCalibDate(calib): 

1761 """Extract common calibration metadata values that will be written to 

1762 output header. 

1763 

1764 Parameters 

1765 ---------- 

1766 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1767 Calibration to pull date information from. 

1768 

1769 Returns 

1770 ------- 

1771 dateString : `str` 

1772 Calibration creation date string to add to header. 

1773 """ 

1774 if hasattr(calib, "getMetadata"): 

1775 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1776 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1777 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1778 else: 

1779 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1780 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1781 else: 

1782 return "Unknown Unknown" 

1783 

1784 def compareCameraKeywords(self, exposureMetadata, calib, calibName): 

1785 """Compare header keywords to confirm camera states match. 

1786 

1787 Parameters 

1788 ---------- 

1789 exposureMetadata : `lsst.daf.base.PropertySet` 

1790 Header for the exposure being processed. 

1791 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1792 Calibration to be applied. 

1793 calibName : `str` 

1794 Calib type for log message. 

1795 """ 

1796 try: 

1797 calibMetadata = calib.getMetadata() 

1798 except AttributeError: 

1799 return 

1800 for keyword in self.config.cameraKeywordsToCompare: 

1801 if keyword in exposureMetadata and keyword in calibMetadata: 

1802 if exposureMetadata[keyword] != calibMetadata[keyword]: 

1803 if self.config.doRaiseOnCalibMismatch: 

1804 raise RuntimeError("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1805 calibName, keyword, 

1806 exposureMetadata[keyword], calibMetadata[keyword]) 

1807 else: 

1808 self.log.warning("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1809 calibName, keyword, 

1810 exposureMetadata[keyword], calibMetadata[keyword]) 

1811 else: 

1812 self.log.debug("Sequencer keyword %s not found.", keyword) 

1813 

1814 def convertIntToFloat(self, exposure): 

1815 """Convert exposure image from uint16 to float. 

1816 

1817 If the exposure does not need to be converted, the input is 

1818 immediately returned. For exposures that are converted to use 

1819 floating point pixels, the variance is set to unity and the 

1820 mask to zero. 

1821 

1822 Parameters 

1823 ---------- 

1824 exposure : `lsst.afw.image.Exposure` 

1825 The raw exposure to be converted. 

1826 

1827 Returns 

1828 ------- 

1829 newexposure : `lsst.afw.image.Exposure` 

1830 The input ``exposure``, converted to floating point pixels. 

1831 

1832 Raises 

1833 ------ 

1834 RuntimeError 

1835 Raised if the exposure type cannot be converted to float. 

1836 

1837 """ 

1838 if isinstance(exposure, afwImage.ExposureF): 

1839 # Nothing to be done 

1840 self.log.debug("Exposure already of type float.") 

1841 return exposure 

1842 if not hasattr(exposure, "convertF"): 

1843 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1844 

1845 newexposure = exposure.convertF() 

1846 newexposure.variance[:] = 1 

1847 newexposure.mask[:] = 0x0 

1848 

1849 return newexposure 

1850 

1851 def maskAmplifier(self, ccdExposure, amp, defects): 

1852 """Identify bad amplifiers, saturated and suspect pixels. 

1853 

1854 Parameters 

1855 ---------- 

1856 ccdExposure : `lsst.afw.image.Exposure` 

1857 Input exposure to be masked. 

1858 amp : `lsst.afw.cameraGeom.Amplifier` 

1859 Catalog of parameters defining the amplifier on this 

1860 exposure to mask. 

1861 defects : `lsst.ip.isr.Defects` 

1862 List of defects. Used to determine if the entire 

1863 amplifier is bad. 

1864 

1865 Returns 

1866 ------- 

1867 badAmp : `Bool` 

1868 If this is true, the entire amplifier area is covered by 

1869 defects and unusable. 

1870 

1871 """ 

1872 maskedImage = ccdExposure.getMaskedImage() 

1873 

1874 badAmp = False 

1875 

1876 # Check if entire amp region is defined as a defect 

1877 # NB: need to use amp.getBBox() for correct comparison with current 

1878 # defects definition. 

1879 if defects is not None: 

1880 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1881 

1882 # In the case of a bad amp, we will set mask to "BAD" 

1883 # (here use amp.getRawBBox() for correct association with pixels in 

1884 # current ccdExposure). 

1885 if badAmp: 

1886 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1887 afwImage.PARENT) 

1888 maskView = dataView.getMask() 

1889 maskView |= maskView.getPlaneBitMask("BAD") 

1890 del maskView 

1891 return badAmp 

1892 

1893 # Mask remaining defects after assembleCcd() to allow for defects that 

1894 # cross amplifier boundaries. Saturation and suspect pixels can be 

1895 # masked now, though. 

1896 limits = dict() 

1897 if self.config.doSaturation and not badAmp: 

1898 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1899 if self.config.doSuspect and not badAmp: 

1900 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1901 if math.isfinite(self.config.saturation): 

1902 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1903 

1904 for maskName, maskThreshold in limits.items(): 

1905 if not math.isnan(maskThreshold): 

1906 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1907 isrFunctions.makeThresholdMask( 

1908 maskedImage=dataView, 

1909 threshold=maskThreshold, 

1910 growFootprints=0, 

1911 maskName=maskName 

1912 ) 

1913 

1914 # Determine if we've fully masked this amplifier with SUSPECT and 

1915 # SAT pixels. 

1916 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1917 afwImage.PARENT) 

1918 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1919 self.config.suspectMaskName]) 

1920 if numpy.all(maskView.getArray() & maskVal > 0): 

1921 badAmp = True 

1922 maskView |= maskView.getPlaneBitMask("BAD") 

1923 

1924 return badAmp 

1925 

1926 def overscanCorrection(self, ccdExposure, amp): 

1927 """Apply overscan correction in place. 

1928 

1929 This method does initial pixel rejection of the overscan 

1930 region. The overscan can also be optionally segmented to 

1931 allow for discontinuous overscan responses to be fit 

1932 separately. The actual overscan subtraction is performed by 

1933 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

1934 after the amplifier is preprocessed. 

1935 

1936 Parameters 

1937 ---------- 

1938 ccdExposure : `lsst.afw.image.Exposure` 

1939 Exposure to have overscan correction performed. 

1940 amp : `lsst.afw.cameraGeom.Amplifer` 

1941 The amplifier to consider while correcting the overscan. 

1942 

1943 Returns 

1944 ------- 

1945 overscanResults : `lsst.pipe.base.Struct` 

1946 Result struct with components: 

1947 

1948 ``imageFit`` 

1949 Value or fit subtracted from the amplifier image data. 

1950 (scalar or `lsst.afw.image.Image`) 

1951 ``overscanFit`` 

1952 Value or fit subtracted from the overscan image data. 

1953 (scalar or `lsst.afw.image.Image`) 

1954 ``overscanImage`` 

1955 Image of the overscan region with the overscan 

1956 correction applied. This quantity is used to estimate 

1957 the amplifier read noise empirically. 

1958 (`lsst.afw.image.Image`) 

1959 ``edgeMask`` 

1960 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

1961 ``overscanMean`` 

1962 Median overscan fit value. (`float`) 

1963 ``overscanSigma`` 

1964 Clipped standard deviation of the overscan after 

1965 correction. (`float`) 

1966 

1967 Raises 

1968 ------ 

1969 RuntimeError 

1970 Raised if the ``amp`` does not contain raw pixel information. 

1971 

1972 See Also 

1973 -------- 

1974 lsst.ip.isr.overscan.OverscanTask 

1975 """ 

1976 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1977 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1978 return None 

1979 

1980 # Perform overscan correction on subregions. 

1981 overscanResults = self.overscan.run(ccdExposure, amp) 

1982 

1983 metadata = ccdExposure.getMetadata() 

1984 ampName = amp.getName() 

1985 

1986 keyBase = "LSST ISR OVERSCAN" 

1987 # Updated quantities 

1988 if isinstance(overscanResults.overscanMean, float): 

1989 # Serial overscan correction only: 

1990 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

1991 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

1992 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

1993 

1994 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

1995 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

1996 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

1997 elif isinstance(overscanResults.overscanMean, tuple): 

1998 # Both serial and parallel overscan have run: 

1999 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

2000 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

2001 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

2002 

2003 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

2004 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

2005 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

2006 

2007 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

2008 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

2009 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

2010 

2011 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

2012 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

2013 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

2014 else: 

2015 self.log.warning("Unexpected type for overscan values; none added to header.") 

2016 

2017 return overscanResults 

2018 

2019 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2020 """Set the variance plane using the gain and read noise 

2021 

2022 The read noise is calculated from the ``overscanImage`` if the 

2023 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2024 the value from the amplifier data is used. 

2025 

2026 Parameters 

2027 ---------- 

2028 ampExposure : `lsst.afw.image.Exposure` 

2029 Exposure to process. 

2030 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

2031 Amplifier detector data. 

2032 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2033 Image of overscan, required only for empirical read noise. 

2034 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2035 PTC dataset containing the gains and read noise. 

2036 

2037 Raises 

2038 ------ 

2039 RuntimeError 

2040 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2041 are ``True``, but ptcDataset is not provided. 

2042 

2043 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2044 ``overscanImage`` is ``None``. 

2045 

2046 See also 

2047 -------- 

2048 lsst.ip.isr.isrFunctions.updateVariance 

2049 """ 

2050 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2051 if self.config.usePtcGains: 

2052 if ptcDataset is None: 

2053 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2054 else: 

2055 gain = ptcDataset.gain[amp.getName()] 

2056 self.log.info("Using gain from Photon Transfer Curve.") 

2057 else: 

2058 gain = amp.getGain() 

2059 

2060 if math.isnan(gain): 

2061 gain = 1.0 

2062 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2063 elif gain <= 0: 

2064 patchedGain = 1.0 

2065 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2066 amp.getName(), gain, patchedGain) 

2067 gain = patchedGain 

2068 

2069 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2070 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(), 

2071 [self.config.saturatedMaskName, 

2072 self.config.suspectMaskName, 

2073 "BAD", "NO_DATA"]) 

2074 allPixels = ampExposure.getWidth() * ampExposure.getHeight() 

2075 if allPixels == badPixels: 

2076 # If the image is bad, do not raise. 

2077 self.log.info("Skipping empirical read noise for amp %s. No good pixels.", 

2078 amp.getName()) 

2079 else: 

2080 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2081 

2082 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2083 stats = afwMath.StatisticsControl() 

2084 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2085 readNoise = afwMath.makeStatistics(overscanImage.getImage(), 

2086 afwMath.STDEVCLIP, stats).getValue() 

2087 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2088 amp.getName(), readNoise) 

2089 elif self.config.usePtcReadNoise: 

2090 if ptcDataset is None: 

2091 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2092 else: 

2093 readNoise = ptcDataset.noise[amp.getName()] 

2094 self.log.info("Using read noise from Photon Transfer Curve.") 

2095 else: 

2096 readNoise = amp.getReadNoise() 

2097 

2098 metadata = ampExposure.getMetadata() 

2099 metadata[f'LSST GAIN {amp.getName()}'] = gain 

2100 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise 

2101 

2102 isrFunctions.updateVariance( 

2103 maskedImage=ampExposure.getMaskedImage(), 

2104 gain=gain, 

2105 readNoise=readNoise, 

2106 ) 

2107 

2108 def maskNegativeVariance(self, exposure): 

2109 """Identify and mask pixels with negative variance values. 

2110 

2111 Parameters 

2112 ---------- 

2113 exposure : `lsst.afw.image.Exposure` 

2114 Exposure to process. 

2115 

2116 See Also 

2117 -------- 

2118 lsst.ip.isr.isrFunctions.updateVariance 

2119 """ 

2120 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2121 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2122 exposure.mask.array[bad] |= maskPlane 

2123 

2124 def darkCorrection(self, exposure, darkExposure, invert=False): 

2125 """Apply dark correction in place. 

2126 

2127 Parameters 

2128 ---------- 

2129 exposure : `lsst.afw.image.Exposure` 

2130 Exposure to process. 

2131 darkExposure : `lsst.afw.image.Exposure` 

2132 Dark exposure of the same size as ``exposure``. 

2133 invert : `Bool`, optional 

2134 If True, re-add the dark to an already corrected image. 

2135 

2136 Raises 

2137 ------ 

2138 RuntimeError 

2139 Raised if either ``exposure`` or ``darkExposure`` do not 

2140 have their dark time defined. 

2141 

2142 See Also 

2143 -------- 

2144 lsst.ip.isr.isrFunctions.darkCorrection 

2145 """ 

2146 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2147 if math.isnan(expScale): 

2148 raise RuntimeError("Exposure darktime is NAN.") 

2149 if darkExposure.getInfo().getVisitInfo() is not None \ 

2150 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2151 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2152 else: 

2153 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2154 # so getDarkTime() does not exist. 

2155 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2156 darkScale = 1.0 

2157 

2158 isrFunctions.darkCorrection( 

2159 maskedImage=exposure.getMaskedImage(), 

2160 darkMaskedImage=darkExposure.getMaskedImage(), 

2161 expScale=expScale, 

2162 darkScale=darkScale, 

2163 invert=invert, 

2164 trimToFit=self.config.doTrimToMatchCalib 

2165 ) 

2166 

2167 def doLinearize(self, detector): 

2168 """Check if linearization is needed for the detector cameraGeom. 

2169 

2170 Checks config.doLinearize and the linearity type of the first 

2171 amplifier. 

2172 

2173 Parameters 

2174 ---------- 

2175 detector : `lsst.afw.cameraGeom.Detector` 

2176 Detector to get linearity type from. 

2177 

2178 Returns 

2179 ------- 

2180 doLinearize : `Bool` 

2181 If True, linearization should be performed. 

2182 """ 

2183 return self.config.doLinearize and \ 

2184 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2185 

2186 def flatCorrection(self, exposure, flatExposure, invert=False): 

2187 """Apply flat correction in place. 

2188 

2189 Parameters 

2190 ---------- 

2191 exposure : `lsst.afw.image.Exposure` 

2192 Exposure to process. 

2193 flatExposure : `lsst.afw.image.Exposure` 

2194 Flat exposure of the same size as ``exposure``. 

2195 invert : `Bool`, optional 

2196 If True, unflatten an already flattened image. 

2197 

2198 See Also 

2199 -------- 

2200 lsst.ip.isr.isrFunctions.flatCorrection 

2201 """ 

2202 isrFunctions.flatCorrection( 

2203 maskedImage=exposure.getMaskedImage(), 

2204 flatMaskedImage=flatExposure.getMaskedImage(), 

2205 scalingType=self.config.flatScalingType, 

2206 userScale=self.config.flatUserScale, 

2207 invert=invert, 

2208 trimToFit=self.config.doTrimToMatchCalib 

2209 ) 

2210 

2211 def saturationDetection(self, exposure, amp): 

2212 """Detect and mask saturated pixels in config.saturatedMaskName. 

2213 

2214 Parameters 

2215 ---------- 

2216 exposure : `lsst.afw.image.Exposure` 

2217 Exposure to process. Only the amplifier DataSec is processed. 

2218 amp : `lsst.afw.cameraGeom.Amplifier` 

2219 Amplifier detector data. 

2220 

2221 See Also 

2222 -------- 

2223 lsst.ip.isr.isrFunctions.makeThresholdMask 

2224 """ 

2225 if not math.isnan(amp.getSaturation()): 

2226 maskedImage = exposure.getMaskedImage() 

2227 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2228 isrFunctions.makeThresholdMask( 

2229 maskedImage=dataView, 

2230 threshold=amp.getSaturation(), 

2231 growFootprints=0, 

2232 maskName=self.config.saturatedMaskName, 

2233 ) 

2234 

2235 def saturationInterpolation(self, exposure): 

2236 """Interpolate over saturated pixels, in place. 

2237 

2238 This method should be called after `saturationDetection`, to 

2239 ensure that the saturated pixels have been identified in the 

2240 SAT mask. It should also be called after `assembleCcd`, since 

2241 saturated regions may cross amplifier boundaries. 

2242 

2243 Parameters 

2244 ---------- 

2245 exposure : `lsst.afw.image.Exposure` 

2246 Exposure to process. 

2247 

2248 See Also 

2249 -------- 

2250 lsst.ip.isr.isrTask.saturationDetection 

2251 lsst.ip.isr.isrFunctions.interpolateFromMask 

2252 """ 

2253 isrFunctions.interpolateFromMask( 

2254 maskedImage=exposure.getMaskedImage(), 

2255 fwhm=self.config.fwhm, 

2256 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2257 maskNameList=list(self.config.saturatedMaskName), 

2258 ) 

2259 

2260 def suspectDetection(self, exposure, amp): 

2261 """Detect and mask suspect pixels in config.suspectMaskName. 

2262 

2263 Parameters 

2264 ---------- 

2265 exposure : `lsst.afw.image.Exposure` 

2266 Exposure to process. Only the amplifier DataSec is processed. 

2267 amp : `lsst.afw.cameraGeom.Amplifier` 

2268 Amplifier detector data. 

2269 

2270 See Also 

2271 -------- 

2272 lsst.ip.isr.isrFunctions.makeThresholdMask 

2273 

2274 Notes 

2275 ----- 

2276 Suspect pixels are pixels whose value is greater than 

2277 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2278 affected by unknown systematics; for example if non-linearity 

2279 corrections above a certain level are unstable then that would be a 

2280 useful value for suspectLevel. A value of `nan` indicates that no such 

2281 level exists and no pixels are to be masked as suspicious. 

2282 """ 

2283 suspectLevel = amp.getSuspectLevel() 

2284 if math.isnan(suspectLevel): 

2285 return 

2286 

2287 maskedImage = exposure.getMaskedImage() 

2288 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2289 isrFunctions.makeThresholdMask( 

2290 maskedImage=dataView, 

2291 threshold=suspectLevel, 

2292 growFootprints=0, 

2293 maskName=self.config.suspectMaskName, 

2294 ) 

2295 

2296 def maskDefect(self, exposure, defectBaseList): 

2297 """Mask defects using mask plane "BAD", in place. 

2298 

2299 Parameters 

2300 ---------- 

2301 exposure : `lsst.afw.image.Exposure` 

2302 Exposure to process. 

2303 defectBaseList : defect-type 

2304 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2305 or `list` of `lsst.afw.image.DefectBase`. 

2306 

2307 Notes 

2308 ----- 

2309 Call this after CCD assembly, since defects may cross amplifier 

2310 boundaries. 

2311 """ 

2312 maskedImage = exposure.getMaskedImage() 

2313 if not isinstance(defectBaseList, Defects): 

2314 # Promotes DefectBase to Defect 

2315 defectList = Defects(defectBaseList) 

2316 else: 

2317 defectList = defectBaseList 

2318 defectList.maskPixels(maskedImage, maskName="BAD") 

2319 

2320 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2321 """Mask edge pixels with applicable mask plane. 

2322 

2323 Parameters 

2324 ---------- 

2325 exposure : `lsst.afw.image.Exposure` 

2326 Exposure to process. 

2327 numEdgePixels : `int`, optional 

2328 Number of edge pixels to mask. 

2329 maskPlane : `str`, optional 

2330 Mask plane name to use. 

2331 level : `str`, optional 

2332 Level at which to mask edges. 

2333 """ 

2334 maskedImage = exposure.getMaskedImage() 

2335 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2336 

2337 if numEdgePixels > 0: 

2338 if level == 'DETECTOR': 

2339 boxes = [maskedImage.getBBox()] 

2340 elif level == 'AMP': 

2341 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2342 

2343 for box in boxes: 

2344 # This makes a bbox numEdgeSuspect pixels smaller than the 

2345 # image on each side 

2346 subImage = maskedImage[box] 

2347 box.grow(-numEdgePixels) 

2348 # Mask pixels outside box 

2349 SourceDetectionTask.setEdgeBits( 

2350 subImage, 

2351 box, 

2352 maskBitMask) 

2353 

2354 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2355 """Mask and interpolate defects using mask plane "BAD", in place. 

2356 

2357 Parameters 

2358 ---------- 

2359 exposure : `lsst.afw.image.Exposure` 

2360 Exposure to process. 

2361 defectBaseList : defects-like 

2362 List of defects to mask and interpolate. Can be 

2363 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2364 

2365 See Also 

2366 -------- 

2367 lsst.ip.isr.isrTask.maskDefect 

2368 """ 

2369 self.maskDefect(exposure, defectBaseList) 

2370 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2371 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2372 isrFunctions.interpolateFromMask( 

2373 maskedImage=exposure.getMaskedImage(), 

2374 fwhm=self.config.fwhm, 

2375 growSaturatedFootprints=0, 

2376 maskNameList=["BAD"], 

2377 ) 

2378 

2379 def maskNan(self, exposure): 

2380 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2381 

2382 Parameters 

2383 ---------- 

2384 exposure : `lsst.afw.image.Exposure` 

2385 Exposure to process. 

2386 

2387 Notes 

2388 ----- 

2389 We mask over all non-finite values (NaN, inf), including those 

2390 that are masked with other bits (because those may or may not be 

2391 interpolated over later, and we want to remove all NaN/infs). 

2392 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2393 preserve the historical name. 

2394 """ 

2395 maskedImage = exposure.getMaskedImage() 

2396 

2397 # Find and mask NaNs 

2398 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2399 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2400 numNans = maskNans(maskedImage, maskVal) 

2401 self.metadata["NUMNANS"] = numNans 

2402 if numNans > 0: 

2403 self.log.warning("There were %d unmasked NaNs.", numNans) 

2404 

2405 def maskAndInterpolateNan(self, exposure): 

2406 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2407 in place. 

2408 

2409 Parameters 

2410 ---------- 

2411 exposure : `lsst.afw.image.Exposure` 

2412 Exposure to process. 

2413 

2414 See Also 

2415 -------- 

2416 lsst.ip.isr.isrTask.maskNan 

2417 """ 

2418 self.maskNan(exposure) 

2419 isrFunctions.interpolateFromMask( 

2420 maskedImage=exposure.getMaskedImage(), 

2421 fwhm=self.config.fwhm, 

2422 growSaturatedFootprints=0, 

2423 maskNameList=["UNMASKEDNAN"], 

2424 ) 

2425 

2426 def measureBackground(self, exposure, IsrQaConfig=None): 

2427 """Measure the image background in subgrids, for quality control. 

2428 

2429 Parameters 

2430 ---------- 

2431 exposure : `lsst.afw.image.Exposure` 

2432 Exposure to process. 

2433 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2434 Configuration object containing parameters on which background 

2435 statistics and subgrids to use. 

2436 """ 

2437 if IsrQaConfig is not None: 

2438 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2439 IsrQaConfig.flatness.nIter) 

2440 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2441 statsControl.setAndMask(maskVal) 

2442 maskedImage = exposure.getMaskedImage() 

2443 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2444 skyLevel = stats.getValue(afwMath.MEDIAN) 

2445 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2446 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2447 metadata = exposure.getMetadata() 

2448 metadata["SKYLEVEL"] = skyLevel 

2449 metadata["SKYSIGMA"] = skySigma 

2450 

2451 # calcluating flatlevel over the subgrids 

2452 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2453 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2454 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2455 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2456 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2457 skyLevels = numpy.zeros((nX, nY)) 

2458 

2459 for j in range(nY): 

2460 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2461 for i in range(nX): 

2462 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2463 

2464 xLLC = xc - meshXHalf 

2465 yLLC = yc - meshYHalf 

2466 xURC = xc + meshXHalf - 1 

2467 yURC = yc + meshYHalf - 1 

2468 

2469 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2470 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2471 

2472 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2473 

2474 good = numpy.where(numpy.isfinite(skyLevels)) 

2475 skyMedian = numpy.median(skyLevels[good]) 

2476 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2477 flatness_rms = numpy.std(flatness) 

2478 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2479 

2480 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2481 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2482 nX, nY, flatness_pp, flatness_rms) 

2483 

2484 metadata["FLATNESS_PP"] = float(flatness_pp) 

2485 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2486 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2487 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2488 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2489 

2490 def roughZeroPoint(self, exposure): 

2491 """Set an approximate magnitude zero point for the exposure. 

2492 

2493 Parameters 

2494 ---------- 

2495 exposure : `lsst.afw.image.Exposure` 

2496 Exposure to process. 

2497 """ 

2498 filterLabel = exposure.getFilter() 

2499 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2500 

2501 if physicalFilter in self.config.fluxMag0T1: 

2502 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2503 else: 

2504 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2505 fluxMag0 = self.config.defaultFluxMag0T1 

2506 

2507 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2508 if not expTime > 0: # handle NaN as well as <= 0 

2509 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2510 return 

2511 

2512 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2513 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2514 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2515 

2516 @contextmanager 

2517 def flatContext(self, exp, flat, dark=None): 

2518 """Context manager that applies and removes flats and darks, 

2519 if the task is configured to apply them. 

2520 

2521 Parameters 

2522 ---------- 

2523 exp : `lsst.afw.image.Exposure` 

2524 Exposure to process. 

2525 flat : `lsst.afw.image.Exposure` 

2526 Flat exposure the same size as ``exp``. 

2527 dark : `lsst.afw.image.Exposure`, optional 

2528 Dark exposure the same size as ``exp``. 

2529 

2530 Yields 

2531 ------ 

2532 exp : `lsst.afw.image.Exposure` 

2533 The flat and dark corrected exposure. 

2534 """ 

2535 if self.config.doDark and dark is not None: 

2536 self.darkCorrection(exp, dark) 

2537 if self.config.doFlat: 

2538 self.flatCorrection(exp, flat) 

2539 try: 

2540 yield exp 

2541 finally: 

2542 if self.config.doFlat: 

2543 self.flatCorrection(exp, flat, invert=True) 

2544 if self.config.doDark and dark is not None: 

2545 self.darkCorrection(exp, dark, invert=True) 

2546 

2547 def debugView(self, exposure, stepname): 

2548 """Utility function to examine ISR exposure at different stages. 

2549 

2550 Parameters 

2551 ---------- 

2552 exposure : `lsst.afw.image.Exposure` 

2553 Exposure to view. 

2554 stepname : `str` 

2555 State of processing to view. 

2556 """ 

2557 frame = getDebugFrame(self._display, stepname) 

2558 if frame: 

2559 display = getDisplay(frame) 

2560 display.scale('asinh', 'zscale') 

2561 display.mtv(exposure) 

2562 prompt = "Press Enter to continue [c]... " 

2563 while True: 

2564 ans = input(prompt).lower() 

2565 if ans in ("", "c",): 

2566 break 

2567 

2568 

2569class FakeAmp(object): 

2570 """A Detector-like object that supports returning gain and saturation level 

2571 

2572 This is used when the input exposure does not have a detector. 

2573 

2574 Parameters 

2575 ---------- 

2576 exposure : `lsst.afw.image.Exposure` 

2577 Exposure to generate a fake amplifier for. 

2578 config : `lsst.ip.isr.isrTaskConfig` 

2579 Configuration to apply to the fake amplifier. 

2580 """ 

2581 

2582 def __init__(self, exposure, config): 

2583 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2584 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2585 self._gain = config.gain 

2586 self._readNoise = config.readNoise 

2587 self._saturation = config.saturation 

2588 

2589 def getBBox(self): 

2590 return self._bbox 

2591 

2592 def getRawBBox(self): 

2593 return self._bbox 

2594 

2595 def getRawHorizontalOverscanBBox(self): 

2596 return self._RawHorizontalOverscanBBox 

2597 

2598 def getGain(self): 

2599 return self._gain 

2600 

2601 def getReadNoise(self): 

2602 return self._readNoise 

2603 

2604 def getSaturation(self): 

2605 return self._saturation 

2606 

2607 def getSuspectLevel(self): 

2608 return float("NaN")