Coverage for python/lsst/ip/isr/isrTask.py: 16%

938 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-30 12:54 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from .ptcDataset import PhotonTransferCurveDataset 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.DataCoordinate` 

75 Expanded data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(registry.dimensions.conform(["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 records = {k: newDataId.records[k] for k in newDataId.dimensions.elements} 

95 return [ref.expanded(registry.expandDataId(ref.dataId, records=records)) for ref in results] 

96 

97 

98class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

99 dimensions={"instrument", "exposure", "detector"}, 

100 defaultTemplates={}): 

101 ccdExposure = cT.Input( 

102 name="raw", 

103 doc="Input exposure to process.", 

104 storageClass="Exposure", 

105 dimensions=["instrument", "exposure", "detector"], 

106 ) 

107 camera = cT.PrerequisiteInput( 

108 name="camera", 

109 storageClass="Camera", 

110 doc="Input camera to construct complete exposures.", 

111 dimensions=["instrument"], 

112 isCalibration=True, 

113 ) 

114 

115 crosstalk = cT.PrerequisiteInput( 

116 name="crosstalk", 

117 doc="Input crosstalk object", 

118 storageClass="CrosstalkCalib", 

119 dimensions=["instrument", "detector"], 

120 isCalibration=True, 

121 minimum=0, # can fall back to cameraGeom 

122 ) 

123 crosstalkSources = cT.PrerequisiteInput( 

124 name="isrOverscanCorrected", 

125 doc="Overscan corrected input images.", 

126 storageClass="Exposure", 

127 dimensions=["instrument", "exposure", "detector"], 

128 deferLoad=True, 

129 multiple=True, 

130 lookupFunction=crosstalkSourceLookup, 

131 minimum=0, # not needed for all instruments, no config to control this 

132 ) 

133 bias = cT.PrerequisiteInput( 

134 name="bias", 

135 doc="Input bias calibration.", 

136 storageClass="ExposureF", 

137 dimensions=["instrument", "detector"], 

138 isCalibration=True, 

139 ) 

140 dark = cT.PrerequisiteInput( 

141 name='dark', 

142 doc="Input dark calibration.", 

143 storageClass="ExposureF", 

144 dimensions=["instrument", "detector"], 

145 isCalibration=True, 

146 ) 

147 flat = cT.PrerequisiteInput( 

148 name="flat", 

149 doc="Input flat calibration.", 

150 storageClass="ExposureF", 

151 dimensions=["instrument", "physical_filter", "detector"], 

152 isCalibration=True, 

153 ) 

154 ptc = cT.PrerequisiteInput( 

155 name="ptc", 

156 doc="Input Photon Transfer Curve dataset", 

157 storageClass="PhotonTransferCurveDataset", 

158 dimensions=["instrument", "detector"], 

159 isCalibration=True, 

160 ) 

161 fringes = cT.PrerequisiteInput( 

162 name="fringe", 

163 doc="Input fringe calibration.", 

164 storageClass="ExposureF", 

165 dimensions=["instrument", "physical_filter", "detector"], 

166 isCalibration=True, 

167 minimum=0, # only needed for some bands, even when enabled 

168 ) 

169 strayLightData = cT.PrerequisiteInput( 

170 name='yBackground', 

171 doc="Input stray light calibration.", 

172 storageClass="StrayLightData", 

173 dimensions=["instrument", "physical_filter", "detector"], 

174 deferLoad=True, 

175 isCalibration=True, 

176 minimum=0, # only needed for some bands, even when enabled 

177 ) 

178 bfKernel = cT.PrerequisiteInput( 

179 name='bfKernel', 

180 doc="Input brighter-fatter kernel.", 

181 storageClass="NumpyArray", 

182 dimensions=["instrument"], 

183 isCalibration=True, 

184 minimum=0, # can use either bfKernel or newBFKernel 

185 ) 

186 newBFKernel = cT.PrerequisiteInput( 

187 name='brighterFatterKernel', 

188 doc="Newer complete kernel + gain solutions.", 

189 storageClass="BrighterFatterKernel", 

190 dimensions=["instrument", "detector"], 

191 isCalibration=True, 

192 minimum=0, # can use either bfKernel or newBFKernel 

193 ) 

194 defects = cT.PrerequisiteInput( 

195 name='defects', 

196 doc="Input defect tables.", 

197 storageClass="Defects", 

198 dimensions=["instrument", "detector"], 

199 isCalibration=True, 

200 ) 

201 linearizer = cT.PrerequisiteInput( 

202 name='linearizer', 

203 storageClass="Linearizer", 

204 doc="Linearity correction calibration.", 

205 dimensions=["instrument", "detector"], 

206 isCalibration=True, 

207 minimum=0, # can fall back to cameraGeom 

208 ) 

209 opticsTransmission = cT.PrerequisiteInput( 

210 name="transmission_optics", 

211 storageClass="TransmissionCurve", 

212 doc="Transmission curve due to the optics.", 

213 dimensions=["instrument"], 

214 isCalibration=True, 

215 ) 

216 filterTransmission = cT.PrerequisiteInput( 

217 name="transmission_filter", 

218 storageClass="TransmissionCurve", 

219 doc="Transmission curve due to the filter.", 

220 dimensions=["instrument", "physical_filter"], 

221 isCalibration=True, 

222 ) 

223 sensorTransmission = cT.PrerequisiteInput( 

224 name="transmission_sensor", 

225 storageClass="TransmissionCurve", 

226 doc="Transmission curve due to the sensor.", 

227 dimensions=["instrument", "detector"], 

228 isCalibration=True, 

229 ) 

230 atmosphereTransmission = cT.PrerequisiteInput( 

231 name="transmission_atmosphere", 

232 storageClass="TransmissionCurve", 

233 doc="Transmission curve due to the atmosphere.", 

234 dimensions=["instrument"], 

235 isCalibration=True, 

236 ) 

237 illumMaskedImage = cT.PrerequisiteInput( 

238 name="illum", 

239 doc="Input illumination correction.", 

240 storageClass="MaskedImageF", 

241 dimensions=["instrument", "physical_filter", "detector"], 

242 isCalibration=True, 

243 ) 

244 deferredChargeCalib = cT.PrerequisiteInput( 

245 name="cpCtiCalib", 

246 doc="Deferred charge/CTI correction dataset.", 

247 storageClass="IsrCalib", 

248 dimensions=["instrument", "detector"], 

249 isCalibration=True, 

250 ) 

251 

252 outputExposure = cT.Output( 

253 name='postISRCCD', 

254 doc="Output ISR processed exposure.", 

255 storageClass="Exposure", 

256 dimensions=["instrument", "exposure", "detector"], 

257 ) 

258 preInterpExposure = cT.Output( 

259 name='preInterpISRCCD', 

260 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

261 storageClass="ExposureF", 

262 dimensions=["instrument", "exposure", "detector"], 

263 ) 

264 outputBin1Exposure = cT.Output( 

265 name="postIsrBin1", 

266 doc="First binned image.", 

267 storageClass="ExposureF", 

268 dimensions=["instrument", "exposure", "detector"], 

269 ) 

270 outputBin2Exposure = cT.Output( 

271 name="postIsrBin2", 

272 doc="Second binned image.", 

273 storageClass="ExposureF", 

274 dimensions=["instrument", "exposure", "detector"], 

275 ) 

276 

277 outputOssThumbnail = cT.Output( 

278 name="OssThumb", 

279 doc="Output Overscan-subtracted thumbnail image.", 

280 storageClass="Thumbnail", 

281 dimensions=["instrument", "exposure", "detector"], 

282 ) 

283 outputFlattenedThumbnail = cT.Output( 

284 name="FlattenedThumb", 

285 doc="Output flat-corrected thumbnail image.", 

286 storageClass="Thumbnail", 

287 dimensions=["instrument", "exposure", "detector"], 

288 ) 

289 outputStatistics = cT.Output( 

290 name="isrStatistics", 

291 doc="Output of additional statistics table.", 

292 storageClass="StructuredDataDict", 

293 dimensions=["instrument", "exposure", "detector"], 

294 ) 

295 

296 def __init__(self, *, config=None): 

297 super().__init__(config=config) 

298 

299 if config.doBias is not True: 

300 self.prerequisiteInputs.remove("bias") 

301 if config.doLinearize is not True: 

302 self.prerequisiteInputs.remove("linearizer") 

303 if config.doCrosstalk is not True: 

304 self.prerequisiteInputs.remove("crosstalkSources") 

305 self.prerequisiteInputs.remove("crosstalk") 

306 if config.doBrighterFatter is not True: 

307 self.prerequisiteInputs.remove("bfKernel") 

308 self.prerequisiteInputs.remove("newBFKernel") 

309 if config.doDefect is not True: 

310 self.prerequisiteInputs.remove("defects") 

311 if config.doDark is not True: 

312 self.prerequisiteInputs.remove("dark") 

313 if config.doFlat is not True: 

314 self.prerequisiteInputs.remove("flat") 

315 if config.doFringe is not True: 

316 self.prerequisiteInputs.remove("fringes") 

317 if config.doStrayLight is not True: 

318 self.prerequisiteInputs.remove("strayLightData") 

319 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

320 self.prerequisiteInputs.remove("ptc") 

321 if config.doAttachTransmissionCurve is not True: 

322 self.prerequisiteInputs.remove("opticsTransmission") 

323 self.prerequisiteInputs.remove("filterTransmission") 

324 self.prerequisiteInputs.remove("sensorTransmission") 

325 self.prerequisiteInputs.remove("atmosphereTransmission") 

326 else: 

327 if config.doUseOpticsTransmission is not True: 

328 self.prerequisiteInputs.remove("opticsTransmission") 

329 if config.doUseFilterTransmission is not True: 

330 self.prerequisiteInputs.remove("filterTransmission") 

331 if config.doUseSensorTransmission is not True: 

332 self.prerequisiteInputs.remove("sensorTransmission") 

333 if config.doUseAtmosphereTransmission is not True: 

334 self.prerequisiteInputs.remove("atmosphereTransmission") 

335 if config.doIlluminationCorrection is not True: 

336 self.prerequisiteInputs.remove("illumMaskedImage") 

337 if config.doDeferredCharge is not True: 

338 self.prerequisiteInputs.remove("deferredChargeCalib") 

339 

340 if config.doWrite is not True: 

341 self.outputs.remove("outputExposure") 

342 self.outputs.remove("preInterpExposure") 

343 self.outputs.remove("outputFlattenedThumbnail") 

344 self.outputs.remove("outputOssThumbnail") 

345 self.outputs.remove("outputStatistics") 

346 self.outputs.remove("outputBin1Exposure") 

347 self.outputs.remove("outputBin2Exposure") 

348 

349 if config.doBinnedExposures is not True: 

350 self.outputs.remove("outputBin1Exposure") 

351 self.outputs.remove("outputBin2Exposure") 

352 if config.doSaveInterpPixels is not True: 

353 self.outputs.remove("preInterpExposure") 

354 if config.qa.doThumbnailOss is not True: 

355 self.outputs.remove("outputOssThumbnail") 

356 if config.qa.doThumbnailFlattened is not True: 

357 self.outputs.remove("outputFlattenedThumbnail") 

358 if config.doCalculateStatistics is not True: 

359 self.outputs.remove("outputStatistics") 

360 

361 

362class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

363 pipelineConnections=IsrTaskConnections): 

364 """Configuration parameters for IsrTask. 

365 

366 Items are grouped in the order in which they are executed by the task. 

367 """ 

368 datasetType = pexConfig.Field( 

369 dtype=str, 

370 doc="Dataset type for input data; users will typically leave this alone, " 

371 "but camera-specific ISR tasks will override it", 

372 default="raw", 

373 ) 

374 

375 fallbackFilterName = pexConfig.Field( 

376 dtype=str, 

377 doc="Fallback default filter name for calibrations.", 

378 optional=True 

379 ) 

380 useFallbackDate = pexConfig.Field( 

381 dtype=bool, 

382 doc="Pass observation date when using fallback filter.", 

383 default=False, 

384 ) 

385 expectWcs = pexConfig.Field( 

386 dtype=bool, 

387 default=True, 

388 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

389 ) 

390 fwhm = pexConfig.Field( 

391 dtype=float, 

392 doc="FWHM of PSF in arcseconds (currently unused).", 

393 default=1.0, 

394 ) 

395 qa = pexConfig.ConfigField( 

396 dtype=isrQa.IsrQaConfig, 

397 doc="QA related configuration options.", 

398 ) 

399 doHeaderProvenance = pexConfig.Field( 

400 dtype=bool, 

401 default=True, 

402 doc="Write calibration identifiers into output exposure header?", 

403 ) 

404 

405 # Calib checking configuration: 

406 doRaiseOnCalibMismatch = pexConfig.Field( 

407 dtype=bool, 

408 default=False, 

409 doc="Should IsrTask halt if exposure and calibration header values do not match?", 

410 ) 

411 cameraKeywordsToCompare = pexConfig.ListField( 

412 dtype=str, 

413 doc="List of header keywords to compare between exposure and calibrations.", 

414 default=[], 

415 ) 

416 

417 # Image conversion configuration 

418 doConvertIntToFloat = pexConfig.Field( 

419 dtype=bool, 

420 doc="Convert integer raw images to floating point values?", 

421 default=True, 

422 ) 

423 

424 # Saturated pixel handling. 

425 doSaturation = pexConfig.Field( 

426 dtype=bool, 

427 doc="Mask saturated pixels? NB: this is totally independent of the" 

428 " interpolation option - this is ONLY setting the bits in the mask." 

429 " To have them interpolated make sure doSaturationInterpolation=True", 

430 default=True, 

431 ) 

432 saturatedMaskName = pexConfig.Field( 

433 dtype=str, 

434 doc="Name of mask plane to use in saturation detection and interpolation", 

435 default="SAT", 

436 ) 

437 saturation = pexConfig.Field( 

438 dtype=float, 

439 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

440 default=float("NaN"), 

441 ) 

442 growSaturationFootprintSize = pexConfig.Field( 

443 dtype=int, 

444 doc="Number of pixels by which to grow the saturation footprints", 

445 default=1, 

446 ) 

447 

448 # Suspect pixel handling. 

449 doSuspect = pexConfig.Field( 

450 dtype=bool, 

451 doc="Mask suspect pixels?", 

452 default=False, 

453 ) 

454 suspectMaskName = pexConfig.Field( 

455 dtype=str, 

456 doc="Name of mask plane to use for suspect pixels", 

457 default="SUSPECT", 

458 ) 

459 numEdgeSuspect = pexConfig.Field( 

460 dtype=int, 

461 doc="Number of edge pixels to be flagged as untrustworthy.", 

462 default=0, 

463 ) 

464 edgeMaskLevel = pexConfig.ChoiceField( 

465 dtype=str, 

466 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

467 default="DETECTOR", 

468 allowed={ 

469 'DETECTOR': 'Mask only the edges of the full detector.', 

470 'AMP': 'Mask edges of each amplifier.', 

471 }, 

472 ) 

473 

474 # Initial masking options. 

475 doSetBadRegions = pexConfig.Field( 

476 dtype=bool, 

477 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

478 default=True, 

479 ) 

480 badStatistic = pexConfig.ChoiceField( 

481 dtype=str, 

482 doc="How to estimate the average value for BAD regions.", 

483 default='MEANCLIP', 

484 allowed={ 

485 "MEANCLIP": "Correct using the (clipped) mean of good data", 

486 "MEDIAN": "Correct using the median of the good data", 

487 }, 

488 ) 

489 

490 # Overscan subtraction configuration. 

491 doOverscan = pexConfig.Field( 

492 dtype=bool, 

493 doc="Do overscan subtraction?", 

494 default=True, 

495 ) 

496 overscan = pexConfig.ConfigurableField( 

497 target=OverscanCorrectionTask, 

498 doc="Overscan subtraction task for image segments.", 

499 ) 

500 

501 # Amplifier to CCD assembly configuration 

502 doAssembleCcd = pexConfig.Field( 

503 dtype=bool, 

504 default=True, 

505 doc="Assemble amp-level exposures into a ccd-level exposure?" 

506 ) 

507 assembleCcd = pexConfig.ConfigurableField( 

508 target=AssembleCcdTask, 

509 doc="CCD assembly task", 

510 ) 

511 

512 # General calibration configuration. 

513 doAssembleIsrExposures = pexConfig.Field( 

514 dtype=bool, 

515 default=False, 

516 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

517 ) 

518 doTrimToMatchCalib = pexConfig.Field( 

519 dtype=bool, 

520 default=False, 

521 doc="Trim raw data to match calibration bounding boxes?" 

522 ) 

523 

524 # Bias subtraction. 

525 doBias = pexConfig.Field( 

526 dtype=bool, 

527 doc="Apply bias frame correction?", 

528 default=True, 

529 ) 

530 biasDataProductName = pexConfig.Field( 

531 dtype=str, 

532 doc="Name of the bias data product", 

533 default="bias", 

534 ) 

535 doBiasBeforeOverscan = pexConfig.Field( 

536 dtype=bool, 

537 doc="Reverse order of overscan and bias correction.", 

538 default=False 

539 ) 

540 

541 # Deferred charge correction. 

542 doDeferredCharge = pexConfig.Field( 

543 dtype=bool, 

544 doc="Apply deferred charge correction?", 

545 default=False, 

546 ) 

547 deferredChargeCorrection = pexConfig.ConfigurableField( 

548 target=DeferredChargeTask, 

549 doc="Deferred charge correction task.", 

550 ) 

551 

552 # Variance construction 

553 doVariance = pexConfig.Field( 

554 dtype=bool, 

555 doc="Calculate variance?", 

556 default=True 

557 ) 

558 gain = pexConfig.Field( 

559 dtype=float, 

560 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

561 default=float("NaN"), 

562 ) 

563 readNoise = pexConfig.Field( 

564 dtype=float, 

565 doc="The read noise to use if no Detector is present in the Exposure", 

566 default=0.0, 

567 ) 

568 doEmpiricalReadNoise = pexConfig.Field( 

569 dtype=bool, 

570 default=False, 

571 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

572 ) 

573 usePtcReadNoise = pexConfig.Field( 

574 dtype=bool, 

575 default=False, 

576 doc="Use readnoise values from the Photon Transfer Curve?" 

577 ) 

578 maskNegativeVariance = pexConfig.Field( 

579 dtype=bool, 

580 default=True, 

581 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

582 "in the measurement of the overscan at an edge due to the data falling off faster " 

583 "than the overscan model can account for it." 

584 ) 

585 negativeVarianceMaskName = pexConfig.Field( 

586 dtype=str, 

587 default="BAD", 

588 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

589 ) 

590 # Linearization. 

591 doLinearize = pexConfig.Field( 

592 dtype=bool, 

593 doc="Correct for nonlinearity of the detector's response?", 

594 default=True, 

595 ) 

596 

597 # Crosstalk. 

598 doCrosstalk = pexConfig.Field( 

599 dtype=bool, 

600 doc="Apply intra-CCD crosstalk correction?", 

601 default=False, 

602 ) 

603 doCrosstalkBeforeAssemble = pexConfig.Field( 

604 dtype=bool, 

605 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

606 default=False, 

607 ) 

608 crosstalk = pexConfig.ConfigurableField( 

609 target=CrosstalkTask, 

610 doc="Intra-CCD crosstalk correction", 

611 ) 

612 

613 # Masking options. 

614 doDefect = pexConfig.Field( 

615 dtype=bool, 

616 doc="Apply correction for CCD defects, e.g. hot pixels?", 

617 default=True, 

618 ) 

619 doNanMasking = pexConfig.Field( 

620 dtype=bool, 

621 doc="Mask non-finite (NAN, inf) pixels?", 

622 default=True, 

623 ) 

624 doWidenSaturationTrails = pexConfig.Field( 

625 dtype=bool, 

626 doc="Widen bleed trails based on their width?", 

627 default=True 

628 ) 

629 

630 # Brighter-Fatter correction. 

631 doBrighterFatter = pexConfig.Field( 

632 dtype=bool, 

633 default=False, 

634 doc="Apply the brighter-fatter correction?" 

635 ) 

636 doFluxConservingBrighterFatterCorrection = pexConfig.Field( 

637 dtype=bool, 

638 default=False, 

639 doc="Apply the flux-conserving BFE correction by Miller et al.?" 

640 ) 

641 brighterFatterLevel = pexConfig.ChoiceField( 

642 dtype=str, 

643 default="DETECTOR", 

644 doc="The level at which to correct for brighter-fatter.", 

645 allowed={ 

646 "AMP": "Every amplifier treated separately.", 

647 "DETECTOR": "One kernel per detector", 

648 } 

649 ) 

650 brighterFatterMaxIter = pexConfig.Field( 

651 dtype=int, 

652 default=10, 

653 doc="Maximum number of iterations for the brighter-fatter correction" 

654 ) 

655 brighterFatterThreshold = pexConfig.Field( 

656 dtype=float, 

657 default=1000, 

658 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

659 "absolute value of the difference between the current corrected image and the one " 

660 "from the previous iteration summed over all the pixels." 

661 ) 

662 brighterFatterApplyGain = pexConfig.Field( 

663 dtype=bool, 

664 default=True, 

665 doc="Should the gain be applied when applying the brighter-fatter correction?" 

666 ) 

667 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

668 dtype=str, 

669 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

670 "correction.", 

671 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

672 ) 

673 brighterFatterMaskGrowSize = pexConfig.Field( 

674 dtype=int, 

675 default=0, 

676 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

677 "when brighter-fatter correction is applied." 

678 ) 

679 

680 # Dark subtraction. 

681 doDark = pexConfig.Field( 

682 dtype=bool, 

683 doc="Apply dark frame correction?", 

684 default=True, 

685 ) 

686 darkDataProductName = pexConfig.Field( 

687 dtype=str, 

688 doc="Name of the dark data product", 

689 default="dark", 

690 ) 

691 

692 # Camera-specific stray light removal. 

693 doStrayLight = pexConfig.Field( 

694 dtype=bool, 

695 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

696 default=False, 

697 ) 

698 strayLight = pexConfig.ConfigurableField( 

699 target=StrayLightTask, 

700 doc="y-band stray light correction" 

701 ) 

702 

703 # Flat correction. 

704 doFlat = pexConfig.Field( 

705 dtype=bool, 

706 doc="Apply flat field correction?", 

707 default=True, 

708 ) 

709 flatDataProductName = pexConfig.Field( 

710 dtype=str, 

711 doc="Name of the flat data product", 

712 default="flat", 

713 ) 

714 flatScalingType = pexConfig.ChoiceField( 

715 dtype=str, 

716 doc="The method for scaling the flat on the fly.", 

717 default='USER', 

718 allowed={ 

719 "USER": "Scale by flatUserScale", 

720 "MEAN": "Scale by the inverse of the mean", 

721 "MEDIAN": "Scale by the inverse of the median", 

722 }, 

723 ) 

724 flatUserScale = pexConfig.Field( 

725 dtype=float, 

726 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

727 default=1.0, 

728 ) 

729 doTweakFlat = pexConfig.Field( 

730 dtype=bool, 

731 doc="Tweak flats to match observed amplifier ratios?", 

732 default=False 

733 ) 

734 

735 # Amplifier normalization based on gains instead of using flats 

736 # configuration. 

737 doApplyGains = pexConfig.Field( 

738 dtype=bool, 

739 doc="Correct the amplifiers for their gains instead of applying flat correction", 

740 default=False, 

741 ) 

742 usePtcGains = pexConfig.Field( 

743 dtype=bool, 

744 doc="Use the gain values from the input Photon Transfer Curve?", 

745 default=False, 

746 ) 

747 normalizeGains = pexConfig.Field( 

748 dtype=bool, 

749 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

750 default=False, 

751 ) 

752 

753 # Fringe correction. 

754 doFringe = pexConfig.Field( 

755 dtype=bool, 

756 doc="Apply fringe correction?", 

757 default=True, 

758 ) 

759 fringe = pexConfig.ConfigurableField( 

760 target=FringeTask, 

761 doc="Fringe subtraction task", 

762 ) 

763 fringeAfterFlat = pexConfig.Field( 

764 dtype=bool, 

765 doc="Do fringe subtraction after flat-fielding?", 

766 default=True, 

767 ) 

768 

769 # Amp offset correction. 

770 doAmpOffset = pexConfig.Field( 

771 doc="Calculate and apply amp offset corrections?", 

772 dtype=bool, 

773 default=False, 

774 ) 

775 ampOffset = pexConfig.ConfigurableField( 

776 doc="Amp offset correction task.", 

777 target=AmpOffsetTask, 

778 ) 

779 

780 # Initial CCD-level background statistics options. 

781 doMeasureBackground = pexConfig.Field( 

782 dtype=bool, 

783 doc="Measure the background level on the reduced image?", 

784 default=False, 

785 ) 

786 

787 # Camera-specific masking configuration. 

788 doCameraSpecificMasking = pexConfig.Field( 

789 dtype=bool, 

790 doc="Mask camera-specific bad regions?", 

791 default=False, 

792 ) 

793 masking = pexConfig.ConfigurableField( 

794 target=MaskingTask, 

795 doc="Masking task." 

796 ) 

797 

798 # Interpolation options. 

799 doInterpolate = pexConfig.Field( 

800 dtype=bool, 

801 doc="Interpolate masked pixels?", 

802 default=True, 

803 ) 

804 doSaturationInterpolation = pexConfig.Field( 

805 dtype=bool, 

806 doc="Perform interpolation over pixels masked as saturated?" 

807 " NB: This is independent of doSaturation; if that is False this plane" 

808 " will likely be blank, resulting in a no-op here.", 

809 default=True, 

810 ) 

811 doNanInterpolation = pexConfig.Field( 

812 dtype=bool, 

813 doc="Perform interpolation over pixels masked as NaN?" 

814 " NB: This is independent of doNanMasking; if that is False this plane" 

815 " will likely be blank, resulting in a no-op here.", 

816 default=True, 

817 ) 

818 doNanInterpAfterFlat = pexConfig.Field( 

819 dtype=bool, 

820 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

821 "also have to interpolate them before flat-fielding."), 

822 default=False, 

823 ) 

824 maskListToInterpolate = pexConfig.ListField( 

825 dtype=str, 

826 doc="List of mask planes that should be interpolated.", 

827 default=['SAT', 'BAD'], 

828 ) 

829 doSaveInterpPixels = pexConfig.Field( 

830 dtype=bool, 

831 doc="Save a copy of the pre-interpolated pixel values?", 

832 default=False, 

833 ) 

834 

835 # Default photometric calibration options. 

836 fluxMag0T1 = pexConfig.DictField( 

837 keytype=str, 

838 itemtype=float, 

839 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

840 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

841 )) 

842 ) 

843 defaultFluxMag0T1 = pexConfig.Field( 

844 dtype=float, 

845 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

846 default=pow(10.0, 0.4*28.0) 

847 ) 

848 

849 # Vignette correction configuration. 

850 doVignette = pexConfig.Field( 

851 dtype=bool, 

852 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

853 "according to vignetting parameters?"), 

854 default=False, 

855 ) 

856 doMaskVignettePolygon = pexConfig.Field( 

857 dtype=bool, 

858 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

859 "is False"), 

860 default=True, 

861 ) 

862 vignetteValue = pexConfig.Field( 

863 dtype=float, 

864 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

865 optional=True, 

866 default=None, 

867 ) 

868 vignette = pexConfig.ConfigurableField( 

869 target=VignetteTask, 

870 doc="Vignetting task.", 

871 ) 

872 

873 # Transmission curve configuration. 

874 doAttachTransmissionCurve = pexConfig.Field( 

875 dtype=bool, 

876 default=False, 

877 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

878 ) 

879 doUseOpticsTransmission = pexConfig.Field( 

880 dtype=bool, 

881 default=True, 

882 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

883 ) 

884 doUseFilterTransmission = pexConfig.Field( 

885 dtype=bool, 

886 default=True, 

887 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

888 ) 

889 doUseSensorTransmission = pexConfig.Field( 

890 dtype=bool, 

891 default=True, 

892 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

893 ) 

894 doUseAtmosphereTransmission = pexConfig.Field( 

895 dtype=bool, 

896 default=True, 

897 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

898 ) 

899 

900 # Illumination correction. 

901 doIlluminationCorrection = pexConfig.Field( 

902 dtype=bool, 

903 default=False, 

904 doc="Perform illumination correction?" 

905 ) 

906 illuminationCorrectionDataProductName = pexConfig.Field( 

907 dtype=str, 

908 doc="Name of the illumination correction data product.", 

909 default="illumcor", 

910 ) 

911 illumScale = pexConfig.Field( 

912 dtype=float, 

913 doc="Scale factor for the illumination correction.", 

914 default=1.0, 

915 ) 

916 illumFilters = pexConfig.ListField( 

917 dtype=str, 

918 default=[], 

919 doc="Only perform illumination correction for these filters." 

920 ) 

921 

922 # Calculate image quality statistics? 

923 doStandardStatistics = pexConfig.Field( 

924 dtype=bool, 

925 doc="Should standard image quality statistics be calculated?", 

926 default=True, 

927 ) 

928 # Calculate additional statistics? 

929 doCalculateStatistics = pexConfig.Field( 

930 dtype=bool, 

931 doc="Should additional ISR statistics be calculated?", 

932 default=False, 

933 ) 

934 isrStats = pexConfig.ConfigurableField( 

935 target=IsrStatisticsTask, 

936 doc="Task to calculate additional statistics.", 

937 ) 

938 

939 # Make binned images? 

940 doBinnedExposures = pexConfig.Field( 

941 dtype=bool, 

942 doc="Should binned exposures be calculated?", 

943 default=False, 

944 ) 

945 binFactor1 = pexConfig.Field( 945 ↛ exitline 945 didn't jump to the function exit

946 dtype=int, 

947 doc="Binning factor for first binned exposure. This is intended for a finely binned output.", 

948 default=8, 

949 check=lambda x: x > 1, 

950 ) 

951 binFactor2 = pexConfig.Field( 951 ↛ exitline 951 didn't jump to the function exit

952 dtype=int, 

953 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.", 

954 default=64, 

955 check=lambda x: x > 1, 

956 ) 

957 

958 # Write the outputs to disk. If ISR is run as a subtask, this may not 

959 # be needed. 

960 doWrite = pexConfig.Field( 

961 dtype=bool, 

962 doc="Persist postISRCCD?", 

963 default=True, 

964 ) 

965 

966 def validate(self): 

967 super().validate() 

968 if self.doFlat and self.doApplyGains: 

969 raise ValueError("You may not specify both doFlat and doApplyGains") 

970 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

971 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

972 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

973 self.maskListToInterpolate.append(self.saturatedMaskName) 

974 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

975 self.maskListToInterpolate.remove(self.saturatedMaskName) 

976 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

977 self.maskListToInterpolate.append("UNMASKEDNAN") 

978 

979 

980class IsrTask(pipeBase.PipelineTask): 

981 """Apply common instrument signature correction algorithms to a raw frame. 

982 

983 The process for correcting imaging data is very similar from 

984 camera to camera. This task provides a vanilla implementation of 

985 doing these corrections, including the ability to turn certain 

986 corrections off if they are not needed. The inputs to the primary 

987 method, `run()`, are a raw exposure to be corrected and the 

988 calibration data products. The raw input is a single chip sized 

989 mosaic of all amps including overscans and other non-science 

990 pixels. 

991 

992 The __init__ method sets up the subtasks for ISR processing, using 

993 the defaults from `lsst.ip.isr`. 

994 

995 Parameters 

996 ---------- 

997 args : `list` 

998 Positional arguments passed to the Task constructor. 

999 None used at this time. 

1000 kwargs : `dict`, optional 

1001 Keyword arguments passed on to the Task constructor. 

1002 None used at this time. 

1003 """ 

1004 ConfigClass = IsrTaskConfig 

1005 _DefaultName = "isr" 

1006 

1007 def __init__(self, **kwargs): 

1008 super().__init__(**kwargs) 

1009 self.makeSubtask("assembleCcd") 

1010 self.makeSubtask("crosstalk") 

1011 self.makeSubtask("strayLight") 

1012 self.makeSubtask("fringe") 

1013 self.makeSubtask("masking") 

1014 self.makeSubtask("overscan") 

1015 self.makeSubtask("vignette") 

1016 self.makeSubtask("ampOffset") 

1017 self.makeSubtask("deferredChargeCorrection") 

1018 self.makeSubtask("isrStats") 

1019 

1020 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1021 inputs = butlerQC.get(inputRefs) 

1022 

1023 try: 

1024 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

1025 except Exception as e: 

1026 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

1027 (inputRefs, e)) 

1028 

1029 detector = inputs['ccdExposure'].getDetector() 

1030 

1031 # This is use for header provenance. 

1032 additionalInputDates = {} 

1033 

1034 if self.config.doCrosstalk is True: 

1035 # Crosstalk sources need to be defined by the pipeline 

1036 # yaml if they exist. 

1037 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

1038 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

1039 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

1040 else: 

1041 coeffVector = (self.config.crosstalk.crosstalkValues 

1042 if self.config.crosstalk.useConfigCoefficients else None) 

1043 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1044 inputs['crosstalk'] = crosstalkCalib 

1045 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1046 if 'crosstalkSources' not in inputs: 

1047 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1048 

1049 if self.doLinearize(detector) is True: 

1050 if 'linearizer' in inputs: 

1051 if isinstance(inputs['linearizer'], dict): 

1052 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1053 linearizer.fromYaml(inputs['linearizer']) 

1054 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1055 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1056 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1057 detector=detector, 

1058 log=self.log) 

1059 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1060 else: 

1061 linearizer = inputs['linearizer'] 

1062 linearizer.log = self.log 

1063 inputs['linearizer'] = linearizer 

1064 else: 

1065 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1066 self.log.warning("Constructing linearizer from cameraGeom information.") 

1067 

1068 if self.config.doDefect is True: 

1069 if "defects" in inputs and inputs['defects'] is not None: 

1070 # defects is loaded as a BaseCatalog with columns 

1071 # x0, y0, width, height. Masking expects a list of defects 

1072 # defined by their bounding box 

1073 if not isinstance(inputs["defects"], Defects): 

1074 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1075 

1076 # Load the correct style of brighter-fatter kernel, and repack 

1077 # the information as a numpy array. 

1078 brighterFatterSource = None 

1079 if self.config.doBrighterFatter: 

1080 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1081 if brighterFatterKernel is None: 

1082 # This type of kernel must be in (y, x) index 

1083 # ordering, as it used directly as the .array 

1084 # component of the afwImage kernel. 

1085 brighterFatterKernel = inputs.get('bfKernel', None) 

1086 brighterFatterSource = 'bfKernel' 

1087 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1088 

1089 if brighterFatterKernel is None: 

1090 # This was requested by the config, but none were found. 

1091 raise RuntimeError("No brighter-fatter kernel was supplied.") 

1092 elif not isinstance(brighterFatterKernel, numpy.ndarray): 

1093 # This is a ISR calib kernel. These kernels are 

1094 # generated in (x, y) index ordering, and need to be 

1095 # transposed to be used directly as the .array 

1096 # component of the afwImage kernel. This is done 

1097 # explicitly below when setting the ``bfKernel`` 

1098 # input. 

1099 brighterFatterSource = 'newBFKernel' 

1100 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1101 

1102 detName = detector.getName() 

1103 level = brighterFatterKernel.level 

1104 

1105 # This is expected to be a dictionary of amp-wise gains. 

1106 inputs['bfGains'] = brighterFatterKernel.gain 

1107 if self.config.brighterFatterLevel == 'DETECTOR': 

1108 kernel = None 

1109 if level == 'DETECTOR': 

1110 if detName in brighterFatterKernel.detKernels: 

1111 kernel = brighterFatterKernel.detKernels[detName] 

1112 else: 

1113 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1114 elif level == 'AMP': 

1115 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1116 "fatter kernels.") 

1117 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1118 kernel = brighterFatterKernel.detKernels[detName] 

1119 if kernel is None: 

1120 raise RuntimeError("Could not identify brighter-fatter kernel!") 

1121 # Do the one single transpose here so the kernel 

1122 # can be directly loaded into the afwImage .array 

1123 # component. 

1124 inputs['bfKernel'] = numpy.transpose(kernel) 

1125 elif self.config.brighterFatterLevel == 'AMP': 

1126 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1127 

1128 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1129 expId = inputs['ccdExposure'].info.id 

1130 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1131 expId=expId, 

1132 assembler=self.assembleCcd 

1133 if self.config.doAssembleIsrExposures else None) 

1134 else: 

1135 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1136 

1137 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1138 if 'strayLightData' not in inputs: 

1139 inputs['strayLightData'] = None 

1140 

1141 if self.config.doHeaderProvenance: 

1142 # Add calibration provenanace info to header. 

1143 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1144 

1145 # These inputs change name during this step. These should 

1146 # have matching entries in the additionalInputDates dict. 

1147 additionalInputs = [] 

1148 if self.config.doBrighterFatter: 

1149 additionalInputs.append(brighterFatterSource) 

1150 

1151 for inputName in sorted(list(inputs.keys()) + additionalInputs): 

1152 reference = getattr(inputRefs, inputName, None) 

1153 if reference is not None and hasattr(reference, "run"): 

1154 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1155 runValue = reference.run 

1156 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1157 idValue = str(reference.id) 

1158 dateKey = f"LSST CALIB DATE {inputName.upper()}" 

1159 

1160 if inputName in additionalInputDates: 

1161 dateValue = additionalInputDates[inputName] 

1162 else: 

1163 dateValue = self.extractCalibDate(inputs[inputName]) 

1164 

1165 exposureMetadata[runKey] = runValue 

1166 exposureMetadata[idKey] = idValue 

1167 exposureMetadata[dateKey] = dateValue 

1168 

1169 outputs = self.run(**inputs) 

1170 butlerQC.put(outputs, outputRefs) 

1171 

1172 @timeMethod 

1173 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1174 crosstalk=None, crosstalkSources=None, 

1175 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1176 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1177 sensorTransmission=None, atmosphereTransmission=None, 

1178 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1179 deferredChargeCalib=None, 

1180 ): 

1181 """Perform instrument signature removal on an exposure. 

1182 

1183 Steps included in the ISR processing, in order performed, are: 

1184 

1185 - saturation and suspect pixel masking 

1186 - overscan subtraction 

1187 - CCD assembly of individual amplifiers 

1188 - bias subtraction 

1189 - variance image construction 

1190 - linearization of non-linear response 

1191 - crosstalk masking 

1192 - brighter-fatter correction 

1193 - dark subtraction 

1194 - fringe correction 

1195 - stray light subtraction 

1196 - flat correction 

1197 - masking of known defects and camera specific features 

1198 - vignette calculation 

1199 - appending transmission curve and distortion model 

1200 

1201 Parameters 

1202 ---------- 

1203 ccdExposure : `lsst.afw.image.Exposure` 

1204 The raw exposure that is to be run through ISR. The 

1205 exposure is modified by this method. 

1206 camera : `lsst.afw.cameraGeom.Camera`, optional 

1207 The camera geometry for this exposure. Required if 

1208 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1209 ``flat`` does not have an associated detector. 

1210 bias : `lsst.afw.image.Exposure`, optional 

1211 Bias calibration frame. 

1212 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1213 Functor for linearization. 

1214 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1215 Calibration for crosstalk. 

1216 crosstalkSources : `list`, optional 

1217 List of possible crosstalk sources. 

1218 dark : `lsst.afw.image.Exposure`, optional 

1219 Dark calibration frame. 

1220 flat : `lsst.afw.image.Exposure`, optional 

1221 Flat calibration frame. 

1222 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1223 Photon transfer curve dataset, with, e.g., gains 

1224 and read noise. 

1225 bfKernel : `numpy.ndarray`, optional 

1226 Brighter-fatter kernel. 

1227 bfGains : `dict` of `float`, optional 

1228 Gains used to override the detector's nominal gains for the 

1229 brighter-fatter correction. A dict keyed by amplifier name for 

1230 the detector in question. 

1231 defects : `lsst.ip.isr.Defects`, optional 

1232 List of defects. 

1233 fringes : `lsst.pipe.base.Struct`, optional 

1234 Struct containing the fringe correction data, with 

1235 elements: 

1236 

1237 ``fringes`` 

1238 fringe calibration frame (`lsst.afw.image.Exposure`) 

1239 ``seed`` 

1240 random seed derived from the ``ccdExposureId`` for random 

1241 number generator (`numpy.uint32`) 

1242 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1243 A ``TransmissionCurve`` that represents the throughput of the, 

1244 optics, to be evaluated in focal-plane coordinates. 

1245 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1246 A ``TransmissionCurve`` that represents the throughput of the 

1247 filter itself, to be evaluated in focal-plane coordinates. 

1248 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1249 A ``TransmissionCurve`` that represents the throughput of the 

1250 sensor itself, to be evaluated in post-assembly trimmed detector 

1251 coordinates. 

1252 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1253 A ``TransmissionCurve`` that represents the throughput of the 

1254 atmosphere, assumed to be spatially constant. 

1255 detectorNum : `int`, optional 

1256 The integer number for the detector to process. 

1257 strayLightData : `object`, optional 

1258 Opaque object containing calibration information for stray-light 

1259 correction. If `None`, no correction will be performed. 

1260 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1261 Illumination correction image. 

1262 

1263 Returns 

1264 ------- 

1265 result : `lsst.pipe.base.Struct` 

1266 Result struct with component: 

1267 

1268 ``exposure`` 

1269 The fully ISR corrected exposure. 

1270 (`lsst.afw.image.Exposure`) 

1271 ``outputExposure`` 

1272 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1273 ``ossThumb`` 

1274 Thumbnail image of the exposure after overscan subtraction. 

1275 (`numpy.ndarray`) 

1276 ``flattenedThumb`` 

1277 Thumbnail image of the exposure after flat-field correction. 

1278 (`numpy.ndarray`) 

1279 ``outputStatistics`` 

1280 Values of the additional statistics calculated. 

1281 

1282 Raises 

1283 ------ 

1284 RuntimeError 

1285 Raised if a configuration option is set to `True`, but the 

1286 required calibration data has not been specified. 

1287 

1288 Notes 

1289 ----- 

1290 The current processed exposure can be viewed by setting the 

1291 appropriate `lsstDebug` entries in the ``debug.display`` 

1292 dictionary. The names of these entries correspond to some of 

1293 the `IsrTaskConfig` Boolean options, with the value denoting the 

1294 frame to use. The exposure is shown inside the matching 

1295 option check and after the processing of that step has 

1296 finished. The steps with debug points are: 

1297 

1298 * doAssembleCcd 

1299 * doBias 

1300 * doCrosstalk 

1301 * doBrighterFatter 

1302 * doDark 

1303 * doFringe 

1304 * doStrayLight 

1305 * doFlat 

1306 

1307 In addition, setting the ``postISRCCD`` entry displays the 

1308 exposure after all ISR processing has finished. 

1309 """ 

1310 

1311 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1312 bias = self.ensureExposure(bias, camera, detectorNum) 

1313 dark = self.ensureExposure(dark, camera, detectorNum) 

1314 flat = self.ensureExposure(flat, camera, detectorNum) 

1315 

1316 ccd = ccdExposure.getDetector() 

1317 filterLabel = ccdExposure.getFilter() 

1318 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1319 

1320 if not ccd: 

1321 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1322 ccd = [FakeAmp(ccdExposure, self.config)] 

1323 

1324 # Validate Input 

1325 if self.config.doBias and bias is None: 

1326 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1327 if self.doLinearize(ccd) and linearizer is None: 

1328 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1329 if self.config.doBrighterFatter and bfKernel is None: 

1330 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1331 if self.config.doDark and dark is None: 

1332 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1333 if self.config.doFlat and flat is None: 

1334 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1335 if self.config.doDefect and defects is None: 

1336 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1337 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1338 and fringes.fringes is None): 

1339 # The `fringes` object needs to be a pipeBase.Struct, as 

1340 # we use it as a `dict` for the parameters of 

1341 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1342 # not be `None` if `doFringe=True`. Otherwise, raise. 

1343 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1344 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1345 and illumMaskedImage is None): 

1346 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1347 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1348 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1349 if (self.config.usePtcGains and ptc is None): 

1350 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

1351 if (self.config.usePtcReadNoise and ptc is None): 

1352 raise RuntimeError("No ptcDataset provided to use PTC read noise.") 

1353 

1354 # Validate that the inputs match the exposure configuration. 

1355 exposureMetadata = ccdExposure.getMetadata() 

1356 if self.config.doBias: 

1357 self.compareCameraKeywords(exposureMetadata, bias, "bias") 

1358 if self.config.doBrighterFatter: 

1359 self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter") 

1360 if self.config.doCrosstalk: 

1361 self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk") 

1362 if self.config.doDark: 

1363 self.compareCameraKeywords(exposureMetadata, dark, "dark") 

1364 if self.config.doDefect: 

1365 self.compareCameraKeywords(exposureMetadata, defects, "defects") 

1366 if self.config.doDeferredCharge: 

1367 self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI") 

1368 if self.config.doFlat: 

1369 self.compareCameraKeywords(exposureMetadata, flat, "flat") 

1370 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1371 self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe") 

1372 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1373 self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination") 

1374 if self.doLinearize(ccd): 

1375 self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer") 

1376 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1377 self.compareCameraKeywords(exposureMetadata, ptc, "PTC") 

1378 if self.config.doStrayLight: 

1379 self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight") 

1380 

1381 # Start in ADU. Update units to electrons when gain is applied: 

1382 # updateVariance, applyGains 

1383 # Check if needed during/after BFE correction, CTI correction. 

1384 exposureMetadata["LSST ISR UNITS"] = "ADU" 

1385 

1386 # Begin ISR processing. 

1387 if self.config.doConvertIntToFloat: 

1388 self.log.info("Converting exposure to floating point values.") 

1389 ccdExposure = self.convertIntToFloat(ccdExposure) 

1390 

1391 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1392 self.log.info("Applying bias correction.") 

1393 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1394 trimToFit=self.config.doTrimToMatchCalib) 

1395 self.debugView(ccdExposure, "doBias") 

1396 

1397 # Amplifier level processing. 

1398 overscans = [] 

1399 

1400 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1401 # This will attempt to mask bleed pixels across all amplifiers. 

1402 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1403 

1404 for amp in ccd: 

1405 # if ccdExposure is one amp, 

1406 # check for coverage to prevent performing ops multiple times 

1407 if ccdExposure.getBBox().contains(amp.getBBox()): 

1408 # Check for fully masked bad amplifiers, 

1409 # and generate masks for SUSPECT and SATURATED values. 

1410 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1411 

1412 if self.config.doOverscan and not badAmp: 

1413 # Overscan correction on amp-by-amp basis. 

1414 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1415 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1416 if overscanResults is not None and \ 

1417 self.config.qa is not None and self.config.qa.saveStats is True: 

1418 if isinstance(overscanResults.overscanMean, float): 

1419 # Only serial overscan was run 

1420 mean = overscanResults.overscanMean 

1421 sigma = overscanResults.overscanSigma 

1422 residMean = overscanResults.residualMean 

1423 residSigma = overscanResults.residualSigma 

1424 else: 

1425 # Both serial and parallel overscan were 

1426 # run. Only report serial here. 

1427 mean = overscanResults.overscanMean[0] 

1428 sigma = overscanResults.overscanSigma[0] 

1429 residMean = overscanResults.residualMean[0] 

1430 residSigma = overscanResults.residualSigma[0] 

1431 

1432 self.metadata[f"FIT MEDIAN {amp.getName()}"] = mean 

1433 self.metadata[f"FIT STDEV {amp.getName()}"] = sigma 

1434 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1435 amp.getName(), mean, sigma) 

1436 

1437 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = residMean 

1438 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = residSigma 

1439 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1440 amp.getName(), residMean, residSigma) 

1441 

1442 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1443 else: 

1444 if badAmp: 

1445 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1446 overscanResults = None 

1447 

1448 overscans.append(overscanResults if overscanResults is not None else None) 

1449 else: 

1450 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1451 

1452 # Define an effective PTC that will contain the gain and readout 

1453 # noise to be used throughout the ISR task. 

1454 ptc = self.defineEffectivePtc(ptc, ccd, bfGains, overscans, exposureMetadata) 

1455 

1456 if self.config.doDeferredCharge: 

1457 self.log.info("Applying deferred charge/CTI correction.") 

1458 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1459 self.debugView(ccdExposure, "doDeferredCharge") 

1460 

1461 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1462 self.log.info("Applying crosstalk correction.") 

1463 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1464 crosstalkSources=crosstalkSources, camera=camera) 

1465 self.debugView(ccdExposure, "doCrosstalk") 

1466 

1467 if self.config.doAssembleCcd: 

1468 self.log.info("Assembling CCD from amplifiers.") 

1469 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1470 

1471 if self.config.expectWcs and not ccdExposure.getWcs(): 

1472 self.log.warning("No WCS found in input exposure.") 

1473 self.debugView(ccdExposure, "doAssembleCcd") 

1474 

1475 ossThumb = None 

1476 if self.config.qa.doThumbnailOss: 

1477 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1478 

1479 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1480 self.log.info("Applying bias correction.") 

1481 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1482 trimToFit=self.config.doTrimToMatchCalib) 

1483 self.debugView(ccdExposure, "doBias") 

1484 

1485 if self.config.doVariance: 

1486 for amp, overscanResults in zip(ccd, overscans): 

1487 if ccdExposure.getBBox().contains(amp.getBBox()): 

1488 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1489 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1490 self.updateVariance(ampExposure, amp, ptc) 

1491 

1492 if self.config.qa is not None and self.config.qa.saveStats is True: 

1493 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1494 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1495 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1496 qaStats.getValue(afwMath.MEDIAN) 

1497 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1498 qaStats.getValue(afwMath.STDEVCLIP) 

1499 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1500 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1501 qaStats.getValue(afwMath.STDEVCLIP)) 

1502 if self.config.maskNegativeVariance: 

1503 self.maskNegativeVariance(ccdExposure) 

1504 

1505 if self.doLinearize(ccd): 

1506 self.log.info("Applying linearizer.") 

1507 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1508 detector=ccd, log=self.log) 

1509 

1510 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1511 self.log.info("Applying crosstalk correction.") 

1512 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1513 crosstalkSources=crosstalkSources, isTrimmed=True) 

1514 self.debugView(ccdExposure, "doCrosstalk") 

1515 

1516 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1517 # widen trails, and do anything else the camera needs. Saturated and 

1518 # suspect pixels have already been masked. 

1519 if self.config.doDefect: 

1520 self.log.info("Masking defects.") 

1521 self.maskDefect(ccdExposure, defects) 

1522 

1523 if self.config.numEdgeSuspect > 0: 

1524 self.log.info("Masking edges as SUSPECT.") 

1525 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1526 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1527 

1528 if self.config.doNanMasking: 

1529 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1530 self.maskNan(ccdExposure) 

1531 

1532 if self.config.doWidenSaturationTrails: 

1533 self.log.info("Widening saturation trails.") 

1534 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1535 

1536 if self.config.doCameraSpecificMasking: 

1537 self.log.info("Masking regions for camera specific reasons.") 

1538 self.masking.run(ccdExposure) 

1539 

1540 if self.config.doBrighterFatter: 

1541 # We need to apply flats and darks before we can interpolate, and 

1542 # we need to interpolate before we do B-F, but we do B-F without 

1543 # the flats and darks applied so we can work in units of electrons 

1544 # or holes. This context manager applies and then removes the darks 

1545 # and flats. 

1546 # 

1547 # We also do not want to interpolate values here, so operate on 

1548 # temporary images so we can apply only the BF-correction and roll 

1549 # back the interpolation. 

1550 interpExp = ccdExposure.clone() 

1551 with self.flatContext(interpExp, flat, dark): 

1552 isrFunctions.interpolateFromMask( 

1553 maskedImage=interpExp.getMaskedImage(), 

1554 fwhm=self.config.fwhm, 

1555 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1556 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1557 ) 

1558 bfExp = interpExp.clone() 

1559 

1560 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1561 type(bfKernel), type(bfGains)) 

1562 if self.config.doFluxConservingBrighterFatterCorrection: 

1563 bfResults = isrFunctions.fluxConservingBrighterFatterCorrection( 

1564 bfExp, 

1565 bfKernel, 

1566 self.config.brighterFatterMaxIter, 

1567 self.config.brighterFatterThreshold, 

1568 self.config.brighterFatterApplyGain, 

1569 bfGains 

1570 ) 

1571 else: 

1572 bfResults = isrFunctions.brighterFatterCorrection( 

1573 bfExp, 

1574 bfKernel, 

1575 self.config.brighterFatterMaxIter, 

1576 self.config.brighterFatterThreshold, 

1577 self.config.brighterFatterApplyGain, 

1578 bfGains 

1579 ) 

1580 if bfResults[1] == self.config.brighterFatterMaxIter - 1: 

1581 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1582 bfResults[0]) 

1583 else: 

1584 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1585 bfResults[1]) 

1586 image = ccdExposure.getMaskedImage().getImage() 

1587 bfCorr = bfExp.getMaskedImage().getImage() 

1588 bfCorr -= interpExp.getMaskedImage().getImage() 

1589 image += bfCorr 

1590 

1591 # Applying the brighter-fatter correction applies a 

1592 # convolution to the science image. At the edges this 

1593 # convolution may not have sufficient valid pixels to 

1594 # produce a valid correction. Mark pixels within the size 

1595 # of the brighter-fatter kernel as EDGE to warn of this 

1596 # fact. 

1597 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1598 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1599 maskPlane="EDGE") 

1600 

1601 if self.config.brighterFatterMaskGrowSize > 0: 

1602 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1603 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1604 isrFunctions.growMasks(ccdExposure.getMask(), 

1605 radius=self.config.brighterFatterMaskGrowSize, 

1606 maskNameList=maskPlane, 

1607 maskValue=maskPlane) 

1608 

1609 self.debugView(ccdExposure, "doBrighterFatter") 

1610 

1611 if self.config.doDark: 

1612 self.log.info("Applying dark correction.") 

1613 self.darkCorrection(ccdExposure, dark) 

1614 self.debugView(ccdExposure, "doDark") 

1615 

1616 if self.config.doFringe and not self.config.fringeAfterFlat: 

1617 self.log.info("Applying fringe correction before flat.") 

1618 self.fringe.run(ccdExposure, **fringes.getDict()) 

1619 self.debugView(ccdExposure, "doFringe") 

1620 

1621 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1622 self.log.info("Checking strayLight correction.") 

1623 self.strayLight.run(ccdExposure, strayLightData) 

1624 self.debugView(ccdExposure, "doStrayLight") 

1625 

1626 if self.config.doFlat: 

1627 self.log.info("Applying flat correction.") 

1628 self.flatCorrection(ccdExposure, flat) 

1629 self.debugView(ccdExposure, "doFlat") 

1630 

1631 if self.config.doApplyGains: 

1632 self.log.info("Applying gain correction instead of flat.") 

1633 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1634 ptcGains=ptc.gain) 

1635 exposureMetadata["LSST ISR UNITS"] = "electrons" 

1636 

1637 if self.config.doFringe and self.config.fringeAfterFlat: 

1638 self.log.info("Applying fringe correction after flat.") 

1639 self.fringe.run(ccdExposure, **fringes.getDict()) 

1640 

1641 if self.config.doVignette: 

1642 if self.config.doMaskVignettePolygon: 

1643 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1644 else: 

1645 self.log.info("Constructing and attaching vignette polygon.") 

1646 self.vignettePolygon = self.vignette.run( 

1647 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1648 vignetteValue=self.config.vignetteValue, log=self.log) 

1649 

1650 if self.config.doAttachTransmissionCurve: 

1651 self.log.info("Adding transmission curves.") 

1652 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1653 filterTransmission=filterTransmission, 

1654 sensorTransmission=sensorTransmission, 

1655 atmosphereTransmission=atmosphereTransmission) 

1656 

1657 flattenedThumb = None 

1658 if self.config.qa.doThumbnailFlattened: 

1659 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1660 

1661 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1662 self.log.info("Performing illumination correction.") 

1663 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1664 illumMaskedImage, illumScale=self.config.illumScale, 

1665 trimToFit=self.config.doTrimToMatchCalib) 

1666 

1667 preInterpExp = None 

1668 if self.config.doSaveInterpPixels: 

1669 preInterpExp = ccdExposure.clone() 

1670 

1671 # Reset and interpolate bad pixels. 

1672 # 

1673 # Large contiguous bad regions (which should have the BAD mask 

1674 # bit set) should have their values set to the image median. 

1675 # This group should include defects and bad amplifiers. As the 

1676 # area covered by these defects are large, there's little 

1677 # reason to expect that interpolation would provide a more 

1678 # useful value. 

1679 # 

1680 # Smaller defects can be safely interpolated after the larger 

1681 # regions have had their pixel values reset. This ensures 

1682 # that the remaining defects adjacent to bad amplifiers (as an 

1683 # example) do not attempt to interpolate extreme values. 

1684 if self.config.doSetBadRegions: 

1685 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1686 if badPixelCount > 0: 

1687 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1688 

1689 if self.config.doInterpolate: 

1690 self.log.info("Interpolating masked pixels.") 

1691 isrFunctions.interpolateFromMask( 

1692 maskedImage=ccdExposure.getMaskedImage(), 

1693 fwhm=self.config.fwhm, 

1694 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1695 maskNameList=list(self.config.maskListToInterpolate) 

1696 ) 

1697 

1698 self.roughZeroPoint(ccdExposure) 

1699 

1700 # correct for amp offsets within the CCD 

1701 if self.config.doAmpOffset: 

1702 self.log.info("Correcting amp offsets.") 

1703 self.ampOffset.run(ccdExposure) 

1704 

1705 if self.config.doMeasureBackground: 

1706 self.log.info("Measuring background level.") 

1707 self.measureBackground(ccdExposure, self.config.qa) 

1708 

1709 if self.config.qa is not None and self.config.qa.saveStats is True: 

1710 for amp in ccd: 

1711 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1712 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1713 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1714 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1715 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1716 qaStats.getValue(afwMath.STDEVCLIP) 

1717 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1718 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1719 qaStats.getValue(afwMath.STDEVCLIP)) 

1720 

1721 # Calculate standard image quality statistics 

1722 if self.config.doStandardStatistics: 

1723 metadata = ccdExposure.getMetadata() 

1724 for amp in ccd: 

1725 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1726 ampName = amp.getName() 

1727 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1728 ampExposure.getMaskedImage(), 

1729 [self.config.saturatedMaskName] 

1730 ) 

1731 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1732 ampExposure.getMaskedImage(), 

1733 ["BAD"] 

1734 ) 

1735 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1736 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1737 

1738 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1739 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1740 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1741 

1742 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1743 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1744 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1745 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1746 else: 

1747 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1748 

1749 # calculate additional statistics. 

1750 outputStatistics = None 

1751 if self.config.doCalculateStatistics: 

1752 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1753 ptc=ptc).results 

1754 

1755 # do any binning. 

1756 outputBin1Exposure = None 

1757 outputBin2Exposure = None 

1758 if self.config.doBinnedExposures: 

1759 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure) 

1760 

1761 self.debugView(ccdExposure, "postISRCCD") 

1762 

1763 return pipeBase.Struct( 

1764 exposure=ccdExposure, 

1765 ossThumb=ossThumb, 

1766 flattenedThumb=flattenedThumb, 

1767 

1768 outputBin1Exposure=outputBin1Exposure, 

1769 outputBin2Exposure=outputBin2Exposure, 

1770 

1771 preInterpExposure=preInterpExp, 

1772 outputExposure=ccdExposure, 

1773 outputOssThumbnail=ossThumb, 

1774 outputFlattenedThumbnail=flattenedThumb, 

1775 outputStatistics=outputStatistics, 

1776 ) 

1777 

1778 def defineEffectivePtc(self, ptcDataset, detector, bfGains, overScans, metadata): 

1779 """Define an effective Photon Transfer Curve dataset 

1780 with nominal gains and noise. 

1781 

1782 Parameters 

1783 ------ 

1784 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset` 

1785 Input Photon Transfer Curve dataset. 

1786 detector : `lsst.afw.cameraGeom.Detector` 

1787 Detector object. 

1788 bfGains : `dict` 

1789 Gains from running the brighter-fatter code. 

1790 A dict keyed by amplifier name for the detector 

1791 in question. 

1792 ovserScans : `list` [`lsst.pipe.base.Struct`] 

1793 List of overscanResults structures 

1794 metadata : `lsst.daf.base.PropertyList` 

1795 Exposure metadata to update gain and noise provenance. 

1796 

1797 Returns 

1798 ------- 

1799 effectivePtc : `lsst.ip.isr.PhotonTransferCurveDataset` 

1800 PTC dataset containing gains and readout noise 

1801 values to be used throughout 

1802 Instrument Signature Removal. 

1803 """ 

1804 amps = detector.getAmplifiers() 

1805 ampNames = [amp.getName() for amp in amps] 

1806 detName = detector.getName() 

1807 effectivePtc = PhotonTransferCurveDataset(ampNames, 'EFFECTIVE_PTC', 1) 

1808 boolGainMismatch = False 

1809 doWarningPtcValidation = True 

1810 

1811 for amp, overscanResults in zip(amps, overScans): 

1812 ampName = amp.getName() 

1813 # Gain: 

1814 # Try first with the PTC gains. 

1815 gainProvenanceString = "amp" 

1816 if self.config.usePtcGains: 

1817 gain = ptcDataset.gain[ampName] 

1818 gainProvenanceString = "ptc" 

1819 self.log.debug("Using gain from Photon Transfer Curve.") 

1820 else: 

1821 # Try then with the amplifier gain. 

1822 # We already have a detector at this point. If there was no 

1823 # detector to begin with, one would have been created with 

1824 # self.config.gain and self.config.noise. Same comment 

1825 # applies for the noise block below. 

1826 gain = amp.getGain() 

1827 

1828 # Check if the gain up to this point differs from the 

1829 # gain in bfGains. If so, raise or warn, accordingly. 

1830 if not boolGainMismatch and bfGains is not None and ampName in bfGains: 

1831 bfGain = bfGains[ampName] 

1832 if not math.isclose(gain, bfGain, rel_tol=1e-4): 

1833 if self.config.doRaiseOnCalibMismatch: 

1834 raise RuntimeError("Gain mismatch for det %s amp %s: " 

1835 "(gain (%s): %s, bfGain: %s)", 

1836 detName, ampName, gainProvenanceString, 

1837 gain, bfGain) 

1838 else: 

1839 self.log.warning("Gain mismatch for det %s amp %s: " 

1840 "(gain (%s): %s, bfGain: %s)", 

1841 detName, ampName, gainProvenanceString, 

1842 gain, bfGain) 

1843 boolGainMismatch = True 

1844 

1845 # Noise: 

1846 # Try first with the empirical noise from the overscan. 

1847 noiseProvenanceString = "amp" 

1848 if self.config.doEmpiricalReadNoise and overscanResults is not None: 

1849 noiseProvenanceString = "serial overscan" 

1850 if isinstance(overscanResults.residualSigma, float): 

1851 # Only serial overscan was run 

1852 noise = overscanResults.residualSigma 

1853 else: 

1854 # Both serial and parallel overscan were 

1855 # run. Only report noise from serial here. 

1856 noise = overscanResults.residualSigma[0] 

1857 elif self.config.usePtcReadNoise: 

1858 # Try then with the PTC noise. 

1859 noise = ptcDataset.noise[amp.getName()] 

1860 noiseProvenanceString = "ptc" 

1861 self.log.debug("Using noise from Photon Transfer Curve.") 

1862 else: 

1863 # Finally, try with the amplifier noise. 

1864 # We already have a detector at this point. If there 

1865 # was no detector to begin with, one would have 

1866 # been created with self.config.gain and 

1867 # self.config.noise. 

1868 noise = amp.getReadNoise() 

1869 

1870 if math.isnan(gain): 

1871 gain = 1.0 

1872 self.log.warning("Gain for amp %s set to NAN! Updating to" 

1873 " 1.0 to generate Poisson variance.", ampName) 

1874 elif gain <= 0: 

1875 patchedGain = 1.0 

1876 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

1877 ampName, gain, patchedGain) 

1878 gain = patchedGain 

1879 

1880 effectivePtc.gain[ampName] = gain 

1881 effectivePtc.noise[ampName] = noise 

1882 # Make sure noise,turnoff, and gain make sense 

1883 effectivePtc.validateGainNoiseTurnoffValues(ampName, doWarn=doWarningPtcValidation) 

1884 doWarningPtcValidation = False 

1885 

1886 metadata[f"LSST GAIN {amp.getName()}"] = effectivePtc.gain[ampName] 

1887 metadata[f"LSST READNOISE {amp.getName()}"] = effectivePtc.noise[ampName] 

1888 

1889 self.log.info("Det: %s - Noise provenance: %s, Gain provenance: %s", 

1890 detName, 

1891 noiseProvenanceString, 

1892 gainProvenanceString) 

1893 metadata["LSST ISR GAIN SOURCE"] = gainProvenanceString 

1894 metadata["LSST ISR NOISE SOURCE"] = noiseProvenanceString 

1895 

1896 return effectivePtc 

1897 

1898 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1899 """Ensure that the data returned by Butler is a fully constructed exp. 

1900 

1901 ISR requires exposure-level image data for historical reasons, so if we 

1902 did not recieve that from Butler, construct it from what we have, 

1903 modifying the input in place. 

1904 

1905 Parameters 

1906 ---------- 

1907 inputExp : `lsst.afw.image` image-type. 

1908 The input data structure obtained from Butler. 

1909 Can be `lsst.afw.image.Exposure`, 

1910 `lsst.afw.image.DecoratedImageU`, 

1911 or `lsst.afw.image.ImageF` 

1912 camera : `lsst.afw.cameraGeom.camera`, optional 

1913 The camera associated with the image. Used to find the appropriate 

1914 detector if detector is not already set. 

1915 detectorNum : `int`, optional 

1916 The detector in the camera to attach, if the detector is not 

1917 already set. 

1918 

1919 Returns 

1920 ------- 

1921 inputExp : `lsst.afw.image.Exposure` 

1922 The re-constructed exposure, with appropriate detector parameters. 

1923 

1924 Raises 

1925 ------ 

1926 TypeError 

1927 Raised if the input data cannot be used to construct an exposure. 

1928 """ 

1929 if isinstance(inputExp, afwImage.DecoratedImageU): 

1930 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1931 elif isinstance(inputExp, afwImage.ImageF): 

1932 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1933 elif isinstance(inputExp, afwImage.MaskedImageF): 

1934 inputExp = afwImage.makeExposure(inputExp) 

1935 elif isinstance(inputExp, afwImage.Exposure): 

1936 pass 

1937 elif inputExp is None: 

1938 # Assume this will be caught by the setup if it is a problem. 

1939 return inputExp 

1940 else: 

1941 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1942 (type(inputExp), )) 

1943 

1944 if inputExp.getDetector() is None: 

1945 if camera is None or detectorNum is None: 

1946 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1947 'without a detector set.') 

1948 inputExp.setDetector(camera[detectorNum]) 

1949 

1950 return inputExp 

1951 

1952 @staticmethod 

1953 def extractCalibDate(calib): 

1954 """Extract common calibration metadata values that will be written to 

1955 output header. 

1956 

1957 Parameters 

1958 ---------- 

1959 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1960 Calibration to pull date information from. 

1961 

1962 Returns 

1963 ------- 

1964 dateString : `str` 

1965 Calibration creation date string to add to header. 

1966 """ 

1967 if hasattr(calib, "getMetadata"): 

1968 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1969 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1970 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1971 else: 

1972 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1973 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1974 else: 

1975 return "Unknown Unknown" 

1976 

1977 def compareCameraKeywords(self, exposureMetadata, calib, calibName): 

1978 """Compare header keywords to confirm camera states match. 

1979 

1980 Parameters 

1981 ---------- 

1982 exposureMetadata : `lsst.daf.base.PropertySet` 

1983 Header for the exposure being processed. 

1984 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1985 Calibration to be applied. 

1986 calibName : `str` 

1987 Calib type for log message. 

1988 """ 

1989 try: 

1990 calibMetadata = calib.getMetadata() 

1991 except AttributeError: 

1992 return 

1993 for keyword in self.config.cameraKeywordsToCompare: 

1994 if keyword in exposureMetadata and keyword in calibMetadata: 

1995 if exposureMetadata[keyword] != calibMetadata[keyword]: 

1996 if self.config.doRaiseOnCalibMismatch: 

1997 raise RuntimeError("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1998 calibName, keyword, 

1999 exposureMetadata[keyword], calibMetadata[keyword]) 

2000 else: 

2001 self.log.warning("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

2002 calibName, keyword, 

2003 exposureMetadata[keyword], calibMetadata[keyword]) 

2004 else: 

2005 self.log.debug("Sequencer keyword %s not found.", keyword) 

2006 

2007 def convertIntToFloat(self, exposure): 

2008 """Convert exposure image from uint16 to float. 

2009 

2010 If the exposure does not need to be converted, the input is 

2011 immediately returned. For exposures that are converted to use 

2012 floating point pixels, the variance is set to unity and the 

2013 mask to zero. 

2014 

2015 Parameters 

2016 ---------- 

2017 exposure : `lsst.afw.image.Exposure` 

2018 The raw exposure to be converted. 

2019 

2020 Returns 

2021 ------- 

2022 newexposure : `lsst.afw.image.Exposure` 

2023 The input ``exposure``, converted to floating point pixels. 

2024 

2025 Raises 

2026 ------ 

2027 RuntimeError 

2028 Raised if the exposure type cannot be converted to float. 

2029 

2030 """ 

2031 if isinstance(exposure, afwImage.ExposureF): 

2032 # Nothing to be done 

2033 self.log.debug("Exposure already of type float.") 

2034 return exposure 

2035 if not hasattr(exposure, "convertF"): 

2036 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

2037 

2038 newexposure = exposure.convertF() 

2039 newexposure.variance[:] = 1 

2040 newexposure.mask[:] = 0x0 

2041 

2042 return newexposure 

2043 

2044 def maskAmplifier(self, ccdExposure, amp, defects): 

2045 """Identify bad amplifiers, saturated and suspect pixels. 

2046 

2047 Parameters 

2048 ---------- 

2049 ccdExposure : `lsst.afw.image.Exposure` 

2050 Input exposure to be masked. 

2051 amp : `lsst.afw.cameraGeom.Amplifier` 

2052 Catalog of parameters defining the amplifier on this 

2053 exposure to mask. 

2054 defects : `lsst.ip.isr.Defects` 

2055 List of defects. Used to determine if the entire 

2056 amplifier is bad. 

2057 

2058 Returns 

2059 ------- 

2060 badAmp : `Bool` 

2061 If this is true, the entire amplifier area is covered by 

2062 defects and unusable. 

2063 

2064 """ 

2065 maskedImage = ccdExposure.getMaskedImage() 

2066 

2067 badAmp = False 

2068 

2069 # Check if entire amp region is defined as a defect 

2070 # NB: need to use amp.getBBox() for correct comparison with current 

2071 # defects definition. 

2072 if defects is not None: 

2073 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

2074 

2075 # In the case of a bad amp, we will set mask to "BAD" 

2076 # (here use amp.getRawBBox() for correct association with pixels in 

2077 # current ccdExposure). 

2078 if badAmp: 

2079 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

2080 afwImage.PARENT) 

2081 maskView = dataView.getMask() 

2082 maskView |= maskView.getPlaneBitMask("BAD") 

2083 del maskView 

2084 return badAmp 

2085 

2086 # Mask remaining defects after assembleCcd() to allow for defects that 

2087 # cross amplifier boundaries. Saturation and suspect pixels can be 

2088 # masked now, though. 

2089 limits = dict() 

2090 if self.config.doSaturation and not badAmp: 

2091 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

2092 if self.config.doSuspect and not badAmp: 

2093 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

2094 if math.isfinite(self.config.saturation): 

2095 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

2096 

2097 for maskName, maskThreshold in limits.items(): 

2098 if not math.isnan(maskThreshold): 

2099 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2100 isrFunctions.makeThresholdMask( 

2101 maskedImage=dataView, 

2102 threshold=maskThreshold, 

2103 growFootprints=0, 

2104 maskName=maskName 

2105 ) 

2106 

2107 # Determine if we've fully masked this amplifier with SUSPECT and 

2108 # SAT pixels. 

2109 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

2110 afwImage.PARENT) 

2111 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

2112 self.config.suspectMaskName]) 

2113 if numpy.all(maskView.getArray() & maskVal > 0): 

2114 badAmp = True 

2115 maskView |= maskView.getPlaneBitMask("BAD") 

2116 

2117 return badAmp 

2118 

2119 def overscanCorrection(self, ccdExposure, amp): 

2120 """Apply overscan correction in place. 

2121 

2122 This method does initial pixel rejection of the overscan 

2123 region. The overscan can also be optionally segmented to 

2124 allow for discontinuous overscan responses to be fit 

2125 separately. The actual overscan subtraction is performed by 

2126 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

2127 after the amplifier is preprocessed. 

2128 

2129 Parameters 

2130 ---------- 

2131 ccdExposure : `lsst.afw.image.Exposure` 

2132 Exposure to have overscan correction performed. 

2133 amp : `lsst.afw.cameraGeom.Amplifer` 

2134 The amplifier to consider while correcting the overscan. 

2135 

2136 Returns 

2137 ------- 

2138 overscanResults : `lsst.pipe.base.Struct` 

2139 Result struct with components: 

2140 

2141 ``imageFit`` 

2142 Value or fit subtracted from the amplifier image data. 

2143 (scalar or `lsst.afw.image.Image`) 

2144 ``overscanFit`` 

2145 Value or fit subtracted from the overscan image data. 

2146 (scalar or `lsst.afw.image.Image`) 

2147 ``overscanImage`` 

2148 Image of the overscan region with the overscan 

2149 correction applied. This quantity is used to estimate 

2150 the amplifier read noise empirically. 

2151 (`lsst.afw.image.Image`) 

2152 ``edgeMask`` 

2153 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

2154 ``overscanMean`` 

2155 Median overscan fit value. (`float`) 

2156 ``overscanSigma`` 

2157 Clipped standard deviation of the overscan after 

2158 correction. (`float`) 

2159 

2160 Raises 

2161 ------ 

2162 RuntimeError 

2163 Raised if the ``amp`` does not contain raw pixel information. 

2164 

2165 See Also 

2166 -------- 

2167 lsst.ip.isr.overscan.OverscanTask 

2168 """ 

2169 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2170 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2171 return None 

2172 

2173 # Perform overscan correction on subregions. 

2174 overscanResults = self.overscan.run(ccdExposure, amp) 

2175 

2176 metadata = ccdExposure.getMetadata() 

2177 ampName = amp.getName() 

2178 

2179 keyBase = "LSST ISR OVERSCAN" 

2180 # Updated quantities 

2181 if isinstance(overscanResults.overscanMean, float): 

2182 # Serial overscan correction only: 

2183 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

2184 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

2185 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

2186 

2187 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

2188 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

2189 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

2190 elif isinstance(overscanResults.overscanMean, tuple): 

2191 # Both serial and parallel overscan have run: 

2192 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

2193 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

2194 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

2195 

2196 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

2197 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

2198 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

2199 

2200 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

2201 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

2202 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

2203 

2204 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

2205 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

2206 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

2207 else: 

2208 self.log.warning("Unexpected type for overscan values; none added to header.") 

2209 

2210 return overscanResults 

2211 

2212 def updateVariance(self, ampExposure, amp, ptcDataset): 

2213 """Set the variance plane using the gain and read noise 

2214 

2215 The read noise is calculated from the ``overscanImage`` if the 

2216 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2217 the value from the amplifier data is used. 

2218 

2219 Parameters 

2220 ---------- 

2221 ampExposure : `lsst.afw.image.Exposure` 

2222 Exposure to process. 

2223 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

2224 Amplifier detector data. 

2225 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset` 

2226 Effective PTC dataset containing the gains and read noise. 

2227 

2228 See also 

2229 -------- 

2230 lsst.ip.isr.isrFunctions.updateVariance 

2231 """ 

2232 ampName = amp.getName() 

2233 # At this point, the effective PTC should have 

2234 # gain and noise values. 

2235 gain = ptcDataset.gain[ampName] 

2236 readNoise = ptcDataset.noise[ampName] 

2237 

2238 isrFunctions.updateVariance( 

2239 maskedImage=ampExposure.getMaskedImage(), 

2240 gain=gain, 

2241 readNoise=readNoise, 

2242 ) 

2243 

2244 def maskNegativeVariance(self, exposure): 

2245 """Identify and mask pixels with negative variance values. 

2246 

2247 Parameters 

2248 ---------- 

2249 exposure : `lsst.afw.image.Exposure` 

2250 Exposure to process. 

2251 

2252 See Also 

2253 -------- 

2254 lsst.ip.isr.isrFunctions.updateVariance 

2255 """ 

2256 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2257 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2258 exposure.mask.array[bad] |= maskPlane 

2259 

2260 def darkCorrection(self, exposure, darkExposure, invert=False): 

2261 """Apply dark correction in place. 

2262 

2263 Parameters 

2264 ---------- 

2265 exposure : `lsst.afw.image.Exposure` 

2266 Exposure to process. 

2267 darkExposure : `lsst.afw.image.Exposure` 

2268 Dark exposure of the same size as ``exposure``. 

2269 invert : `Bool`, optional 

2270 If True, re-add the dark to an already corrected image. 

2271 

2272 Raises 

2273 ------ 

2274 RuntimeError 

2275 Raised if either ``exposure`` or ``darkExposure`` do not 

2276 have their dark time defined. 

2277 

2278 See Also 

2279 -------- 

2280 lsst.ip.isr.isrFunctions.darkCorrection 

2281 """ 

2282 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2283 if math.isnan(expScale): 

2284 raise RuntimeError("Exposure darktime is NAN.") 

2285 if darkExposure.getInfo().getVisitInfo() is not None \ 

2286 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2287 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2288 else: 

2289 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2290 # so getDarkTime() does not exist. 

2291 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2292 darkScale = 1.0 

2293 

2294 isrFunctions.darkCorrection( 

2295 maskedImage=exposure.getMaskedImage(), 

2296 darkMaskedImage=darkExposure.getMaskedImage(), 

2297 expScale=expScale, 

2298 darkScale=darkScale, 

2299 invert=invert, 

2300 trimToFit=self.config.doTrimToMatchCalib 

2301 ) 

2302 

2303 def doLinearize(self, detector): 

2304 """Check if linearization is needed for the detector cameraGeom. 

2305 

2306 Checks config.doLinearize and the linearity type of the first 

2307 amplifier. 

2308 

2309 Parameters 

2310 ---------- 

2311 detector : `lsst.afw.cameraGeom.Detector` 

2312 Detector to get linearity type from. 

2313 

2314 Returns 

2315 ------- 

2316 doLinearize : `Bool` 

2317 If True, linearization should be performed. 

2318 """ 

2319 return self.config.doLinearize and \ 

2320 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2321 

2322 def flatCorrection(self, exposure, flatExposure, invert=False): 

2323 """Apply flat correction in place. 

2324 

2325 Parameters 

2326 ---------- 

2327 exposure : `lsst.afw.image.Exposure` 

2328 Exposure to process. 

2329 flatExposure : `lsst.afw.image.Exposure` 

2330 Flat exposure of the same size as ``exposure``. 

2331 invert : `Bool`, optional 

2332 If True, unflatten an already flattened image. 

2333 

2334 See Also 

2335 -------- 

2336 lsst.ip.isr.isrFunctions.flatCorrection 

2337 """ 

2338 isrFunctions.flatCorrection( 

2339 maskedImage=exposure.getMaskedImage(), 

2340 flatMaskedImage=flatExposure.getMaskedImage(), 

2341 scalingType=self.config.flatScalingType, 

2342 userScale=self.config.flatUserScale, 

2343 invert=invert, 

2344 trimToFit=self.config.doTrimToMatchCalib 

2345 ) 

2346 

2347 def saturationDetection(self, exposure, amp): 

2348 """Detect and mask saturated pixels in config.saturatedMaskName. 

2349 

2350 Parameters 

2351 ---------- 

2352 exposure : `lsst.afw.image.Exposure` 

2353 Exposure to process. Only the amplifier DataSec is processed. 

2354 amp : `lsst.afw.cameraGeom.Amplifier` 

2355 Amplifier detector data. 

2356 

2357 See Also 

2358 -------- 

2359 lsst.ip.isr.isrFunctions.makeThresholdMask 

2360 """ 

2361 if not math.isnan(amp.getSaturation()): 

2362 maskedImage = exposure.getMaskedImage() 

2363 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2364 isrFunctions.makeThresholdMask( 

2365 maskedImage=dataView, 

2366 threshold=amp.getSaturation(), 

2367 growFootprints=0, 

2368 maskName=self.config.saturatedMaskName, 

2369 ) 

2370 

2371 def saturationInterpolation(self, exposure): 

2372 """Interpolate over saturated pixels, in place. 

2373 

2374 This method should be called after `saturationDetection`, to 

2375 ensure that the saturated pixels have been identified in the 

2376 SAT mask. It should also be called after `assembleCcd`, since 

2377 saturated regions may cross amplifier boundaries. 

2378 

2379 Parameters 

2380 ---------- 

2381 exposure : `lsst.afw.image.Exposure` 

2382 Exposure to process. 

2383 

2384 See Also 

2385 -------- 

2386 lsst.ip.isr.isrTask.saturationDetection 

2387 lsst.ip.isr.isrFunctions.interpolateFromMask 

2388 """ 

2389 isrFunctions.interpolateFromMask( 

2390 maskedImage=exposure.getMaskedImage(), 

2391 fwhm=self.config.fwhm, 

2392 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2393 maskNameList=list(self.config.saturatedMaskName), 

2394 ) 

2395 

2396 def suspectDetection(self, exposure, amp): 

2397 """Detect and mask suspect pixels in config.suspectMaskName. 

2398 

2399 Parameters 

2400 ---------- 

2401 exposure : `lsst.afw.image.Exposure` 

2402 Exposure to process. Only the amplifier DataSec is processed. 

2403 amp : `lsst.afw.cameraGeom.Amplifier` 

2404 Amplifier detector data. 

2405 

2406 See Also 

2407 -------- 

2408 lsst.ip.isr.isrFunctions.makeThresholdMask 

2409 

2410 Notes 

2411 ----- 

2412 Suspect pixels are pixels whose value is greater than 

2413 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2414 affected by unknown systematics; for example if non-linearity 

2415 corrections above a certain level are unstable then that would be a 

2416 useful value for suspectLevel. A value of `nan` indicates that no such 

2417 level exists and no pixels are to be masked as suspicious. 

2418 """ 

2419 suspectLevel = amp.getSuspectLevel() 

2420 if math.isnan(suspectLevel): 

2421 return 

2422 

2423 maskedImage = exposure.getMaskedImage() 

2424 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2425 isrFunctions.makeThresholdMask( 

2426 maskedImage=dataView, 

2427 threshold=suspectLevel, 

2428 growFootprints=0, 

2429 maskName=self.config.suspectMaskName, 

2430 ) 

2431 

2432 def maskDefect(self, exposure, defectBaseList): 

2433 """Mask defects using mask plane "BAD", in place. 

2434 

2435 Parameters 

2436 ---------- 

2437 exposure : `lsst.afw.image.Exposure` 

2438 Exposure to process. 

2439 defectBaseList : defect-type 

2440 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2441 or `list` of `lsst.afw.image.DefectBase`. 

2442 

2443 Notes 

2444 ----- 

2445 Call this after CCD assembly, since defects may cross amplifier 

2446 boundaries. 

2447 """ 

2448 maskedImage = exposure.getMaskedImage() 

2449 if not isinstance(defectBaseList, Defects): 

2450 # Promotes DefectBase to Defect 

2451 defectList = Defects(defectBaseList) 

2452 else: 

2453 defectList = defectBaseList 

2454 defectList.maskPixels(maskedImage, maskName="BAD") 

2455 

2456 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2457 """Mask edge pixels with applicable mask plane. 

2458 

2459 Parameters 

2460 ---------- 

2461 exposure : `lsst.afw.image.Exposure` 

2462 Exposure to process. 

2463 numEdgePixels : `int`, optional 

2464 Number of edge pixels to mask. 

2465 maskPlane : `str`, optional 

2466 Mask plane name to use. 

2467 level : `str`, optional 

2468 Level at which to mask edges. 

2469 """ 

2470 maskedImage = exposure.getMaskedImage() 

2471 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2472 

2473 if numEdgePixels > 0: 

2474 if level == 'DETECTOR': 

2475 boxes = [maskedImage.getBBox()] 

2476 elif level == 'AMP': 

2477 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2478 

2479 for box in boxes: 

2480 # This makes a bbox numEdgeSuspect pixels smaller than the 

2481 # image on each side 

2482 subImage = maskedImage[box] 

2483 box.grow(-numEdgePixels) 

2484 # Mask pixels outside box 

2485 SourceDetectionTask.setEdgeBits( 

2486 subImage, 

2487 box, 

2488 maskBitMask) 

2489 

2490 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2491 """Mask and interpolate defects using mask plane "BAD", in place. 

2492 

2493 Parameters 

2494 ---------- 

2495 exposure : `lsst.afw.image.Exposure` 

2496 Exposure to process. 

2497 defectBaseList : defects-like 

2498 List of defects to mask and interpolate. Can be 

2499 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2500 

2501 See Also 

2502 -------- 

2503 lsst.ip.isr.isrTask.maskDefect 

2504 """ 

2505 self.maskDefect(exposure, defectBaseList) 

2506 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2507 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2508 isrFunctions.interpolateFromMask( 

2509 maskedImage=exposure.getMaskedImage(), 

2510 fwhm=self.config.fwhm, 

2511 growSaturatedFootprints=0, 

2512 maskNameList=["BAD"], 

2513 ) 

2514 

2515 def maskNan(self, exposure): 

2516 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2517 

2518 Parameters 

2519 ---------- 

2520 exposure : `lsst.afw.image.Exposure` 

2521 Exposure to process. 

2522 

2523 Notes 

2524 ----- 

2525 We mask over all non-finite values (NaN, inf), including those 

2526 that are masked with other bits (because those may or may not be 

2527 interpolated over later, and we want to remove all NaN/infs). 

2528 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2529 preserve the historical name. 

2530 """ 

2531 maskedImage = exposure.getMaskedImage() 

2532 

2533 # Find and mask NaNs 

2534 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2535 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2536 numNans = maskNans(maskedImage, maskVal) 

2537 self.metadata["NUMNANS"] = numNans 

2538 if numNans > 0: 

2539 self.log.warning("There were %d unmasked NaNs.", numNans) 

2540 

2541 def maskAndInterpolateNan(self, exposure): 

2542 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2543 in place. 

2544 

2545 Parameters 

2546 ---------- 

2547 exposure : `lsst.afw.image.Exposure` 

2548 Exposure to process. 

2549 

2550 See Also 

2551 -------- 

2552 lsst.ip.isr.isrTask.maskNan 

2553 """ 

2554 self.maskNan(exposure) 

2555 isrFunctions.interpolateFromMask( 

2556 maskedImage=exposure.getMaskedImage(), 

2557 fwhm=self.config.fwhm, 

2558 growSaturatedFootprints=0, 

2559 maskNameList=["UNMASKEDNAN"], 

2560 ) 

2561 

2562 def measureBackground(self, exposure, IsrQaConfig=None): 

2563 """Measure the image background in subgrids, for quality control. 

2564 

2565 Parameters 

2566 ---------- 

2567 exposure : `lsst.afw.image.Exposure` 

2568 Exposure to process. 

2569 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2570 Configuration object containing parameters on which background 

2571 statistics and subgrids to use. 

2572 """ 

2573 if IsrQaConfig is not None: 

2574 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2575 IsrQaConfig.flatness.nIter) 

2576 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2577 statsControl.setAndMask(maskVal) 

2578 maskedImage = exposure.getMaskedImage() 

2579 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2580 skyLevel = stats.getValue(afwMath.MEDIAN) 

2581 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2582 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2583 metadata = exposure.getMetadata() 

2584 metadata["SKYLEVEL"] = skyLevel 

2585 metadata["SKYSIGMA"] = skySigma 

2586 

2587 # calcluating flatlevel over the subgrids 

2588 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2589 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2590 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2591 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2592 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2593 skyLevels = numpy.zeros((nX, nY)) 

2594 

2595 for j in range(nY): 

2596 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2597 for i in range(nX): 

2598 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2599 

2600 xLLC = xc - meshXHalf 

2601 yLLC = yc - meshYHalf 

2602 xURC = xc + meshXHalf - 1 

2603 yURC = yc + meshYHalf - 1 

2604 

2605 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2606 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2607 

2608 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2609 

2610 good = numpy.where(numpy.isfinite(skyLevels)) 

2611 skyMedian = numpy.median(skyLevels[good]) 

2612 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2613 flatness_rms = numpy.std(flatness) 

2614 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2615 

2616 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2617 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2618 nX, nY, flatness_pp, flatness_rms) 

2619 

2620 metadata["FLATNESS_PP"] = float(flatness_pp) 

2621 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2622 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2623 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2624 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2625 

2626 def roughZeroPoint(self, exposure): 

2627 """Set an approximate magnitude zero point for the exposure. 

2628 

2629 Parameters 

2630 ---------- 

2631 exposure : `lsst.afw.image.Exposure` 

2632 Exposure to process. 

2633 """ 

2634 filterLabel = exposure.getFilter() 

2635 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2636 

2637 if physicalFilter in self.config.fluxMag0T1: 

2638 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2639 else: 

2640 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2641 fluxMag0 = self.config.defaultFluxMag0T1 

2642 

2643 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2644 if not expTime > 0: # handle NaN as well as <= 0 

2645 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2646 return 

2647 

2648 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2649 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2650 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2651 

2652 @contextmanager 

2653 def flatContext(self, exp, flat, dark=None): 

2654 """Context manager that applies and removes flats and darks, 

2655 if the task is configured to apply them. 

2656 

2657 Parameters 

2658 ---------- 

2659 exp : `lsst.afw.image.Exposure` 

2660 Exposure to process. 

2661 flat : `lsst.afw.image.Exposure` 

2662 Flat exposure the same size as ``exp``. 

2663 dark : `lsst.afw.image.Exposure`, optional 

2664 Dark exposure the same size as ``exp``. 

2665 

2666 Yields 

2667 ------ 

2668 exp : `lsst.afw.image.Exposure` 

2669 The flat and dark corrected exposure. 

2670 """ 

2671 if self.config.doDark and dark is not None: 

2672 self.darkCorrection(exp, dark) 

2673 if self.config.doFlat: 

2674 self.flatCorrection(exp, flat) 

2675 try: 

2676 yield exp 

2677 finally: 

2678 if self.config.doFlat: 

2679 self.flatCorrection(exp, flat, invert=True) 

2680 if self.config.doDark and dark is not None: 

2681 self.darkCorrection(exp, dark, invert=True) 

2682 

2683 def makeBinnedImages(self, exposure): 

2684 """Make visualizeVisit style binned exposures. 

2685 

2686 Parameters 

2687 ---------- 

2688 exposure : `lsst.afw.image.Exposure` 

2689 Exposure to bin. 

2690 

2691 Returns 

2692 ------- 

2693 bin1 : `lsst.afw.image.Exposure` 

2694 Binned exposure using binFactor1. 

2695 bin2 : `lsst.afw.image.Exposure` 

2696 Binned exposure using binFactor2. 

2697 """ 

2698 mi = exposure.getMaskedImage() 

2699 

2700 bin1 = afwMath.binImage(mi, self.config.binFactor1) 

2701 bin2 = afwMath.binImage(mi, self.config.binFactor2) 

2702 

2703 return bin1, bin2 

2704 

2705 def debugView(self, exposure, stepname): 

2706 """Utility function to examine ISR exposure at different stages. 

2707 

2708 Parameters 

2709 ---------- 

2710 exposure : `lsst.afw.image.Exposure` 

2711 Exposure to view. 

2712 stepname : `str` 

2713 State of processing to view. 

2714 """ 

2715 frame = getDebugFrame(self._display, stepname) 

2716 if frame: 

2717 display = getDisplay(frame) 

2718 display.scale('asinh', 'zscale') 

2719 display.mtv(exposure) 

2720 prompt = "Press Enter to continue [c]... " 

2721 while True: 

2722 ans = input(prompt).lower() 

2723 if ans in ("", "c",): 

2724 break 

2725 

2726 

2727class FakeAmp(object): 

2728 """A Detector-like object that supports returning gain and saturation level 

2729 

2730 This is used when the input exposure does not have a detector. 

2731 

2732 Parameters 

2733 ---------- 

2734 exposure : `lsst.afw.image.Exposure` 

2735 Exposure to generate a fake amplifier for. 

2736 config : `lsst.ip.isr.isrTaskConfig` 

2737 Configuration to apply to the fake amplifier. 

2738 """ 

2739 

2740 def __init__(self, exposure, config): 

2741 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2742 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2743 self._gain = config.gain 

2744 self._readNoise = config.readNoise 

2745 self._saturation = config.saturation 

2746 

2747 def getBBox(self): 

2748 return self._bbox 

2749 

2750 def getRawBBox(self): 

2751 return self._bbox 

2752 

2753 def getRawHorizontalOverscanBBox(self): 

2754 return self._RawHorizontalOverscanBBox 

2755 

2756 def getGain(self): 

2757 return self._gain 

2758 

2759 def getReadNoise(self): 

2760 return self._readNoise 

2761 

2762 def getSaturation(self): 

2763 return self._saturation 

2764 

2765 def getSuspectLevel(self): 

2766 return float("NaN")