Coverage for python/lsst/ip/isr/isrTask.py: 16%

942 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-02 11:55 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from .ptcDataset import PhotonTransferCurveDataset 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.DataCoordinate` 

75 Expanded data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(registry.dimensions.conform(["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 records = {k: newDataId.records[k] for k in newDataId.dimensions.elements} 

95 return [ref.expanded(registry.expandDataId(ref.dataId, records=records)) for ref in results] 

96 

97 

98class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

99 dimensions={"instrument", "exposure", "detector"}, 

100 defaultTemplates={}): 

101 ccdExposure = cT.Input( 

102 name="raw", 

103 doc="Input exposure to process.", 

104 storageClass="Exposure", 

105 dimensions=["instrument", "exposure", "detector"], 

106 ) 

107 camera = cT.PrerequisiteInput( 

108 name="camera", 

109 storageClass="Camera", 

110 doc="Input camera to construct complete exposures.", 

111 dimensions=["instrument"], 

112 isCalibration=True, 

113 ) 

114 

115 crosstalk = cT.PrerequisiteInput( 

116 name="crosstalk", 

117 doc="Input crosstalk object", 

118 storageClass="CrosstalkCalib", 

119 dimensions=["instrument", "detector"], 

120 isCalibration=True, 

121 minimum=0, # can fall back to cameraGeom 

122 ) 

123 crosstalkSources = cT.PrerequisiteInput( 

124 name="isrOverscanCorrected", 

125 doc="Overscan corrected input images.", 

126 storageClass="Exposure", 

127 dimensions=["instrument", "exposure", "detector"], 

128 deferLoad=True, 

129 multiple=True, 

130 lookupFunction=crosstalkSourceLookup, 

131 minimum=0, # not needed for all instruments, no config to control this 

132 ) 

133 bias = cT.PrerequisiteInput( 

134 name="bias", 

135 doc="Input bias calibration.", 

136 storageClass="ExposureF", 

137 dimensions=["instrument", "detector"], 

138 isCalibration=True, 

139 ) 

140 dark = cT.PrerequisiteInput( 

141 name='dark', 

142 doc="Input dark calibration.", 

143 storageClass="ExposureF", 

144 dimensions=["instrument", "detector"], 

145 isCalibration=True, 

146 ) 

147 flat = cT.PrerequisiteInput( 

148 name="flat", 

149 doc="Input flat calibration.", 

150 storageClass="ExposureF", 

151 dimensions=["instrument", "physical_filter", "detector"], 

152 isCalibration=True, 

153 ) 

154 ptc = cT.PrerequisiteInput( 

155 name="ptc", 

156 doc="Input Photon Transfer Curve dataset", 

157 storageClass="PhotonTransferCurveDataset", 

158 dimensions=["instrument", "detector"], 

159 isCalibration=True, 

160 ) 

161 fringes = cT.PrerequisiteInput( 

162 name="fringe", 

163 doc="Input fringe calibration.", 

164 storageClass="ExposureF", 

165 dimensions=["instrument", "physical_filter", "detector"], 

166 isCalibration=True, 

167 minimum=0, # only needed for some bands, even when enabled 

168 ) 

169 strayLightData = cT.PrerequisiteInput( 

170 name='yBackground', 

171 doc="Input stray light calibration.", 

172 storageClass="StrayLightData", 

173 dimensions=["instrument", "physical_filter", "detector"], 

174 deferLoad=True, 

175 isCalibration=True, 

176 minimum=0, # only needed for some bands, even when enabled 

177 ) 

178 bfKernel = cT.PrerequisiteInput( 

179 name='bfKernel', 

180 doc="Input brighter-fatter kernel.", 

181 storageClass="NumpyArray", 

182 dimensions=["instrument"], 

183 isCalibration=True, 

184 minimum=0, # can use either bfKernel or newBFKernel 

185 ) 

186 newBFKernel = cT.PrerequisiteInput( 

187 name='brighterFatterKernel', 

188 doc="Newer complete kernel + gain solutions.", 

189 storageClass="BrighterFatterKernel", 

190 dimensions=["instrument", "detector"], 

191 isCalibration=True, 

192 minimum=0, # can use either bfKernel or newBFKernel 

193 ) 

194 defects = cT.PrerequisiteInput( 

195 name='defects', 

196 doc="Input defect tables.", 

197 storageClass="Defects", 

198 dimensions=["instrument", "detector"], 

199 isCalibration=True, 

200 ) 

201 linearizer = cT.PrerequisiteInput( 

202 name='linearizer', 

203 storageClass="Linearizer", 

204 doc="Linearity correction calibration.", 

205 dimensions=["instrument", "detector"], 

206 isCalibration=True, 

207 minimum=0, # can fall back to cameraGeom 

208 ) 

209 opticsTransmission = cT.PrerequisiteInput( 

210 name="transmission_optics", 

211 storageClass="TransmissionCurve", 

212 doc="Transmission curve due to the optics.", 

213 dimensions=["instrument"], 

214 isCalibration=True, 

215 ) 

216 filterTransmission = cT.PrerequisiteInput( 

217 name="transmission_filter", 

218 storageClass="TransmissionCurve", 

219 doc="Transmission curve due to the filter.", 

220 dimensions=["instrument", "physical_filter"], 

221 isCalibration=True, 

222 ) 

223 sensorTransmission = cT.PrerequisiteInput( 

224 name="transmission_sensor", 

225 storageClass="TransmissionCurve", 

226 doc="Transmission curve due to the sensor.", 

227 dimensions=["instrument", "detector"], 

228 isCalibration=True, 

229 ) 

230 atmosphereTransmission = cT.PrerequisiteInput( 

231 name="transmission_atmosphere", 

232 storageClass="TransmissionCurve", 

233 doc="Transmission curve due to the atmosphere.", 

234 dimensions=["instrument"], 

235 isCalibration=True, 

236 ) 

237 illumMaskedImage = cT.PrerequisiteInput( 

238 name="illum", 

239 doc="Input illumination correction.", 

240 storageClass="MaskedImageF", 

241 dimensions=["instrument", "physical_filter", "detector"], 

242 isCalibration=True, 

243 ) 

244 deferredChargeCalib = cT.PrerequisiteInput( 

245 name="cpCtiCalib", 

246 doc="Deferred charge/CTI correction dataset.", 

247 storageClass="IsrCalib", 

248 dimensions=["instrument", "detector"], 

249 isCalibration=True, 

250 ) 

251 

252 outputExposure = cT.Output( 

253 name='postISRCCD', 

254 doc="Output ISR processed exposure.", 

255 storageClass="Exposure", 

256 dimensions=["instrument", "exposure", "detector"], 

257 ) 

258 preInterpExposure = cT.Output( 

259 name='preInterpISRCCD', 

260 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

261 storageClass="ExposureF", 

262 dimensions=["instrument", "exposure", "detector"], 

263 ) 

264 outputBin1Exposure = cT.Output( 

265 name="postIsrBin1", 

266 doc="First binned image.", 

267 storageClass="ExposureF", 

268 dimensions=["instrument", "exposure", "detector"], 

269 ) 

270 outputBin2Exposure = cT.Output( 

271 name="postIsrBin2", 

272 doc="Second binned image.", 

273 storageClass="ExposureF", 

274 dimensions=["instrument", "exposure", "detector"], 

275 ) 

276 

277 outputOssThumbnail = cT.Output( 

278 name="OssThumb", 

279 doc="Output Overscan-subtracted thumbnail image.", 

280 storageClass="Thumbnail", 

281 dimensions=["instrument", "exposure", "detector"], 

282 ) 

283 outputFlattenedThumbnail = cT.Output( 

284 name="FlattenedThumb", 

285 doc="Output flat-corrected thumbnail image.", 

286 storageClass="Thumbnail", 

287 dimensions=["instrument", "exposure", "detector"], 

288 ) 

289 outputStatistics = cT.Output( 

290 name="isrStatistics", 

291 doc="Output of additional statistics table.", 

292 storageClass="StructuredDataDict", 

293 dimensions=["instrument", "exposure", "detector"], 

294 ) 

295 

296 def __init__(self, *, config=None): 

297 super().__init__(config=config) 

298 

299 if config.doBias is not True: 

300 self.prerequisiteInputs.remove("bias") 

301 if config.doLinearize is not True: 

302 self.prerequisiteInputs.remove("linearizer") 

303 if config.doCrosstalk is not True: 

304 self.prerequisiteInputs.remove("crosstalkSources") 

305 self.prerequisiteInputs.remove("crosstalk") 

306 if config.doBrighterFatter is not True: 

307 self.prerequisiteInputs.remove("bfKernel") 

308 self.prerequisiteInputs.remove("newBFKernel") 

309 if config.doDefect is not True: 

310 self.prerequisiteInputs.remove("defects") 

311 if config.doDark is not True: 

312 self.prerequisiteInputs.remove("dark") 

313 if config.doFlat is not True: 

314 self.prerequisiteInputs.remove("flat") 

315 if config.doFringe is not True: 

316 self.prerequisiteInputs.remove("fringes") 

317 if config.doStrayLight is not True: 

318 self.prerequisiteInputs.remove("strayLightData") 

319 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

320 self.prerequisiteInputs.remove("ptc") 

321 if config.doAttachTransmissionCurve is not True: 

322 self.prerequisiteInputs.remove("opticsTransmission") 

323 self.prerequisiteInputs.remove("filterTransmission") 

324 self.prerequisiteInputs.remove("sensorTransmission") 

325 self.prerequisiteInputs.remove("atmosphereTransmission") 

326 else: 

327 if config.doUseOpticsTransmission is not True: 

328 self.prerequisiteInputs.remove("opticsTransmission") 

329 if config.doUseFilterTransmission is not True: 

330 self.prerequisiteInputs.remove("filterTransmission") 

331 if config.doUseSensorTransmission is not True: 

332 self.prerequisiteInputs.remove("sensorTransmission") 

333 if config.doUseAtmosphereTransmission is not True: 

334 self.prerequisiteInputs.remove("atmosphereTransmission") 

335 if config.doIlluminationCorrection is not True: 

336 self.prerequisiteInputs.remove("illumMaskedImage") 

337 if config.doDeferredCharge is not True: 

338 self.prerequisiteInputs.remove("deferredChargeCalib") 

339 

340 if config.doWrite is not True: 

341 self.outputs.remove("outputExposure") 

342 self.outputs.remove("preInterpExposure") 

343 self.outputs.remove("outputFlattenedThumbnail") 

344 self.outputs.remove("outputOssThumbnail") 

345 self.outputs.remove("outputStatistics") 

346 self.outputs.remove("outputBin1Exposure") 

347 self.outputs.remove("outputBin2Exposure") 

348 else: 

349 if config.doBinnedExposures is not True: 

350 self.outputs.remove("outputBin1Exposure") 

351 self.outputs.remove("outputBin2Exposure") 

352 if config.doSaveInterpPixels is not True: 

353 self.outputs.remove("preInterpExposure") 

354 if config.qa.doThumbnailOss is not True: 

355 self.outputs.remove("outputOssThumbnail") 

356 if config.qa.doThumbnailFlattened is not True: 

357 self.outputs.remove("outputFlattenedThumbnail") 

358 if config.doCalculateStatistics is not True: 

359 self.outputs.remove("outputStatistics") 

360 

361 

362class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

363 pipelineConnections=IsrTaskConnections): 

364 """Configuration parameters for IsrTask. 

365 

366 Items are grouped in the order in which they are executed by the task. 

367 """ 

368 datasetType = pexConfig.Field( 

369 dtype=str, 

370 doc="Dataset type for input data; users will typically leave this alone, " 

371 "but camera-specific ISR tasks will override it", 

372 default="raw", 

373 ) 

374 

375 fallbackFilterName = pexConfig.Field( 

376 dtype=str, 

377 doc="Fallback default filter name for calibrations.", 

378 optional=True 

379 ) 

380 useFallbackDate = pexConfig.Field( 

381 dtype=bool, 

382 doc="Pass observation date when using fallback filter.", 

383 default=False, 

384 ) 

385 expectWcs = pexConfig.Field( 

386 dtype=bool, 

387 default=True, 

388 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

389 ) 

390 fwhm = pexConfig.Field( 

391 dtype=float, 

392 doc="FWHM of PSF in arcseconds (currently unused).", 

393 default=1.0, 

394 ) 

395 qa = pexConfig.ConfigField( 

396 dtype=isrQa.IsrQaConfig, 

397 doc="QA related configuration options.", 

398 ) 

399 doHeaderProvenance = pexConfig.Field( 

400 dtype=bool, 

401 default=True, 

402 doc="Write calibration identifiers into output exposure header?", 

403 ) 

404 

405 # Calib checking configuration: 

406 doRaiseOnCalibMismatch = pexConfig.Field( 

407 dtype=bool, 

408 default=False, 

409 doc="Should IsrTask halt if exposure and calibration header values do not match?", 

410 ) 

411 cameraKeywordsToCompare = pexConfig.ListField( 

412 dtype=str, 

413 doc="List of header keywords to compare between exposure and calibrations.", 

414 default=[], 

415 ) 

416 

417 # Image conversion configuration 

418 doConvertIntToFloat = pexConfig.Field( 

419 dtype=bool, 

420 doc="Convert integer raw images to floating point values?", 

421 default=True, 

422 ) 

423 

424 # Saturated pixel handling. 

425 doSaturation = pexConfig.Field( 

426 dtype=bool, 

427 doc="Mask saturated pixels? NB: this is totally independent of the" 

428 " interpolation option - this is ONLY setting the bits in the mask." 

429 " To have them interpolated make sure doSaturationInterpolation=True", 

430 default=True, 

431 ) 

432 saturatedMaskName = pexConfig.Field( 

433 dtype=str, 

434 doc="Name of mask plane to use in saturation detection and interpolation", 

435 default="SAT", 

436 ) 

437 saturation = pexConfig.Field( 

438 dtype=float, 

439 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

440 default=float("NaN"), 

441 ) 

442 growSaturationFootprintSize = pexConfig.Field( 

443 dtype=int, 

444 doc="Number of pixels by which to grow the saturation footprints", 

445 default=1, 

446 ) 

447 

448 # Suspect pixel handling. 

449 doSuspect = pexConfig.Field( 

450 dtype=bool, 

451 doc="Mask suspect pixels?", 

452 default=False, 

453 ) 

454 suspectMaskName = pexConfig.Field( 

455 dtype=str, 

456 doc="Name of mask plane to use for suspect pixels", 

457 default="SUSPECT", 

458 ) 

459 numEdgeSuspect = pexConfig.Field( 

460 dtype=int, 

461 doc="Number of edge pixels to be flagged as untrustworthy.", 

462 default=0, 

463 ) 

464 edgeMaskLevel = pexConfig.ChoiceField( 

465 dtype=str, 

466 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

467 default="DETECTOR", 

468 allowed={ 

469 'DETECTOR': 'Mask only the edges of the full detector.', 

470 'AMP': 'Mask edges of each amplifier.', 

471 }, 

472 ) 

473 

474 # Initial masking options. 

475 doSetBadRegions = pexConfig.Field( 

476 dtype=bool, 

477 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

478 default=True, 

479 ) 

480 badStatistic = pexConfig.ChoiceField( 

481 dtype=str, 

482 doc="How to estimate the average value for BAD regions.", 

483 default='MEANCLIP', 

484 allowed={ 

485 "MEANCLIP": "Correct using the (clipped) mean of good data", 

486 "MEDIAN": "Correct using the median of the good data", 

487 }, 

488 ) 

489 

490 # Overscan subtraction configuration. 

491 doOverscan = pexConfig.Field( 

492 dtype=bool, 

493 doc="Do overscan subtraction?", 

494 default=True, 

495 ) 

496 overscan = pexConfig.ConfigurableField( 

497 target=OverscanCorrectionTask, 

498 doc="Overscan subtraction task for image segments.", 

499 ) 

500 

501 # Amplifier to CCD assembly configuration 

502 doAssembleCcd = pexConfig.Field( 

503 dtype=bool, 

504 default=True, 

505 doc="Assemble amp-level exposures into a ccd-level exposure?" 

506 ) 

507 assembleCcd = pexConfig.ConfigurableField( 

508 target=AssembleCcdTask, 

509 doc="CCD assembly task", 

510 ) 

511 

512 # General calibration configuration. 

513 doAssembleIsrExposures = pexConfig.Field( 

514 dtype=bool, 

515 default=False, 

516 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

517 ) 

518 doTrimToMatchCalib = pexConfig.Field( 

519 dtype=bool, 

520 default=False, 

521 doc="Trim raw data to match calibration bounding boxes?" 

522 ) 

523 

524 # Bias subtraction. 

525 doBias = pexConfig.Field( 

526 dtype=bool, 

527 doc="Apply bias frame correction?", 

528 default=True, 

529 ) 

530 biasDataProductName = pexConfig.Field( 

531 dtype=str, 

532 doc="Name of the bias data product", 

533 default="bias", 

534 ) 

535 doBiasBeforeOverscan = pexConfig.Field( 

536 dtype=bool, 

537 doc="Reverse order of overscan and bias correction.", 

538 default=False 

539 ) 

540 

541 # Deferred charge correction. 

542 doDeferredCharge = pexConfig.Field( 

543 dtype=bool, 

544 doc="Apply deferred charge correction?", 

545 default=False, 

546 ) 

547 deferredChargeCorrection = pexConfig.ConfigurableField( 

548 target=DeferredChargeTask, 

549 doc="Deferred charge correction task.", 

550 ) 

551 

552 # Variance construction 

553 doVariance = pexConfig.Field( 

554 dtype=bool, 

555 doc="Calculate variance?", 

556 default=True 

557 ) 

558 gain = pexConfig.Field( 

559 dtype=float, 

560 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

561 default=float("NaN"), 

562 ) 

563 readNoise = pexConfig.Field( 

564 dtype=float, 

565 doc="The read noise to use if no Detector is present in the Exposure", 

566 default=0.0, 

567 ) 

568 doEmpiricalReadNoise = pexConfig.Field( 

569 dtype=bool, 

570 default=False, 

571 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

572 ) 

573 usePtcReadNoise = pexConfig.Field( 

574 dtype=bool, 

575 default=False, 

576 doc="Use readnoise values from the Photon Transfer Curve?" 

577 ) 

578 maskNegativeVariance = pexConfig.Field( 

579 dtype=bool, 

580 default=True, 

581 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

582 "in the measurement of the overscan at an edge due to the data falling off faster " 

583 "than the overscan model can account for it." 

584 ) 

585 negativeVarianceMaskName = pexConfig.Field( 

586 dtype=str, 

587 default="BAD", 

588 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

589 ) 

590 # Linearization. 

591 doLinearize = pexConfig.Field( 

592 dtype=bool, 

593 doc="Correct for nonlinearity of the detector's response?", 

594 default=True, 

595 ) 

596 

597 # Crosstalk. 

598 doCrosstalk = pexConfig.Field( 

599 dtype=bool, 

600 doc="Apply intra-CCD crosstalk correction?", 

601 default=False, 

602 ) 

603 doCrosstalkBeforeAssemble = pexConfig.Field( 

604 dtype=bool, 

605 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

606 default=False, 

607 ) 

608 crosstalk = pexConfig.ConfigurableField( 

609 target=CrosstalkTask, 

610 doc="Intra-CCD crosstalk correction", 

611 ) 

612 

613 # Masking options. 

614 doDefect = pexConfig.Field( 

615 dtype=bool, 

616 doc="Apply correction for CCD defects, e.g. hot pixels?", 

617 default=True, 

618 ) 

619 doNanMasking = pexConfig.Field( 

620 dtype=bool, 

621 doc="Mask non-finite (NAN, inf) pixels?", 

622 default=True, 

623 ) 

624 doWidenSaturationTrails = pexConfig.Field( 

625 dtype=bool, 

626 doc="Widen bleed trails based on their width?", 

627 default=True 

628 ) 

629 

630 # Brighter-Fatter correction. 

631 doBrighterFatter = pexConfig.Field( 

632 dtype=bool, 

633 default=False, 

634 doc="Apply the brighter-fatter correction?" 

635 ) 

636 doFluxConservingBrighterFatterCorrection = pexConfig.Field( 

637 dtype=bool, 

638 default=False, 

639 doc="Apply the flux-conserving BFE correction by Miller et al.?" 

640 ) 

641 brighterFatterLevel = pexConfig.ChoiceField( 

642 dtype=str, 

643 default="DETECTOR", 

644 doc="The level at which to correct for brighter-fatter.", 

645 allowed={ 

646 "AMP": "Every amplifier treated separately.", 

647 "DETECTOR": "One kernel per detector", 

648 } 

649 ) 

650 brighterFatterMaxIter = pexConfig.Field( 

651 dtype=int, 

652 default=10, 

653 doc="Maximum number of iterations for the brighter-fatter correction" 

654 ) 

655 brighterFatterThreshold = pexConfig.Field( 

656 dtype=float, 

657 default=1000, 

658 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

659 "absolute value of the difference between the current corrected image and the one " 

660 "from the previous iteration summed over all the pixels." 

661 ) 

662 brighterFatterApplyGain = pexConfig.Field( 

663 dtype=bool, 

664 default=True, 

665 doc="Should the gain be applied when applying the brighter-fatter correction?" 

666 ) 

667 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

668 dtype=str, 

669 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

670 "correction.", 

671 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

672 ) 

673 brighterFatterMaskGrowSize = pexConfig.Field( 

674 dtype=int, 

675 default=0, 

676 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

677 "when brighter-fatter correction is applied." 

678 ) 

679 

680 # Dark subtraction. 

681 doDark = pexConfig.Field( 

682 dtype=bool, 

683 doc="Apply dark frame correction?", 

684 default=True, 

685 ) 

686 darkDataProductName = pexConfig.Field( 

687 dtype=str, 

688 doc="Name of the dark data product", 

689 default="dark", 

690 ) 

691 

692 # Camera-specific stray light removal. 

693 doStrayLight = pexConfig.Field( 

694 dtype=bool, 

695 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

696 default=False, 

697 ) 

698 strayLight = pexConfig.ConfigurableField( 

699 target=StrayLightTask, 

700 doc="y-band stray light correction" 

701 ) 

702 

703 # Flat correction. 

704 doFlat = pexConfig.Field( 

705 dtype=bool, 

706 doc="Apply flat field correction?", 

707 default=True, 

708 ) 

709 flatDataProductName = pexConfig.Field( 

710 dtype=str, 

711 doc="Name of the flat data product", 

712 default="flat", 

713 ) 

714 flatScalingType = pexConfig.ChoiceField( 

715 dtype=str, 

716 doc="The method for scaling the flat on the fly.", 

717 default='USER', 

718 allowed={ 

719 "USER": "Scale by flatUserScale", 

720 "MEAN": "Scale by the inverse of the mean", 

721 "MEDIAN": "Scale by the inverse of the median", 

722 }, 

723 ) 

724 flatUserScale = pexConfig.Field( 

725 dtype=float, 

726 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

727 default=1.0, 

728 ) 

729 doTweakFlat = pexConfig.Field( 

730 dtype=bool, 

731 doc="Tweak flats to match observed amplifier ratios?", 

732 default=False 

733 ) 

734 

735 # Amplifier normalization based on gains instead of using flats 

736 # configuration. 

737 doApplyGains = pexConfig.Field( 

738 dtype=bool, 

739 doc="Correct the amplifiers for their gains instead of applying flat correction", 

740 default=False, 

741 ) 

742 usePtcGains = pexConfig.Field( 

743 dtype=bool, 

744 doc="Use the gain values from the input Photon Transfer Curve?", 

745 default=False, 

746 ) 

747 normalizeGains = pexConfig.Field( 

748 dtype=bool, 

749 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

750 default=False, 

751 ) 

752 

753 # Fringe correction. 

754 doFringe = pexConfig.Field( 

755 dtype=bool, 

756 doc="Apply fringe correction?", 

757 default=True, 

758 ) 

759 fringe = pexConfig.ConfigurableField( 

760 target=FringeTask, 

761 doc="Fringe subtraction task", 

762 ) 

763 fringeAfterFlat = pexConfig.Field( 

764 dtype=bool, 

765 doc="Do fringe subtraction after flat-fielding?", 

766 default=True, 

767 ) 

768 

769 # Amp offset correction. 

770 doAmpOffset = pexConfig.Field( 

771 doc="Calculate and apply amp offset corrections?", 

772 dtype=bool, 

773 default=False, 

774 ) 

775 ampOffset = pexConfig.ConfigurableField( 

776 doc="Amp offset correction task.", 

777 target=AmpOffsetTask, 

778 ) 

779 

780 # Initial CCD-level background statistics options. 

781 doMeasureBackground = pexConfig.Field( 

782 dtype=bool, 

783 doc="Measure the background level on the reduced image?", 

784 default=False, 

785 ) 

786 

787 # Camera-specific masking configuration. 

788 doCameraSpecificMasking = pexConfig.Field( 

789 dtype=bool, 

790 doc="Mask camera-specific bad regions?", 

791 default=False, 

792 ) 

793 masking = pexConfig.ConfigurableField( 

794 target=MaskingTask, 

795 doc="Masking task." 

796 ) 

797 

798 # Interpolation options. 

799 doInterpolate = pexConfig.Field( 

800 dtype=bool, 

801 doc="Interpolate masked pixels?", 

802 default=True, 

803 ) 

804 doSaturationInterpolation = pexConfig.Field( 

805 dtype=bool, 

806 doc="Perform interpolation over pixels masked as saturated?" 

807 " NB: This is independent of doSaturation; if that is False this plane" 

808 " will likely be blank, resulting in a no-op here.", 

809 default=True, 

810 ) 

811 doNanInterpolation = pexConfig.Field( 

812 dtype=bool, 

813 doc="Perform interpolation over pixels masked as NaN?" 

814 " NB: This is independent of doNanMasking; if that is False this plane" 

815 " will likely be blank, resulting in a no-op here.", 

816 default=True, 

817 ) 

818 doNanInterpAfterFlat = pexConfig.Field( 

819 dtype=bool, 

820 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

821 "also have to interpolate them before flat-fielding."), 

822 default=False, 

823 ) 

824 maskListToInterpolate = pexConfig.ListField( 

825 dtype=str, 

826 doc="List of mask planes that should be interpolated.", 

827 default=['SAT', 'BAD'], 

828 ) 

829 doSaveInterpPixels = pexConfig.Field( 

830 dtype=bool, 

831 doc="Save a copy of the pre-interpolated pixel values?", 

832 default=False, 

833 ) 

834 

835 # Default photometric calibration options. 

836 fluxMag0T1 = pexConfig.DictField( 

837 keytype=str, 

838 itemtype=float, 

839 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

840 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

841 )) 

842 ) 

843 defaultFluxMag0T1 = pexConfig.Field( 

844 dtype=float, 

845 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

846 default=pow(10.0, 0.4*28.0) 

847 ) 

848 

849 # Vignette correction configuration. 

850 doVignette = pexConfig.Field( 

851 dtype=bool, 

852 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

853 "according to vignetting parameters?"), 

854 default=False, 

855 ) 

856 doMaskVignettePolygon = pexConfig.Field( 

857 dtype=bool, 

858 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

859 "is False"), 

860 default=True, 

861 ) 

862 vignetteValue = pexConfig.Field( 

863 dtype=float, 

864 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

865 optional=True, 

866 default=None, 

867 ) 

868 vignette = pexConfig.ConfigurableField( 

869 target=VignetteTask, 

870 doc="Vignetting task.", 

871 ) 

872 

873 # Transmission curve configuration. 

874 doAttachTransmissionCurve = pexConfig.Field( 

875 dtype=bool, 

876 default=False, 

877 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

878 ) 

879 doUseOpticsTransmission = pexConfig.Field( 

880 dtype=bool, 

881 default=True, 

882 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

883 ) 

884 doUseFilterTransmission = pexConfig.Field( 

885 dtype=bool, 

886 default=True, 

887 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

888 ) 

889 doUseSensorTransmission = pexConfig.Field( 

890 dtype=bool, 

891 default=True, 

892 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

893 ) 

894 doUseAtmosphereTransmission = pexConfig.Field( 

895 dtype=bool, 

896 default=True, 

897 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

898 ) 

899 

900 # Illumination correction. 

901 doIlluminationCorrection = pexConfig.Field( 

902 dtype=bool, 

903 default=False, 

904 doc="Perform illumination correction?" 

905 ) 

906 illuminationCorrectionDataProductName = pexConfig.Field( 

907 dtype=str, 

908 doc="Name of the illumination correction data product.", 

909 default="illumcor", 

910 ) 

911 illumScale = pexConfig.Field( 

912 dtype=float, 

913 doc="Scale factor for the illumination correction.", 

914 default=1.0, 

915 ) 

916 illumFilters = pexConfig.ListField( 

917 dtype=str, 

918 default=[], 

919 doc="Only perform illumination correction for these filters." 

920 ) 

921 

922 # Calculate image quality statistics? 

923 doStandardStatistics = pexConfig.Field( 

924 dtype=bool, 

925 doc="Should standard image quality statistics be calculated?", 

926 default=True, 

927 ) 

928 # Calculate additional statistics? 

929 doCalculateStatistics = pexConfig.Field( 

930 dtype=bool, 

931 doc="Should additional ISR statistics be calculated?", 

932 default=False, 

933 ) 

934 isrStats = pexConfig.ConfigurableField( 

935 target=IsrStatisticsTask, 

936 doc="Task to calculate additional statistics.", 

937 ) 

938 

939 # Make binned images? 

940 doBinnedExposures = pexConfig.Field( 

941 dtype=bool, 

942 doc="Should binned exposures be calculated?", 

943 default=False, 

944 ) 

945 binFactor1 = pexConfig.Field( 945 ↛ exitline 945 didn't jump to the function exit

946 dtype=int, 

947 doc="Binning factor for first binned exposure. This is intended for a finely binned output.", 

948 default=8, 

949 check=lambda x: x > 1, 

950 ) 

951 binFactor2 = pexConfig.Field( 951 ↛ exitline 951 didn't jump to the function exit

952 dtype=int, 

953 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.", 

954 default=64, 

955 check=lambda x: x > 1, 

956 ) 

957 

958 # Write the outputs to disk. If ISR is run as a subtask, this may not 

959 # be needed. 

960 doWrite = pexConfig.Field( 

961 dtype=bool, 

962 doc="Persist postISRCCD?", 

963 default=True, 

964 ) 

965 

966 def validate(self): 

967 super().validate() 

968 if self.doFlat and self.doApplyGains: 

969 raise ValueError("You may not specify both doFlat and doApplyGains") 

970 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

971 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

972 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

973 self.maskListToInterpolate.append(self.saturatedMaskName) 

974 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

975 self.maskListToInterpolate.remove(self.saturatedMaskName) 

976 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

977 self.maskListToInterpolate.append("UNMASKEDNAN") 

978 if self.doCalculateStatistics and self.isrStats.doCtiStatistics: 

979 if self.doApplyGains != self.isrStats.doApplyGainsForCtiStatistics: 

980 raise ValueError("doApplyGains must match isrStats.applyGainForCtiStatistics.") 

981 

982 

983class IsrTask(pipeBase.PipelineTask): 

984 """Apply common instrument signature correction algorithms to a raw frame. 

985 

986 The process for correcting imaging data is very similar from 

987 camera to camera. This task provides a vanilla implementation of 

988 doing these corrections, including the ability to turn certain 

989 corrections off if they are not needed. The inputs to the primary 

990 method, `run()`, are a raw exposure to be corrected and the 

991 calibration data products. The raw input is a single chip sized 

992 mosaic of all amps including overscans and other non-science 

993 pixels. 

994 

995 The __init__ method sets up the subtasks for ISR processing, using 

996 the defaults from `lsst.ip.isr`. 

997 

998 Parameters 

999 ---------- 

1000 args : `list` 

1001 Positional arguments passed to the Task constructor. 

1002 None used at this time. 

1003 kwargs : `dict`, optional 

1004 Keyword arguments passed on to the Task constructor. 

1005 None used at this time. 

1006 """ 

1007 ConfigClass = IsrTaskConfig 

1008 _DefaultName = "isr" 

1009 

1010 def __init__(self, **kwargs): 

1011 super().__init__(**kwargs) 

1012 self.makeSubtask("assembleCcd") 

1013 self.makeSubtask("crosstalk") 

1014 self.makeSubtask("strayLight") 

1015 self.makeSubtask("fringe") 

1016 self.makeSubtask("masking") 

1017 self.makeSubtask("overscan") 

1018 self.makeSubtask("vignette") 

1019 self.makeSubtask("ampOffset") 

1020 self.makeSubtask("deferredChargeCorrection") 

1021 self.makeSubtask("isrStats") 

1022 

1023 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1024 inputs = butlerQC.get(inputRefs) 

1025 

1026 try: 

1027 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

1028 except Exception as e: 

1029 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

1030 (inputRefs, e)) 

1031 

1032 detector = inputs['ccdExposure'].getDetector() 

1033 

1034 # This is use for header provenance. 

1035 additionalInputDates = {} 

1036 

1037 if self.config.doCrosstalk is True: 

1038 # Crosstalk sources need to be defined by the pipeline 

1039 # yaml if they exist. 

1040 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

1041 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

1042 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

1043 else: 

1044 coeffVector = (self.config.crosstalk.crosstalkValues 

1045 if self.config.crosstalk.useConfigCoefficients else None) 

1046 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1047 inputs['crosstalk'] = crosstalkCalib 

1048 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1049 if 'crosstalkSources' not in inputs: 

1050 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1051 

1052 if self.doLinearize(detector) is True: 

1053 if 'linearizer' in inputs: 

1054 if isinstance(inputs['linearizer'], dict): 

1055 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1056 linearizer.fromYaml(inputs['linearizer']) 

1057 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1058 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1059 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1060 detector=detector, 

1061 log=self.log) 

1062 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1063 else: 

1064 linearizer = inputs['linearizer'] 

1065 self.log.info("Loading linearizer from the Butler.") 

1066 linearizer.log = self.log 

1067 inputs['linearizer'] = linearizer 

1068 else: 

1069 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1070 self.log.info("Constructing linearizer from cameraGeom information.") 

1071 

1072 if self.config.doDefect is True: 

1073 if "defects" in inputs and inputs['defects'] is not None: 

1074 # defects is loaded as a BaseCatalog with columns 

1075 # x0, y0, width, height. Masking expects a list of defects 

1076 # defined by their bounding box 

1077 if not isinstance(inputs["defects"], Defects): 

1078 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1079 

1080 # Load the correct style of brighter-fatter kernel, and repack 

1081 # the information as a numpy array. 

1082 brighterFatterSource = None 

1083 if self.config.doBrighterFatter: 

1084 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1085 if brighterFatterKernel is None: 

1086 # This type of kernel must be in (y, x) index 

1087 # ordering, as it used directly as the .array 

1088 # component of the afwImage kernel. 

1089 brighterFatterKernel = inputs.get('bfKernel', None) 

1090 brighterFatterSource = 'bfKernel' 

1091 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1092 

1093 if brighterFatterKernel is None: 

1094 # This was requested by the config, but none were found. 

1095 raise RuntimeError("No brighter-fatter kernel was supplied.") 

1096 elif not isinstance(brighterFatterKernel, numpy.ndarray): 

1097 # This is a ISR calib kernel. These kernels are 

1098 # generated in (x, y) index ordering, and need to be 

1099 # transposed to be used directly as the .array 

1100 # component of the afwImage kernel. This is done 

1101 # explicitly below when setting the ``bfKernel`` 

1102 # input. 

1103 brighterFatterSource = 'newBFKernel' 

1104 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1105 

1106 detName = detector.getName() 

1107 level = brighterFatterKernel.level 

1108 

1109 # This is expected to be a dictionary of amp-wise gains. 

1110 inputs['bfGains'] = brighterFatterKernel.gain 

1111 if self.config.brighterFatterLevel == 'DETECTOR': 

1112 kernel = None 

1113 if level == 'DETECTOR': 

1114 if detName in brighterFatterKernel.detKernels: 

1115 kernel = brighterFatterKernel.detKernels[detName] 

1116 else: 

1117 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1118 elif level == 'AMP': 

1119 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1120 "fatter kernels.") 

1121 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1122 kernel = brighterFatterKernel.detKernels[detName] 

1123 if kernel is None: 

1124 raise RuntimeError("Could not identify brighter-fatter kernel!") 

1125 # Do the one single transpose here so the kernel 

1126 # can be directly loaded into the afwImage .array 

1127 # component. 

1128 inputs['bfKernel'] = numpy.transpose(kernel) 

1129 elif self.config.brighterFatterLevel == 'AMP': 

1130 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1131 

1132 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1133 expId = inputs['ccdExposure'].info.id 

1134 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1135 expId=expId, 

1136 assembler=self.assembleCcd 

1137 if self.config.doAssembleIsrExposures else None) 

1138 else: 

1139 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1140 

1141 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1142 if 'strayLightData' not in inputs: 

1143 inputs['strayLightData'] = None 

1144 

1145 if self.config.doHeaderProvenance: 

1146 # Add calibration provenanace info to header. 

1147 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1148 

1149 # These inputs change name during this step. These should 

1150 # have matching entries in the additionalInputDates dict. 

1151 additionalInputs = [] 

1152 if self.config.doBrighterFatter: 

1153 additionalInputs.append(brighterFatterSource) 

1154 

1155 for inputName in sorted(list(inputs.keys()) + additionalInputs): 

1156 reference = getattr(inputRefs, inputName, None) 

1157 if reference is not None and hasattr(reference, "run"): 

1158 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1159 runValue = reference.run 

1160 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1161 idValue = str(reference.id) 

1162 dateKey = f"LSST CALIB DATE {inputName.upper()}" 

1163 

1164 if inputName in additionalInputDates: 

1165 dateValue = additionalInputDates[inputName] 

1166 else: 

1167 dateValue = self.extractCalibDate(inputs[inputName]) 

1168 

1169 exposureMetadata[runKey] = runValue 

1170 exposureMetadata[idKey] = idValue 

1171 exposureMetadata[dateKey] = dateValue 

1172 

1173 outputs = self.run(**inputs) 

1174 butlerQC.put(outputs, outputRefs) 

1175 

1176 @timeMethod 

1177 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1178 crosstalk=None, crosstalkSources=None, 

1179 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1180 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1181 sensorTransmission=None, atmosphereTransmission=None, 

1182 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1183 deferredChargeCalib=None, 

1184 ): 

1185 """Perform instrument signature removal on an exposure. 

1186 

1187 Steps included in the ISR processing, in order performed, are: 

1188 

1189 - saturation and suspect pixel masking 

1190 - overscan subtraction 

1191 - CCD assembly of individual amplifiers 

1192 - bias subtraction 

1193 - variance image construction 

1194 - linearization of non-linear response 

1195 - crosstalk masking 

1196 - brighter-fatter correction 

1197 - dark subtraction 

1198 - fringe correction 

1199 - stray light subtraction 

1200 - flat correction 

1201 - masking of known defects and camera specific features 

1202 - vignette calculation 

1203 - appending transmission curve and distortion model 

1204 

1205 Parameters 

1206 ---------- 

1207 ccdExposure : `lsst.afw.image.Exposure` 

1208 The raw exposure that is to be run through ISR. The 

1209 exposure is modified by this method. 

1210 camera : `lsst.afw.cameraGeom.Camera`, optional 

1211 The camera geometry for this exposure. Required if 

1212 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1213 ``flat`` does not have an associated detector. 

1214 bias : `lsst.afw.image.Exposure`, optional 

1215 Bias calibration frame. 

1216 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1217 Functor for linearization. 

1218 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1219 Calibration for crosstalk. 

1220 crosstalkSources : `list`, optional 

1221 List of possible crosstalk sources. 

1222 dark : `lsst.afw.image.Exposure`, optional 

1223 Dark calibration frame. 

1224 flat : `lsst.afw.image.Exposure`, optional 

1225 Flat calibration frame. 

1226 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1227 Photon transfer curve dataset, with, e.g., gains 

1228 and read noise. 

1229 bfKernel : `numpy.ndarray`, optional 

1230 Brighter-fatter kernel. 

1231 bfGains : `dict` of `float`, optional 

1232 Gains used to override the detector's nominal gains for the 

1233 brighter-fatter correction. A dict keyed by amplifier name for 

1234 the detector in question. 

1235 defects : `lsst.ip.isr.Defects`, optional 

1236 List of defects. 

1237 fringes : `lsst.pipe.base.Struct`, optional 

1238 Struct containing the fringe correction data, with 

1239 elements: 

1240 

1241 ``fringes`` 

1242 fringe calibration frame (`lsst.afw.image.Exposure`) 

1243 ``seed`` 

1244 random seed derived from the ``ccdExposureId`` for random 

1245 number generator (`numpy.uint32`) 

1246 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1247 A ``TransmissionCurve`` that represents the throughput of the, 

1248 optics, to be evaluated in focal-plane coordinates. 

1249 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1250 A ``TransmissionCurve`` that represents the throughput of the 

1251 filter itself, to be evaluated in focal-plane coordinates. 

1252 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1253 A ``TransmissionCurve`` that represents the throughput of the 

1254 sensor itself, to be evaluated in post-assembly trimmed detector 

1255 coordinates. 

1256 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1257 A ``TransmissionCurve`` that represents the throughput of the 

1258 atmosphere, assumed to be spatially constant. 

1259 detectorNum : `int`, optional 

1260 The integer number for the detector to process. 

1261 strayLightData : `object`, optional 

1262 Opaque object containing calibration information for stray-light 

1263 correction. If `None`, no correction will be performed. 

1264 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1265 Illumination correction image. 

1266 

1267 Returns 

1268 ------- 

1269 result : `lsst.pipe.base.Struct` 

1270 Result struct with component: 

1271 

1272 ``exposure`` 

1273 The fully ISR corrected exposure. 

1274 (`lsst.afw.image.Exposure`) 

1275 ``outputExposure`` 

1276 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1277 ``ossThumb`` 

1278 Thumbnail image of the exposure after overscan subtraction. 

1279 (`numpy.ndarray`) 

1280 ``flattenedThumb`` 

1281 Thumbnail image of the exposure after flat-field correction. 

1282 (`numpy.ndarray`) 

1283 ``outputStatistics`` 

1284 Values of the additional statistics calculated. 

1285 

1286 Raises 

1287 ------ 

1288 RuntimeError 

1289 Raised if a configuration option is set to `True`, but the 

1290 required calibration data has not been specified. 

1291 

1292 Notes 

1293 ----- 

1294 The current processed exposure can be viewed by setting the 

1295 appropriate `lsstDebug` entries in the ``debug.display`` 

1296 dictionary. The names of these entries correspond to some of 

1297 the `IsrTaskConfig` Boolean options, with the value denoting the 

1298 frame to use. The exposure is shown inside the matching 

1299 option check and after the processing of that step has 

1300 finished. The steps with debug points are: 

1301 

1302 * doAssembleCcd 

1303 * doBias 

1304 * doCrosstalk 

1305 * doBrighterFatter 

1306 * doDark 

1307 * doFringe 

1308 * doStrayLight 

1309 * doFlat 

1310 

1311 In addition, setting the ``postISRCCD`` entry displays the 

1312 exposure after all ISR processing has finished. 

1313 """ 

1314 

1315 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1316 bias = self.ensureExposure(bias, camera, detectorNum) 

1317 dark = self.ensureExposure(dark, camera, detectorNum) 

1318 flat = self.ensureExposure(flat, camera, detectorNum) 

1319 

1320 ccd = ccdExposure.getDetector() 

1321 filterLabel = ccdExposure.getFilter() 

1322 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1323 

1324 if not ccd: 

1325 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1326 ccd = [FakeAmp(ccdExposure, self.config)] 

1327 

1328 # Validate Input 

1329 if self.config.doBias and bias is None: 

1330 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1331 if self.doLinearize(ccd) and linearizer is None: 

1332 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1333 if self.config.doBrighterFatter and bfKernel is None: 

1334 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1335 if self.config.doDark and dark is None: 

1336 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1337 if self.config.doFlat and flat is None: 

1338 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1339 if self.config.doDefect and defects is None: 

1340 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1341 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1342 and fringes.fringes is None): 

1343 # The `fringes` object needs to be a pipeBase.Struct, as 

1344 # we use it as a `dict` for the parameters of 

1345 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1346 # not be `None` if `doFringe=True`. Otherwise, raise. 

1347 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1348 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1349 and illumMaskedImage is None): 

1350 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1351 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1352 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1353 if (self.config.usePtcGains and ptc is None): 

1354 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

1355 if (self.config.usePtcReadNoise and ptc is None): 

1356 raise RuntimeError("No ptcDataset provided to use PTC read noise.") 

1357 

1358 # Validate that the inputs match the exposure configuration. 

1359 exposureMetadata = ccdExposure.getMetadata() 

1360 if self.config.doBias: 

1361 self.compareCameraKeywords(exposureMetadata, bias, "bias") 

1362 if self.config.doBrighterFatter: 

1363 self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter") 

1364 if self.config.doCrosstalk: 

1365 self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk") 

1366 if self.config.doDark: 

1367 self.compareCameraKeywords(exposureMetadata, dark, "dark") 

1368 if self.config.doDefect: 

1369 self.compareCameraKeywords(exposureMetadata, defects, "defects") 

1370 if self.config.doDeferredCharge: 

1371 self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI") 

1372 if self.config.doFlat: 

1373 self.compareCameraKeywords(exposureMetadata, flat, "flat") 

1374 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1375 self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe") 

1376 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1377 self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination") 

1378 if self.doLinearize(ccd): 

1379 self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer") 

1380 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1381 self.compareCameraKeywords(exposureMetadata, ptc, "PTC") 

1382 if self.config.doStrayLight: 

1383 self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight") 

1384 

1385 # Start in ADU. Update units to electrons when gain is applied: 

1386 # updateVariance, applyGains 

1387 # Check if needed during/after BFE correction, CTI correction. 

1388 exposureMetadata["LSST ISR UNITS"] = "ADU" 

1389 

1390 # Begin ISR processing. 

1391 if self.config.doConvertIntToFloat: 

1392 self.log.info("Converting exposure to floating point values.") 

1393 ccdExposure = self.convertIntToFloat(ccdExposure) 

1394 

1395 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1396 self.log.info("Applying bias correction.") 

1397 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1398 trimToFit=self.config.doTrimToMatchCalib) 

1399 self.debugView(ccdExposure, "doBias") 

1400 

1401 # Amplifier level processing. 

1402 overscans = [] 

1403 

1404 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1405 # This will attempt to mask bleed pixels across all amplifiers. 

1406 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1407 

1408 for amp in ccd: 

1409 # if ccdExposure is one amp, 

1410 # check for coverage to prevent performing ops multiple times 

1411 if ccdExposure.getBBox().contains(amp.getBBox()): 

1412 # Check for fully masked bad amplifiers, 

1413 # and generate masks for SUSPECT and SATURATED values. 

1414 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1415 

1416 if self.config.doOverscan and not badAmp: 

1417 # Overscan correction on amp-by-amp basis. 

1418 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1419 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1420 if overscanResults is not None and \ 

1421 self.config.qa is not None and self.config.qa.saveStats is True: 

1422 if isinstance(overscanResults.overscanMean, float): 

1423 # Only serial overscan was run 

1424 mean = overscanResults.overscanMean 

1425 sigma = overscanResults.overscanSigma 

1426 residMean = overscanResults.residualMean 

1427 residSigma = overscanResults.residualSigma 

1428 else: 

1429 # Both serial and parallel overscan were 

1430 # run. Only report serial here. 

1431 mean = overscanResults.overscanMean[0] 

1432 sigma = overscanResults.overscanSigma[0] 

1433 residMean = overscanResults.residualMean[0] 

1434 residSigma = overscanResults.residualSigma[0] 

1435 

1436 self.metadata[f"FIT MEDIAN {amp.getName()}"] = mean 

1437 self.metadata[f"FIT STDEV {amp.getName()}"] = sigma 

1438 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1439 amp.getName(), mean, sigma) 

1440 

1441 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = residMean 

1442 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = residSigma 

1443 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1444 amp.getName(), residMean, residSigma) 

1445 

1446 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1447 else: 

1448 if badAmp: 

1449 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1450 overscanResults = None 

1451 

1452 overscans.append(overscanResults if overscanResults is not None else None) 

1453 else: 

1454 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1455 

1456 # Define an effective PTC that will contain the gain and readout 

1457 # noise to be used throughout the ISR task. 

1458 ptc = self.defineEffectivePtc(ptc, ccd, bfGains, overscans, exposureMetadata) 

1459 

1460 if self.config.doDeferredCharge: 

1461 self.log.info("Applying deferred charge/CTI correction.") 

1462 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1463 self.debugView(ccdExposure, "doDeferredCharge") 

1464 

1465 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1466 self.log.info("Applying crosstalk correction.") 

1467 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1468 crosstalkSources=crosstalkSources, camera=camera) 

1469 self.debugView(ccdExposure, "doCrosstalk") 

1470 

1471 if self.config.doAssembleCcd: 

1472 self.log.info("Assembling CCD from amplifiers.") 

1473 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1474 

1475 if self.config.expectWcs and not ccdExposure.getWcs(): 

1476 self.log.warning("No WCS found in input exposure.") 

1477 self.debugView(ccdExposure, "doAssembleCcd") 

1478 

1479 ossThumb = None 

1480 if self.config.qa.doThumbnailOss: 

1481 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1482 

1483 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1484 self.log.info("Applying bias correction.") 

1485 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1486 trimToFit=self.config.doTrimToMatchCalib) 

1487 self.debugView(ccdExposure, "doBias") 

1488 

1489 if self.config.doVariance: 

1490 for amp in ccd: 

1491 if ccdExposure.getBBox().contains(amp.getBBox()): 

1492 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1493 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1494 self.updateVariance(ampExposure, amp, ptc) 

1495 

1496 if self.config.qa is not None and self.config.qa.saveStats is True: 

1497 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1498 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1499 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1500 qaStats.getValue(afwMath.MEDIAN) 

1501 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1502 qaStats.getValue(afwMath.STDEVCLIP) 

1503 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1504 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1505 qaStats.getValue(afwMath.STDEVCLIP)) 

1506 if self.config.maskNegativeVariance: 

1507 self.maskNegativeVariance(ccdExposure) 

1508 

1509 if self.doLinearize(ccd): 

1510 self.log.info("Applying linearizer.") 

1511 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1512 detector=ccd, log=self.log) 

1513 

1514 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1515 self.log.info("Applying crosstalk correction.") 

1516 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1517 crosstalkSources=crosstalkSources, isTrimmed=True) 

1518 self.debugView(ccdExposure, "doCrosstalk") 

1519 

1520 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1521 # widen trails, and do anything else the camera needs. Saturated and 

1522 # suspect pixels have already been masked. 

1523 if self.config.doDefect: 

1524 self.log.info("Masking defects.") 

1525 self.maskDefect(ccdExposure, defects) 

1526 

1527 if self.config.numEdgeSuspect > 0: 

1528 self.log.info("Masking edges as SUSPECT.") 

1529 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1530 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1531 

1532 if self.config.doNanMasking: 

1533 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1534 self.maskNan(ccdExposure) 

1535 

1536 if self.config.doWidenSaturationTrails: 

1537 self.log.info("Widening saturation trails.") 

1538 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1539 

1540 if self.config.doCameraSpecificMasking: 

1541 self.log.info("Masking regions for camera specific reasons.") 

1542 self.masking.run(ccdExposure) 

1543 

1544 if self.config.doBrighterFatter: 

1545 # We need to apply flats and darks before we can interpolate, and 

1546 # we need to interpolate before we do B-F, but we do B-F without 

1547 # the flats and darks applied so we can work in units of electrons 

1548 # or holes. This context manager applies and then removes the darks 

1549 # and flats. 

1550 # 

1551 # We also do not want to interpolate values here, so operate on 

1552 # temporary images so we can apply only the BF-correction and roll 

1553 # back the interpolation. 

1554 interpExp = ccdExposure.clone() 

1555 with self.flatContext(interpExp, flat, dark): 

1556 isrFunctions.interpolateFromMask( 

1557 maskedImage=interpExp.getMaskedImage(), 

1558 fwhm=self.config.fwhm, 

1559 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1560 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1561 ) 

1562 bfExp = interpExp.clone() 

1563 

1564 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1565 type(bfKernel), type(bfGains)) 

1566 if self.config.doFluxConservingBrighterFatterCorrection: 

1567 bfResults = isrFunctions.fluxConservingBrighterFatterCorrection( 

1568 bfExp, 

1569 bfKernel, 

1570 self.config.brighterFatterMaxIter, 

1571 self.config.brighterFatterThreshold, 

1572 self.config.brighterFatterApplyGain, 

1573 bfGains 

1574 ) 

1575 else: 

1576 bfResults = isrFunctions.brighterFatterCorrection( 

1577 bfExp, 

1578 bfKernel, 

1579 self.config.brighterFatterMaxIter, 

1580 self.config.brighterFatterThreshold, 

1581 self.config.brighterFatterApplyGain, 

1582 bfGains 

1583 ) 

1584 if bfResults[1] == self.config.brighterFatterMaxIter - 1: 

1585 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1586 bfResults[0]) 

1587 else: 

1588 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1589 bfResults[1]) 

1590 image = ccdExposure.getMaskedImage().getImage() 

1591 bfCorr = bfExp.getMaskedImage().getImage() 

1592 bfCorr -= interpExp.getMaskedImage().getImage() 

1593 image += bfCorr 

1594 

1595 # Applying the brighter-fatter correction applies a 

1596 # convolution to the science image. At the edges this 

1597 # convolution may not have sufficient valid pixels to 

1598 # produce a valid correction. Mark pixels within the size 

1599 # of the brighter-fatter kernel as EDGE to warn of this 

1600 # fact. 

1601 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1602 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1603 maskPlane="EDGE") 

1604 

1605 if self.config.brighterFatterMaskGrowSize > 0: 

1606 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1607 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1608 isrFunctions.growMasks(ccdExposure.getMask(), 

1609 radius=self.config.brighterFatterMaskGrowSize, 

1610 maskNameList=maskPlane, 

1611 maskValue=maskPlane) 

1612 

1613 self.debugView(ccdExposure, "doBrighterFatter") 

1614 

1615 if self.config.doDark: 

1616 self.log.info("Applying dark correction.") 

1617 self.darkCorrection(ccdExposure, dark) 

1618 self.debugView(ccdExposure, "doDark") 

1619 

1620 if self.config.doFringe and not self.config.fringeAfterFlat: 

1621 self.log.info("Applying fringe correction before flat.") 

1622 self.fringe.run(ccdExposure, **fringes.getDict()) 

1623 self.debugView(ccdExposure, "doFringe") 

1624 

1625 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1626 self.log.info("Checking strayLight correction.") 

1627 self.strayLight.run(ccdExposure, strayLightData) 

1628 self.debugView(ccdExposure, "doStrayLight") 

1629 

1630 if self.config.doFlat: 

1631 self.log.info("Applying flat correction.") 

1632 self.flatCorrection(ccdExposure, flat) 

1633 self.debugView(ccdExposure, "doFlat") 

1634 

1635 if self.config.doApplyGains: 

1636 self.log.info("Applying gain correction instead of flat.") 

1637 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1638 ptcGains=ptc.gain) 

1639 exposureMetadata["LSST ISR UNITS"] = "electrons" 

1640 

1641 if self.config.doFringe and self.config.fringeAfterFlat: 

1642 self.log.info("Applying fringe correction after flat.") 

1643 self.fringe.run(ccdExposure, **fringes.getDict()) 

1644 

1645 if self.config.doVignette: 

1646 if self.config.doMaskVignettePolygon: 

1647 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1648 else: 

1649 self.log.info("Constructing and attaching vignette polygon.") 

1650 self.vignettePolygon = self.vignette.run( 

1651 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1652 vignetteValue=self.config.vignetteValue, log=self.log) 

1653 

1654 if self.config.doAttachTransmissionCurve: 

1655 self.log.info("Adding transmission curves.") 

1656 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1657 filterTransmission=filterTransmission, 

1658 sensorTransmission=sensorTransmission, 

1659 atmosphereTransmission=atmosphereTransmission) 

1660 

1661 flattenedThumb = None 

1662 if self.config.qa.doThumbnailFlattened: 

1663 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1664 

1665 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1666 self.log.info("Performing illumination correction.") 

1667 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1668 illumMaskedImage, illumScale=self.config.illumScale, 

1669 trimToFit=self.config.doTrimToMatchCalib) 

1670 

1671 preInterpExp = None 

1672 if self.config.doSaveInterpPixels: 

1673 preInterpExp = ccdExposure.clone() 

1674 

1675 # Reset and interpolate bad pixels. 

1676 # 

1677 # Large contiguous bad regions (which should have the BAD mask 

1678 # bit set) should have their values set to the image median. 

1679 # This group should include defects and bad amplifiers. As the 

1680 # area covered by these defects are large, there's little 

1681 # reason to expect that interpolation would provide a more 

1682 # useful value. 

1683 # 

1684 # Smaller defects can be safely interpolated after the larger 

1685 # regions have had their pixel values reset. This ensures 

1686 # that the remaining defects adjacent to bad amplifiers (as an 

1687 # example) do not attempt to interpolate extreme values. 

1688 if self.config.doSetBadRegions: 

1689 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1690 if badPixelCount > 0: 

1691 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1692 

1693 if self.config.doInterpolate: 

1694 self.log.info("Interpolating masked pixels.") 

1695 isrFunctions.interpolateFromMask( 

1696 maskedImage=ccdExposure.getMaskedImage(), 

1697 fwhm=self.config.fwhm, 

1698 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1699 maskNameList=list(self.config.maskListToInterpolate) 

1700 ) 

1701 

1702 self.roughZeroPoint(ccdExposure) 

1703 

1704 # correct for amp offsets within the CCD 

1705 if self.config.doAmpOffset: 

1706 self.log.info("Correcting amp offsets.") 

1707 self.ampOffset.run(ccdExposure) 

1708 

1709 if self.config.doMeasureBackground: 

1710 self.log.info("Measuring background level.") 

1711 self.measureBackground(ccdExposure, self.config.qa) 

1712 

1713 if self.config.qa is not None and self.config.qa.saveStats is True: 

1714 for amp in ccd: 

1715 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1716 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1717 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1718 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1719 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1720 qaStats.getValue(afwMath.STDEVCLIP) 

1721 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1722 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1723 qaStats.getValue(afwMath.STDEVCLIP)) 

1724 

1725 # Calculate standard image quality statistics 

1726 if self.config.doStandardStatistics: 

1727 metadata = ccdExposure.getMetadata() 

1728 for amp in ccd: 

1729 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1730 ampName = amp.getName() 

1731 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1732 ampExposure.getMaskedImage(), 

1733 [self.config.saturatedMaskName] 

1734 ) 

1735 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1736 ampExposure.getMaskedImage(), 

1737 ["BAD"] 

1738 ) 

1739 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1740 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1741 

1742 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1743 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1744 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1745 

1746 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1747 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1748 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1749 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1750 else: 

1751 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1752 

1753 # calculate additional statistics. 

1754 outputStatistics = None 

1755 if self.config.doCalculateStatistics: 

1756 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1757 bias=bias, dark=dark, flat=flat, ptc=ptc).results 

1758 

1759 # do any binning. 

1760 outputBin1Exposure = None 

1761 outputBin2Exposure = None 

1762 if self.config.doBinnedExposures: 

1763 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure) 

1764 

1765 self.debugView(ccdExposure, "postISRCCD") 

1766 

1767 return pipeBase.Struct( 

1768 exposure=ccdExposure, 

1769 ossThumb=ossThumb, 

1770 flattenedThumb=flattenedThumb, 

1771 

1772 outputBin1Exposure=outputBin1Exposure, 

1773 outputBin2Exposure=outputBin2Exposure, 

1774 

1775 preInterpExposure=preInterpExp, 

1776 outputExposure=ccdExposure, 

1777 outputOssThumbnail=ossThumb, 

1778 outputFlattenedThumbnail=flattenedThumb, 

1779 outputStatistics=outputStatistics, 

1780 ) 

1781 

1782 def defineEffectivePtc(self, ptcDataset, detector, bfGains, overScans, metadata): 

1783 """Define an effective Photon Transfer Curve dataset 

1784 with nominal gains and noise. 

1785 

1786 Parameters 

1787 ---------- 

1788 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset` 

1789 Input Photon Transfer Curve dataset. 

1790 detector : `lsst.afw.cameraGeom.Detector` 

1791 Detector object. 

1792 bfGains : `dict` 

1793 Gains from running the brighter-fatter code. 

1794 A dict keyed by amplifier name for the detector 

1795 in question. 

1796 ovserScans : `list` [`lsst.pipe.base.Struct`] 

1797 List of overscanResults structures 

1798 metadata : `lsst.daf.base.PropertyList` 

1799 Exposure metadata to update gain and noise provenance. 

1800 

1801 Returns 

1802 ------- 

1803 effectivePtc : `lsst.ip.isr.PhotonTransferCurveDataset` 

1804 PTC dataset containing gains and readout noise 

1805 values to be used throughout 

1806 Instrument Signature Removal. 

1807 """ 

1808 amps = detector.getAmplifiers() 

1809 ampNames = [amp.getName() for amp in amps] 

1810 detName = detector.getName() 

1811 effectivePtc = PhotonTransferCurveDataset(ampNames, 'EFFECTIVE_PTC', 1) 

1812 boolGainMismatch = False 

1813 doWarningPtcValidation = True 

1814 

1815 for amp, overscanResults in zip(amps, overScans): 

1816 ampName = amp.getName() 

1817 # Gain: 

1818 # Try first with the PTC gains. 

1819 gainProvenanceString = "amp" 

1820 if self.config.usePtcGains: 

1821 gain = ptcDataset.gain[ampName] 

1822 gainProvenanceString = "ptc" 

1823 self.log.debug("Using gain from Photon Transfer Curve.") 

1824 else: 

1825 # Try then with the amplifier gain. 

1826 # We already have a detector at this point. If there was no 

1827 # detector to begin with, one would have been created with 

1828 # self.config.gain and self.config.noise. Same comment 

1829 # applies for the noise block below. 

1830 gain = amp.getGain() 

1831 

1832 # Check if the gain up to this point differs from the 

1833 # gain in bfGains. If so, raise or warn, accordingly. 

1834 if not boolGainMismatch and bfGains is not None and ampName in bfGains: 

1835 bfGain = bfGains[ampName] 

1836 if not math.isclose(gain, bfGain, rel_tol=1e-4): 

1837 if self.config.doRaiseOnCalibMismatch: 

1838 raise RuntimeError("Gain mismatch for det %s amp %s: " 

1839 "(gain (%s): %s, bfGain: %s)", 

1840 detName, ampName, gainProvenanceString, 

1841 gain, bfGain) 

1842 else: 

1843 self.log.warning("Gain mismatch for det %s amp %s: " 

1844 "(gain (%s): %s, bfGain: %s)", 

1845 detName, ampName, gainProvenanceString, 

1846 gain, bfGain) 

1847 boolGainMismatch = True 

1848 

1849 # Noise: 

1850 # Try first with the empirical noise from the overscan. 

1851 noiseProvenanceString = "amp" 

1852 if self.config.doEmpiricalReadNoise and overscanResults is not None: 

1853 noiseProvenanceString = "serial overscan" 

1854 if isinstance(overscanResults.residualSigma, float): 

1855 # Only serial overscan was run 

1856 noise = overscanResults.residualSigma 

1857 else: 

1858 # Both serial and parallel overscan were 

1859 # run. Only report noise from serial here. 

1860 noise = overscanResults.residualSigma[0] 

1861 elif self.config.usePtcReadNoise: 

1862 # Try then with the PTC noise. 

1863 noise = ptcDataset.noise[amp.getName()] 

1864 noiseProvenanceString = "ptc" 

1865 self.log.debug("Using noise from Photon Transfer Curve.") 

1866 else: 

1867 # Finally, try with the amplifier noise. 

1868 # We already have a detector at this point. If there 

1869 # was no detector to begin with, one would have 

1870 # been created with self.config.gain and 

1871 # self.config.noise. 

1872 noise = amp.getReadNoise() 

1873 

1874 if math.isnan(gain): 

1875 gain = 1.0 

1876 self.log.warning("Gain for amp %s set to NAN! Updating to" 

1877 " 1.0 to generate Poisson variance.", ampName) 

1878 elif gain <= 0: 

1879 patchedGain = 1.0 

1880 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

1881 ampName, gain, patchedGain) 

1882 gain = patchedGain 

1883 

1884 effectivePtc.gain[ampName] = gain 

1885 effectivePtc.noise[ampName] = noise 

1886 # Make sure noise,turnoff, and gain make sense 

1887 effectivePtc.validateGainNoiseTurnoffValues(ampName, doWarn=doWarningPtcValidation) 

1888 doWarningPtcValidation = False 

1889 

1890 metadata[f"LSST GAIN {amp.getName()}"] = effectivePtc.gain[ampName] 

1891 metadata[f"LSST READNOISE {amp.getName()}"] = effectivePtc.noise[ampName] 

1892 

1893 self.log.info("Det: %s - Noise provenance: %s, Gain provenance: %s", 

1894 detName, 

1895 noiseProvenanceString, 

1896 gainProvenanceString) 

1897 metadata["LSST ISR GAIN SOURCE"] = gainProvenanceString 

1898 metadata["LSST ISR NOISE SOURCE"] = noiseProvenanceString 

1899 

1900 return effectivePtc 

1901 

1902 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1903 """Ensure that the data returned by Butler is a fully constructed exp. 

1904 

1905 ISR requires exposure-level image data for historical reasons, so if we 

1906 did not recieve that from Butler, construct it from what we have, 

1907 modifying the input in place. 

1908 

1909 Parameters 

1910 ---------- 

1911 inputExp : `lsst.afw.image` image-type. 

1912 The input data structure obtained from Butler. 

1913 Can be `lsst.afw.image.Exposure`, 

1914 `lsst.afw.image.DecoratedImageU`, 

1915 or `lsst.afw.image.ImageF` 

1916 camera : `lsst.afw.cameraGeom.camera`, optional 

1917 The camera associated with the image. Used to find the appropriate 

1918 detector if detector is not already set. 

1919 detectorNum : `int`, optional 

1920 The detector in the camera to attach, if the detector is not 

1921 already set. 

1922 

1923 Returns 

1924 ------- 

1925 inputExp : `lsst.afw.image.Exposure` 

1926 The re-constructed exposure, with appropriate detector parameters. 

1927 

1928 Raises 

1929 ------ 

1930 TypeError 

1931 Raised if the input data cannot be used to construct an exposure. 

1932 """ 

1933 if isinstance(inputExp, afwImage.DecoratedImageU): 

1934 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1935 elif isinstance(inputExp, afwImage.ImageF): 

1936 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1937 elif isinstance(inputExp, afwImage.MaskedImageF): 

1938 inputExp = afwImage.makeExposure(inputExp) 

1939 elif isinstance(inputExp, afwImage.Exposure): 

1940 pass 

1941 elif inputExp is None: 

1942 # Assume this will be caught by the setup if it is a problem. 

1943 return inputExp 

1944 else: 

1945 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1946 (type(inputExp), )) 

1947 

1948 if inputExp.getDetector() is None: 

1949 if camera is None or detectorNum is None: 

1950 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1951 'without a detector set.') 

1952 inputExp.setDetector(camera[detectorNum]) 

1953 

1954 return inputExp 

1955 

1956 @staticmethod 

1957 def extractCalibDate(calib): 

1958 """Extract common calibration metadata values that will be written to 

1959 output header. 

1960 

1961 Parameters 

1962 ---------- 

1963 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1964 Calibration to pull date information from. 

1965 

1966 Returns 

1967 ------- 

1968 dateString : `str` 

1969 Calibration creation date string to add to header. 

1970 """ 

1971 if hasattr(calib, "getMetadata"): 

1972 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1973 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1974 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1975 else: 

1976 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1977 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1978 else: 

1979 return "Unknown Unknown" 

1980 

1981 def compareCameraKeywords(self, exposureMetadata, calib, calibName): 

1982 """Compare header keywords to confirm camera states match. 

1983 

1984 Parameters 

1985 ---------- 

1986 exposureMetadata : `lsst.daf.base.PropertySet` 

1987 Header for the exposure being processed. 

1988 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1989 Calibration to be applied. 

1990 calibName : `str` 

1991 Calib type for log message. 

1992 """ 

1993 try: 

1994 calibMetadata = calib.getMetadata() 

1995 except AttributeError: 

1996 return 

1997 for keyword in self.config.cameraKeywordsToCompare: 

1998 if keyword in exposureMetadata and keyword in calibMetadata: 

1999 if exposureMetadata[keyword] != calibMetadata[keyword]: 

2000 if self.config.doRaiseOnCalibMismatch: 

2001 raise RuntimeError("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

2002 calibName, keyword, 

2003 exposureMetadata[keyword], calibMetadata[keyword]) 

2004 else: 

2005 self.log.warning("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

2006 calibName, keyword, 

2007 exposureMetadata[keyword], calibMetadata[keyword]) 

2008 else: 

2009 self.log.debug("Sequencer keyword %s not found.", keyword) 

2010 

2011 def convertIntToFloat(self, exposure): 

2012 """Convert exposure image from uint16 to float. 

2013 

2014 If the exposure does not need to be converted, the input is 

2015 immediately returned. For exposures that are converted to use 

2016 floating point pixels, the variance is set to unity and the 

2017 mask to zero. 

2018 

2019 Parameters 

2020 ---------- 

2021 exposure : `lsst.afw.image.Exposure` 

2022 The raw exposure to be converted. 

2023 

2024 Returns 

2025 ------- 

2026 newexposure : `lsst.afw.image.Exposure` 

2027 The input ``exposure``, converted to floating point pixels. 

2028 

2029 Raises 

2030 ------ 

2031 RuntimeError 

2032 Raised if the exposure type cannot be converted to float. 

2033 

2034 """ 

2035 if isinstance(exposure, afwImage.ExposureF): 

2036 # Nothing to be done 

2037 self.log.debug("Exposure already of type float.") 

2038 return exposure 

2039 if not hasattr(exposure, "convertF"): 

2040 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

2041 

2042 newexposure = exposure.convertF() 

2043 newexposure.variance[:] = 1 

2044 newexposure.mask[:] = 0x0 

2045 

2046 return newexposure 

2047 

2048 def maskAmplifier(self, ccdExposure, amp, defects): 

2049 """Identify bad amplifiers, saturated and suspect pixels. 

2050 

2051 Parameters 

2052 ---------- 

2053 ccdExposure : `lsst.afw.image.Exposure` 

2054 Input exposure to be masked. 

2055 amp : `lsst.afw.cameraGeom.Amplifier` 

2056 Catalog of parameters defining the amplifier on this 

2057 exposure to mask. 

2058 defects : `lsst.ip.isr.Defects` 

2059 List of defects. Used to determine if the entire 

2060 amplifier is bad. 

2061 

2062 Returns 

2063 ------- 

2064 badAmp : `Bool` 

2065 If this is true, the entire amplifier area is covered by 

2066 defects and unusable. 

2067 

2068 """ 

2069 maskedImage = ccdExposure.getMaskedImage() 

2070 

2071 badAmp = False 

2072 

2073 # Check if entire amp region is defined as a defect 

2074 # NB: need to use amp.getBBox() for correct comparison with current 

2075 # defects definition. 

2076 if defects is not None: 

2077 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

2078 

2079 # In the case of a bad amp, we will set mask to "BAD" 

2080 # (here use amp.getRawBBox() for correct association with pixels in 

2081 # current ccdExposure). 

2082 if badAmp: 

2083 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

2084 afwImage.PARENT) 

2085 maskView = dataView.getMask() 

2086 maskView |= maskView.getPlaneBitMask("BAD") 

2087 del maskView 

2088 return badAmp 

2089 

2090 # Mask remaining defects after assembleCcd() to allow for defects that 

2091 # cross amplifier boundaries. Saturation and suspect pixels can be 

2092 # masked now, though. 

2093 limits = dict() 

2094 if self.config.doSaturation and not badAmp: 

2095 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

2096 if self.config.doSuspect and not badAmp: 

2097 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

2098 if math.isfinite(self.config.saturation): 

2099 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

2100 

2101 for maskName, maskThreshold in limits.items(): 

2102 if not math.isnan(maskThreshold): 

2103 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2104 isrFunctions.makeThresholdMask( 

2105 maskedImage=dataView, 

2106 threshold=maskThreshold, 

2107 growFootprints=0, 

2108 maskName=maskName 

2109 ) 

2110 

2111 # Determine if we've fully masked this amplifier with SUSPECT and 

2112 # SAT pixels. 

2113 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

2114 afwImage.PARENT) 

2115 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

2116 self.config.suspectMaskName]) 

2117 if numpy.all(maskView.getArray() & maskVal > 0): 

2118 badAmp = True 

2119 maskView |= maskView.getPlaneBitMask("BAD") 

2120 

2121 return badAmp 

2122 

2123 def overscanCorrection(self, ccdExposure, amp): 

2124 """Apply overscan correction in place. 

2125 

2126 This method does initial pixel rejection of the overscan 

2127 region. The overscan can also be optionally segmented to 

2128 allow for discontinuous overscan responses to be fit 

2129 separately. The actual overscan subtraction is performed by 

2130 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

2131 after the amplifier is preprocessed. 

2132 

2133 Parameters 

2134 ---------- 

2135 ccdExposure : `lsst.afw.image.Exposure` 

2136 Exposure to have overscan correction performed. 

2137 amp : `lsst.afw.cameraGeom.Amplifer` 

2138 The amplifier to consider while correcting the overscan. 

2139 

2140 Returns 

2141 ------- 

2142 overscanResults : `lsst.pipe.base.Struct` 

2143 Result struct with components: 

2144 

2145 ``imageFit`` 

2146 Value or fit subtracted from the amplifier image data. 

2147 (scalar or `lsst.afw.image.Image`) 

2148 ``overscanFit`` 

2149 Value or fit subtracted from the overscan image data. 

2150 (scalar or `lsst.afw.image.Image`) 

2151 ``overscanImage`` 

2152 Image of the overscan region with the overscan 

2153 correction applied. This quantity is used to estimate 

2154 the amplifier read noise empirically. 

2155 (`lsst.afw.image.Image`) 

2156 ``edgeMask`` 

2157 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

2158 ``overscanMean`` 

2159 Median overscan fit value. (`float`) 

2160 ``overscanSigma`` 

2161 Clipped standard deviation of the overscan after 

2162 correction. (`float`) 

2163 

2164 Raises 

2165 ------ 

2166 RuntimeError 

2167 Raised if the ``amp`` does not contain raw pixel information. 

2168 

2169 See Also 

2170 -------- 

2171 lsst.ip.isr.overscan.OverscanTask 

2172 """ 

2173 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2174 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2175 return None 

2176 

2177 # Perform overscan correction on subregions. 

2178 overscanResults = self.overscan.run(ccdExposure, amp) 

2179 

2180 metadata = ccdExposure.getMetadata() 

2181 ampName = amp.getName() 

2182 

2183 keyBase = "LSST ISR OVERSCAN" 

2184 # Updated quantities 

2185 if isinstance(overscanResults.overscanMean, float): 

2186 # Serial overscan correction only: 

2187 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

2188 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

2189 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

2190 

2191 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

2192 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

2193 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

2194 elif isinstance(overscanResults.overscanMean, tuple): 

2195 # Both serial and parallel overscan have run: 

2196 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

2197 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

2198 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

2199 

2200 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

2201 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

2202 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

2203 

2204 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

2205 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

2206 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

2207 

2208 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

2209 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

2210 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

2211 else: 

2212 self.log.warning("Unexpected type for overscan values; none added to header.") 

2213 

2214 return overscanResults 

2215 

2216 def updateVariance(self, ampExposure, amp, ptcDataset): 

2217 """Set the variance plane using the gain and read noise 

2218 

2219 The read noise is calculated from the ``overscanImage`` if the 

2220 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2221 the value from the amplifier data is used. 

2222 

2223 Parameters 

2224 ---------- 

2225 ampExposure : `lsst.afw.image.Exposure` 

2226 Exposure to process. 

2227 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

2228 Amplifier detector data. 

2229 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset` 

2230 Effective PTC dataset containing the gains and read noise. 

2231 

2232 See also 

2233 -------- 

2234 lsst.ip.isr.isrFunctions.updateVariance 

2235 """ 

2236 ampName = amp.getName() 

2237 # At this point, the effective PTC should have 

2238 # gain and noise values. 

2239 gain = ptcDataset.gain[ampName] 

2240 readNoise = ptcDataset.noise[ampName] 

2241 

2242 isrFunctions.updateVariance( 

2243 maskedImage=ampExposure.getMaskedImage(), 

2244 gain=gain, 

2245 readNoise=readNoise, 

2246 ) 

2247 

2248 def maskNegativeVariance(self, exposure): 

2249 """Identify and mask pixels with negative variance values. 

2250 

2251 Parameters 

2252 ---------- 

2253 exposure : `lsst.afw.image.Exposure` 

2254 Exposure to process. 

2255 

2256 See Also 

2257 -------- 

2258 lsst.ip.isr.isrFunctions.updateVariance 

2259 """ 

2260 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2261 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2262 exposure.mask.array[bad] |= maskPlane 

2263 

2264 def darkCorrection(self, exposure, darkExposure, invert=False): 

2265 """Apply dark correction in place. 

2266 

2267 Parameters 

2268 ---------- 

2269 exposure : `lsst.afw.image.Exposure` 

2270 Exposure to process. 

2271 darkExposure : `lsst.afw.image.Exposure` 

2272 Dark exposure of the same size as ``exposure``. 

2273 invert : `Bool`, optional 

2274 If True, re-add the dark to an already corrected image. 

2275 

2276 Raises 

2277 ------ 

2278 RuntimeError 

2279 Raised if either ``exposure`` or ``darkExposure`` do not 

2280 have their dark time defined. 

2281 

2282 See Also 

2283 -------- 

2284 lsst.ip.isr.isrFunctions.darkCorrection 

2285 """ 

2286 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2287 if math.isnan(expScale): 

2288 raise RuntimeError("Exposure darktime is NAN.") 

2289 if darkExposure.getInfo().getVisitInfo() is not None \ 

2290 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2291 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2292 else: 

2293 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2294 # so getDarkTime() does not exist. 

2295 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2296 darkScale = 1.0 

2297 

2298 isrFunctions.darkCorrection( 

2299 maskedImage=exposure.getMaskedImage(), 

2300 darkMaskedImage=darkExposure.getMaskedImage(), 

2301 expScale=expScale, 

2302 darkScale=darkScale, 

2303 invert=invert, 

2304 trimToFit=self.config.doTrimToMatchCalib 

2305 ) 

2306 

2307 def doLinearize(self, detector): 

2308 """Check if linearization is needed for the detector cameraGeom. 

2309 

2310 Checks config.doLinearize and the linearity type of the first 

2311 amplifier. 

2312 

2313 Parameters 

2314 ---------- 

2315 detector : `lsst.afw.cameraGeom.Detector` 

2316 Detector to get linearity type from. 

2317 

2318 Returns 

2319 ------- 

2320 doLinearize : `Bool` 

2321 If True, linearization should be performed. 

2322 """ 

2323 return self.config.doLinearize and \ 

2324 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2325 

2326 def flatCorrection(self, exposure, flatExposure, invert=False): 

2327 """Apply flat correction in place. 

2328 

2329 Parameters 

2330 ---------- 

2331 exposure : `lsst.afw.image.Exposure` 

2332 Exposure to process. 

2333 flatExposure : `lsst.afw.image.Exposure` 

2334 Flat exposure of the same size as ``exposure``. 

2335 invert : `Bool`, optional 

2336 If True, unflatten an already flattened image. 

2337 

2338 See Also 

2339 -------- 

2340 lsst.ip.isr.isrFunctions.flatCorrection 

2341 """ 

2342 isrFunctions.flatCorrection( 

2343 maskedImage=exposure.getMaskedImage(), 

2344 flatMaskedImage=flatExposure.getMaskedImage(), 

2345 scalingType=self.config.flatScalingType, 

2346 userScale=self.config.flatUserScale, 

2347 invert=invert, 

2348 trimToFit=self.config.doTrimToMatchCalib 

2349 ) 

2350 

2351 def saturationDetection(self, exposure, amp): 

2352 """Detect and mask saturated pixels in config.saturatedMaskName. 

2353 

2354 Parameters 

2355 ---------- 

2356 exposure : `lsst.afw.image.Exposure` 

2357 Exposure to process. Only the amplifier DataSec is processed. 

2358 amp : `lsst.afw.cameraGeom.Amplifier` 

2359 Amplifier detector data. 

2360 

2361 See Also 

2362 -------- 

2363 lsst.ip.isr.isrFunctions.makeThresholdMask 

2364 """ 

2365 if not math.isnan(amp.getSaturation()): 

2366 maskedImage = exposure.getMaskedImage() 

2367 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2368 isrFunctions.makeThresholdMask( 

2369 maskedImage=dataView, 

2370 threshold=amp.getSaturation(), 

2371 growFootprints=0, 

2372 maskName=self.config.saturatedMaskName, 

2373 ) 

2374 

2375 def saturationInterpolation(self, exposure): 

2376 """Interpolate over saturated pixels, in place. 

2377 

2378 This method should be called after `saturationDetection`, to 

2379 ensure that the saturated pixels have been identified in the 

2380 SAT mask. It should also be called after `assembleCcd`, since 

2381 saturated regions may cross amplifier boundaries. 

2382 

2383 Parameters 

2384 ---------- 

2385 exposure : `lsst.afw.image.Exposure` 

2386 Exposure to process. 

2387 

2388 See Also 

2389 -------- 

2390 lsst.ip.isr.isrTask.saturationDetection 

2391 lsst.ip.isr.isrFunctions.interpolateFromMask 

2392 """ 

2393 isrFunctions.interpolateFromMask( 

2394 maskedImage=exposure.getMaskedImage(), 

2395 fwhm=self.config.fwhm, 

2396 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2397 maskNameList=list(self.config.saturatedMaskName), 

2398 ) 

2399 

2400 def suspectDetection(self, exposure, amp): 

2401 """Detect and mask suspect pixels in config.suspectMaskName. 

2402 

2403 Parameters 

2404 ---------- 

2405 exposure : `lsst.afw.image.Exposure` 

2406 Exposure to process. Only the amplifier DataSec is processed. 

2407 amp : `lsst.afw.cameraGeom.Amplifier` 

2408 Amplifier detector data. 

2409 

2410 See Also 

2411 -------- 

2412 lsst.ip.isr.isrFunctions.makeThresholdMask 

2413 

2414 Notes 

2415 ----- 

2416 Suspect pixels are pixels whose value is greater than 

2417 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2418 affected by unknown systematics; for example if non-linearity 

2419 corrections above a certain level are unstable then that would be a 

2420 useful value for suspectLevel. A value of `nan` indicates that no such 

2421 level exists and no pixels are to be masked as suspicious. 

2422 """ 

2423 suspectLevel = amp.getSuspectLevel() 

2424 if math.isnan(suspectLevel): 

2425 return 

2426 

2427 maskedImage = exposure.getMaskedImage() 

2428 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2429 isrFunctions.makeThresholdMask( 

2430 maskedImage=dataView, 

2431 threshold=suspectLevel, 

2432 growFootprints=0, 

2433 maskName=self.config.suspectMaskName, 

2434 ) 

2435 

2436 def maskDefect(self, exposure, defectBaseList): 

2437 """Mask defects using mask plane "BAD", in place. 

2438 

2439 Parameters 

2440 ---------- 

2441 exposure : `lsst.afw.image.Exposure` 

2442 Exposure to process. 

2443 defectBaseList : defect-type 

2444 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2445 or `list` of `lsst.afw.image.DefectBase`. 

2446 

2447 Notes 

2448 ----- 

2449 Call this after CCD assembly, since defects may cross amplifier 

2450 boundaries. 

2451 """ 

2452 maskedImage = exposure.getMaskedImage() 

2453 if not isinstance(defectBaseList, Defects): 

2454 # Promotes DefectBase to Defect 

2455 defectList = Defects(defectBaseList) 

2456 else: 

2457 defectList = defectBaseList 

2458 defectList.maskPixels(maskedImage, maskName="BAD") 

2459 

2460 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2461 """Mask edge pixels with applicable mask plane. 

2462 

2463 Parameters 

2464 ---------- 

2465 exposure : `lsst.afw.image.Exposure` 

2466 Exposure to process. 

2467 numEdgePixels : `int`, optional 

2468 Number of edge pixels to mask. 

2469 maskPlane : `str`, optional 

2470 Mask plane name to use. 

2471 level : `str`, optional 

2472 Level at which to mask edges. 

2473 """ 

2474 maskedImage = exposure.getMaskedImage() 

2475 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2476 

2477 if numEdgePixels > 0: 

2478 if level == 'DETECTOR': 

2479 boxes = [maskedImage.getBBox()] 

2480 elif level == 'AMP': 

2481 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2482 

2483 for box in boxes: 

2484 # This makes a bbox numEdgeSuspect pixels smaller than the 

2485 # image on each side 

2486 subImage = maskedImage[box] 

2487 box.grow(-numEdgePixels) 

2488 # Mask pixels outside box 

2489 SourceDetectionTask.setEdgeBits( 

2490 subImage, 

2491 box, 

2492 maskBitMask) 

2493 

2494 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2495 """Mask and interpolate defects using mask plane "BAD", in place. 

2496 

2497 Parameters 

2498 ---------- 

2499 exposure : `lsst.afw.image.Exposure` 

2500 Exposure to process. 

2501 defectBaseList : defects-like 

2502 List of defects to mask and interpolate. Can be 

2503 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2504 

2505 See Also 

2506 -------- 

2507 lsst.ip.isr.isrTask.maskDefect 

2508 """ 

2509 self.maskDefect(exposure, defectBaseList) 

2510 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2511 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2512 isrFunctions.interpolateFromMask( 

2513 maskedImage=exposure.getMaskedImage(), 

2514 fwhm=self.config.fwhm, 

2515 growSaturatedFootprints=0, 

2516 maskNameList=["BAD"], 

2517 ) 

2518 

2519 def maskNan(self, exposure): 

2520 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2521 

2522 Parameters 

2523 ---------- 

2524 exposure : `lsst.afw.image.Exposure` 

2525 Exposure to process. 

2526 

2527 Notes 

2528 ----- 

2529 We mask over all non-finite values (NaN, inf), including those 

2530 that are masked with other bits (because those may or may not be 

2531 interpolated over later, and we want to remove all NaN/infs). 

2532 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2533 preserve the historical name. 

2534 """ 

2535 maskedImage = exposure.getMaskedImage() 

2536 

2537 # Find and mask NaNs 

2538 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2539 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2540 numNans = maskNans(maskedImage, maskVal) 

2541 self.metadata["NUMNANS"] = numNans 

2542 if numNans > 0: 

2543 self.log.warning("There were %d unmasked NaNs.", numNans) 

2544 

2545 def maskAndInterpolateNan(self, exposure): 

2546 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2547 in place. 

2548 

2549 Parameters 

2550 ---------- 

2551 exposure : `lsst.afw.image.Exposure` 

2552 Exposure to process. 

2553 

2554 See Also 

2555 -------- 

2556 lsst.ip.isr.isrTask.maskNan 

2557 """ 

2558 self.maskNan(exposure) 

2559 isrFunctions.interpolateFromMask( 

2560 maskedImage=exposure.getMaskedImage(), 

2561 fwhm=self.config.fwhm, 

2562 growSaturatedFootprints=0, 

2563 maskNameList=["UNMASKEDNAN"], 

2564 ) 

2565 

2566 def measureBackground(self, exposure, IsrQaConfig=None): 

2567 """Measure the image background in subgrids, for quality control. 

2568 

2569 Parameters 

2570 ---------- 

2571 exposure : `lsst.afw.image.Exposure` 

2572 Exposure to process. 

2573 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2574 Configuration object containing parameters on which background 

2575 statistics and subgrids to use. 

2576 """ 

2577 if IsrQaConfig is not None: 

2578 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2579 IsrQaConfig.flatness.nIter) 

2580 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2581 statsControl.setAndMask(maskVal) 

2582 maskedImage = exposure.getMaskedImage() 

2583 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2584 skyLevel = stats.getValue(afwMath.MEDIAN) 

2585 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2586 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2587 metadata = exposure.getMetadata() 

2588 metadata["SKYLEVEL"] = skyLevel 

2589 metadata["SKYSIGMA"] = skySigma 

2590 

2591 # calcluating flatlevel over the subgrids 

2592 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2593 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2594 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2595 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2596 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2597 skyLevels = numpy.zeros((nX, nY)) 

2598 

2599 for j in range(nY): 

2600 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2601 for i in range(nX): 

2602 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2603 

2604 xLLC = xc - meshXHalf 

2605 yLLC = yc - meshYHalf 

2606 xURC = xc + meshXHalf - 1 

2607 yURC = yc + meshYHalf - 1 

2608 

2609 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2610 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2611 

2612 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2613 

2614 good = numpy.where(numpy.isfinite(skyLevels)) 

2615 skyMedian = numpy.median(skyLevels[good]) 

2616 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2617 flatness_rms = numpy.std(flatness) 

2618 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2619 

2620 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2621 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2622 nX, nY, flatness_pp, flatness_rms) 

2623 

2624 metadata["FLATNESS_PP"] = float(flatness_pp) 

2625 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2626 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2627 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2628 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2629 

2630 def roughZeroPoint(self, exposure): 

2631 """Set an approximate magnitude zero point for the exposure. 

2632 

2633 Parameters 

2634 ---------- 

2635 exposure : `lsst.afw.image.Exposure` 

2636 Exposure to process. 

2637 """ 

2638 filterLabel = exposure.getFilter() 

2639 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2640 

2641 if physicalFilter in self.config.fluxMag0T1: 

2642 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2643 else: 

2644 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2645 fluxMag0 = self.config.defaultFluxMag0T1 

2646 

2647 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2648 if not expTime > 0: # handle NaN as well as <= 0 

2649 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2650 return 

2651 

2652 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2653 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2654 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2655 

2656 @contextmanager 

2657 def flatContext(self, exp, flat, dark=None): 

2658 """Context manager that applies and removes flats and darks, 

2659 if the task is configured to apply them. 

2660 

2661 Parameters 

2662 ---------- 

2663 exp : `lsst.afw.image.Exposure` 

2664 Exposure to process. 

2665 flat : `lsst.afw.image.Exposure` 

2666 Flat exposure the same size as ``exp``. 

2667 dark : `lsst.afw.image.Exposure`, optional 

2668 Dark exposure the same size as ``exp``. 

2669 

2670 Yields 

2671 ------ 

2672 exp : `lsst.afw.image.Exposure` 

2673 The flat and dark corrected exposure. 

2674 """ 

2675 if self.config.doDark and dark is not None: 

2676 self.darkCorrection(exp, dark) 

2677 if self.config.doFlat: 

2678 self.flatCorrection(exp, flat) 

2679 try: 

2680 yield exp 

2681 finally: 

2682 if self.config.doFlat: 

2683 self.flatCorrection(exp, flat, invert=True) 

2684 if self.config.doDark and dark is not None: 

2685 self.darkCorrection(exp, dark, invert=True) 

2686 

2687 def makeBinnedImages(self, exposure): 

2688 """Make visualizeVisit style binned exposures. 

2689 

2690 Parameters 

2691 ---------- 

2692 exposure : `lsst.afw.image.Exposure` 

2693 Exposure to bin. 

2694 

2695 Returns 

2696 ------- 

2697 bin1 : `lsst.afw.image.Exposure` 

2698 Binned exposure using binFactor1. 

2699 bin2 : `lsst.afw.image.Exposure` 

2700 Binned exposure using binFactor2. 

2701 """ 

2702 mi = exposure.getMaskedImage() 

2703 

2704 bin1 = afwMath.binImage(mi, self.config.binFactor1) 

2705 bin2 = afwMath.binImage(mi, self.config.binFactor2) 

2706 

2707 return bin1, bin2 

2708 

2709 def debugView(self, exposure, stepname): 

2710 """Utility function to examine ISR exposure at different stages. 

2711 

2712 Parameters 

2713 ---------- 

2714 exposure : `lsst.afw.image.Exposure` 

2715 Exposure to view. 

2716 stepname : `str` 

2717 State of processing to view. 

2718 """ 

2719 frame = getDebugFrame(self._display, stepname) 

2720 if frame: 

2721 display = getDisplay(frame) 

2722 display.scale('asinh', 'zscale') 

2723 display.mtv(exposure) 

2724 prompt = "Press Enter to continue [c]... " 

2725 while True: 

2726 ans = input(prompt).lower() 

2727 if ans in ("", "c",): 

2728 break 

2729 

2730 

2731class FakeAmp(object): 

2732 """A Detector-like object that supports returning gain and saturation level 

2733 

2734 This is used when the input exposure does not have a detector. 

2735 

2736 Parameters 

2737 ---------- 

2738 exposure : `lsst.afw.image.Exposure` 

2739 Exposure to generate a fake amplifier for. 

2740 config : `lsst.ip.isr.isrTaskConfig` 

2741 Configuration to apply to the fake amplifier. 

2742 """ 

2743 

2744 def __init__(self, exposure, config): 

2745 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2746 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2747 self._gain = config.gain 

2748 self._readNoise = config.readNoise 

2749 self._saturation = config.saturation 

2750 

2751 def getBBox(self): 

2752 return self._bbox 

2753 

2754 def getRawBBox(self): 

2755 return self._bbox 

2756 

2757 def getRawHorizontalOverscanBBox(self): 

2758 return self._RawHorizontalOverscanBBox 

2759 

2760 def getGain(self): 

2761 return self._gain 

2762 

2763 def getReadNoise(self): 

2764 return self._readNoise 

2765 

2766 def getSaturation(self): 

2767 return self._saturation 

2768 

2769 def getSuspectLevel(self): 

2770 return float("NaN")