Coverage for python/lsst/ip/isr/isrTask.py: 16%

941 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-03 11:38 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from .ptcDataset import PhotonTransferCurveDataset 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.DataCoordinate` 

75 Expanded data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(registry.dimensions.conform(["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 records = {k: newDataId.records[k] for k in newDataId.dimensions.elements} 

95 return [ref.expanded(registry.expandDataId(ref.dataId, records=records)) for ref in results] 

96 

97 

98class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

99 dimensions={"instrument", "exposure", "detector"}, 

100 defaultTemplates={}): 

101 ccdExposure = cT.Input( 

102 name="raw", 

103 doc="Input exposure to process.", 

104 storageClass="Exposure", 

105 dimensions=["instrument", "exposure", "detector"], 

106 ) 

107 camera = cT.PrerequisiteInput( 

108 name="camera", 

109 storageClass="Camera", 

110 doc="Input camera to construct complete exposures.", 

111 dimensions=["instrument"], 

112 isCalibration=True, 

113 ) 

114 

115 crosstalk = cT.PrerequisiteInput( 

116 name="crosstalk", 

117 doc="Input crosstalk object", 

118 storageClass="CrosstalkCalib", 

119 dimensions=["instrument", "detector"], 

120 isCalibration=True, 

121 minimum=0, # can fall back to cameraGeom 

122 ) 

123 crosstalkSources = cT.PrerequisiteInput( 

124 name="isrOverscanCorrected", 

125 doc="Overscan corrected input images.", 

126 storageClass="Exposure", 

127 dimensions=["instrument", "exposure", "detector"], 

128 deferLoad=True, 

129 multiple=True, 

130 lookupFunction=crosstalkSourceLookup, 

131 minimum=0, # not needed for all instruments, no config to control this 

132 ) 

133 bias = cT.PrerequisiteInput( 

134 name="bias", 

135 doc="Input bias calibration.", 

136 storageClass="ExposureF", 

137 dimensions=["instrument", "detector"], 

138 isCalibration=True, 

139 ) 

140 dark = cT.PrerequisiteInput( 

141 name='dark', 

142 doc="Input dark calibration.", 

143 storageClass="ExposureF", 

144 dimensions=["instrument", "detector"], 

145 isCalibration=True, 

146 ) 

147 flat = cT.PrerequisiteInput( 

148 name="flat", 

149 doc="Input flat calibration.", 

150 storageClass="ExposureF", 

151 dimensions=["instrument", "physical_filter", "detector"], 

152 isCalibration=True, 

153 ) 

154 ptc = cT.PrerequisiteInput( 

155 name="ptc", 

156 doc="Input Photon Transfer Curve dataset", 

157 storageClass="PhotonTransferCurveDataset", 

158 dimensions=["instrument", "detector"], 

159 isCalibration=True, 

160 ) 

161 fringes = cT.PrerequisiteInput( 

162 name="fringe", 

163 doc="Input fringe calibration.", 

164 storageClass="ExposureF", 

165 dimensions=["instrument", "physical_filter", "detector"], 

166 isCalibration=True, 

167 minimum=0, # only needed for some bands, even when enabled 

168 ) 

169 strayLightData = cT.PrerequisiteInput( 

170 name='yBackground', 

171 doc="Input stray light calibration.", 

172 storageClass="StrayLightData", 

173 dimensions=["instrument", "physical_filter", "detector"], 

174 deferLoad=True, 

175 isCalibration=True, 

176 minimum=0, # only needed for some bands, even when enabled 

177 ) 

178 bfKernel = cT.PrerequisiteInput( 

179 name='bfKernel', 

180 doc="Input brighter-fatter kernel.", 

181 storageClass="NumpyArray", 

182 dimensions=["instrument"], 

183 isCalibration=True, 

184 minimum=0, # can use either bfKernel or newBFKernel 

185 ) 

186 newBFKernel = cT.PrerequisiteInput( 

187 name='brighterFatterKernel', 

188 doc="Newer complete kernel + gain solutions.", 

189 storageClass="BrighterFatterKernel", 

190 dimensions=["instrument", "detector"], 

191 isCalibration=True, 

192 minimum=0, # can use either bfKernel or newBFKernel 

193 ) 

194 defects = cT.PrerequisiteInput( 

195 name='defects', 

196 doc="Input defect tables.", 

197 storageClass="Defects", 

198 dimensions=["instrument", "detector"], 

199 isCalibration=True, 

200 ) 

201 linearizer = cT.PrerequisiteInput( 

202 name='linearizer', 

203 storageClass="Linearizer", 

204 doc="Linearity correction calibration.", 

205 dimensions=["instrument", "detector"], 

206 isCalibration=True, 

207 minimum=0, # can fall back to cameraGeom 

208 ) 

209 opticsTransmission = cT.PrerequisiteInput( 

210 name="transmission_optics", 

211 storageClass="TransmissionCurve", 

212 doc="Transmission curve due to the optics.", 

213 dimensions=["instrument"], 

214 isCalibration=True, 

215 ) 

216 filterTransmission = cT.PrerequisiteInput( 

217 name="transmission_filter", 

218 storageClass="TransmissionCurve", 

219 doc="Transmission curve due to the filter.", 

220 dimensions=["instrument", "physical_filter"], 

221 isCalibration=True, 

222 ) 

223 sensorTransmission = cT.PrerequisiteInput( 

224 name="transmission_sensor", 

225 storageClass="TransmissionCurve", 

226 doc="Transmission curve due to the sensor.", 

227 dimensions=["instrument", "detector"], 

228 isCalibration=True, 

229 ) 

230 atmosphereTransmission = cT.PrerequisiteInput( 

231 name="transmission_atmosphere", 

232 storageClass="TransmissionCurve", 

233 doc="Transmission curve due to the atmosphere.", 

234 dimensions=["instrument"], 

235 isCalibration=True, 

236 ) 

237 illumMaskedImage = cT.PrerequisiteInput( 

238 name="illum", 

239 doc="Input illumination correction.", 

240 storageClass="MaskedImageF", 

241 dimensions=["instrument", "physical_filter", "detector"], 

242 isCalibration=True, 

243 ) 

244 deferredChargeCalib = cT.PrerequisiteInput( 

245 name="cpCtiCalib", 

246 doc="Deferred charge/CTI correction dataset.", 

247 storageClass="IsrCalib", 

248 dimensions=["instrument", "detector"], 

249 isCalibration=True, 

250 ) 

251 

252 outputExposure = cT.Output( 

253 name='postISRCCD', 

254 doc="Output ISR processed exposure.", 

255 storageClass="Exposure", 

256 dimensions=["instrument", "exposure", "detector"], 

257 ) 

258 preInterpExposure = cT.Output( 

259 name='preInterpISRCCD', 

260 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

261 storageClass="ExposureF", 

262 dimensions=["instrument", "exposure", "detector"], 

263 ) 

264 outputBin1Exposure = cT.Output( 

265 name="postIsrBin1", 

266 doc="First binned image.", 

267 storageClass="ExposureF", 

268 dimensions=["instrument", "exposure", "detector"], 

269 ) 

270 outputBin2Exposure = cT.Output( 

271 name="postIsrBin2", 

272 doc="Second binned image.", 

273 storageClass="ExposureF", 

274 dimensions=["instrument", "exposure", "detector"], 

275 ) 

276 

277 outputOssThumbnail = cT.Output( 

278 name="OssThumb", 

279 doc="Output Overscan-subtracted thumbnail image.", 

280 storageClass="Thumbnail", 

281 dimensions=["instrument", "exposure", "detector"], 

282 ) 

283 outputFlattenedThumbnail = cT.Output( 

284 name="FlattenedThumb", 

285 doc="Output flat-corrected thumbnail image.", 

286 storageClass="Thumbnail", 

287 dimensions=["instrument", "exposure", "detector"], 

288 ) 

289 outputStatistics = cT.Output( 

290 name="isrStatistics", 

291 doc="Output of additional statistics table.", 

292 storageClass="StructuredDataDict", 

293 dimensions=["instrument", "exposure", "detector"], 

294 ) 

295 

296 def __init__(self, *, config=None): 

297 super().__init__(config=config) 

298 

299 if config.doBias is not True: 

300 self.prerequisiteInputs.remove("bias") 

301 if config.doLinearize is not True: 

302 self.prerequisiteInputs.remove("linearizer") 

303 if config.doCrosstalk is not True: 

304 self.prerequisiteInputs.remove("crosstalkSources") 

305 self.prerequisiteInputs.remove("crosstalk") 

306 if config.doBrighterFatter is not True: 

307 self.prerequisiteInputs.remove("bfKernel") 

308 self.prerequisiteInputs.remove("newBFKernel") 

309 if config.doDefect is not True: 

310 self.prerequisiteInputs.remove("defects") 

311 if config.doDark is not True: 

312 self.prerequisiteInputs.remove("dark") 

313 if config.doFlat is not True: 

314 self.prerequisiteInputs.remove("flat") 

315 if config.doFringe is not True: 

316 self.prerequisiteInputs.remove("fringes") 

317 if config.doStrayLight is not True: 

318 self.prerequisiteInputs.remove("strayLightData") 

319 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

320 self.prerequisiteInputs.remove("ptc") 

321 if config.doAttachTransmissionCurve is not True: 

322 self.prerequisiteInputs.remove("opticsTransmission") 

323 self.prerequisiteInputs.remove("filterTransmission") 

324 self.prerequisiteInputs.remove("sensorTransmission") 

325 self.prerequisiteInputs.remove("atmosphereTransmission") 

326 else: 

327 if config.doUseOpticsTransmission is not True: 

328 self.prerequisiteInputs.remove("opticsTransmission") 

329 if config.doUseFilterTransmission is not True: 

330 self.prerequisiteInputs.remove("filterTransmission") 

331 if config.doUseSensorTransmission is not True: 

332 self.prerequisiteInputs.remove("sensorTransmission") 

333 if config.doUseAtmosphereTransmission is not True: 

334 self.prerequisiteInputs.remove("atmosphereTransmission") 

335 if config.doIlluminationCorrection is not True: 

336 self.prerequisiteInputs.remove("illumMaskedImage") 

337 if config.doDeferredCharge is not True: 

338 self.prerequisiteInputs.remove("deferredChargeCalib") 

339 

340 if config.doWrite is not True: 

341 self.outputs.remove("outputExposure") 

342 self.outputs.remove("preInterpExposure") 

343 self.outputs.remove("outputFlattenedThumbnail") 

344 self.outputs.remove("outputOssThumbnail") 

345 self.outputs.remove("outputStatistics") 

346 self.outputs.remove("outputBin1Exposure") 

347 self.outputs.remove("outputBin2Exposure") 

348 else: 

349 if config.doBinnedExposures is not True: 

350 self.outputs.remove("outputBin1Exposure") 

351 self.outputs.remove("outputBin2Exposure") 

352 if config.doSaveInterpPixels is not True: 

353 self.outputs.remove("preInterpExposure") 

354 if config.qa.doThumbnailOss is not True: 

355 self.outputs.remove("outputOssThumbnail") 

356 if config.qa.doThumbnailFlattened is not True: 

357 self.outputs.remove("outputFlattenedThumbnail") 

358 if config.doCalculateStatistics is not True: 

359 self.outputs.remove("outputStatistics") 

360 

361 

362class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

363 pipelineConnections=IsrTaskConnections): 

364 """Configuration parameters for IsrTask. 

365 

366 Items are grouped in the order in which they are executed by the task. 

367 """ 

368 datasetType = pexConfig.Field( 

369 dtype=str, 

370 doc="Dataset type for input data; users will typically leave this alone, " 

371 "but camera-specific ISR tasks will override it", 

372 default="raw", 

373 ) 

374 

375 fallbackFilterName = pexConfig.Field( 

376 dtype=str, 

377 doc="Fallback default filter name for calibrations.", 

378 optional=True 

379 ) 

380 useFallbackDate = pexConfig.Field( 

381 dtype=bool, 

382 doc="Pass observation date when using fallback filter.", 

383 default=False, 

384 ) 

385 expectWcs = pexConfig.Field( 

386 dtype=bool, 

387 default=True, 

388 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

389 ) 

390 fwhm = pexConfig.Field( 

391 dtype=float, 

392 doc="FWHM of PSF in arcseconds (currently unused).", 

393 default=1.0, 

394 ) 

395 qa = pexConfig.ConfigField( 

396 dtype=isrQa.IsrQaConfig, 

397 doc="QA related configuration options.", 

398 ) 

399 doHeaderProvenance = pexConfig.Field( 

400 dtype=bool, 

401 default=True, 

402 doc="Write calibration identifiers into output exposure header?", 

403 ) 

404 

405 # Calib checking configuration: 

406 doRaiseOnCalibMismatch = pexConfig.Field( 

407 dtype=bool, 

408 default=False, 

409 doc="Should IsrTask halt if exposure and calibration header values do not match?", 

410 ) 

411 cameraKeywordsToCompare = pexConfig.ListField( 

412 dtype=str, 

413 doc="List of header keywords to compare between exposure and calibrations.", 

414 default=[], 

415 ) 

416 

417 # Image conversion configuration 

418 doConvertIntToFloat = pexConfig.Field( 

419 dtype=bool, 

420 doc="Convert integer raw images to floating point values?", 

421 default=True, 

422 ) 

423 

424 # Saturated pixel handling. 

425 doSaturation = pexConfig.Field( 

426 dtype=bool, 

427 doc="Mask saturated pixels? NB: this is totally independent of the" 

428 " interpolation option - this is ONLY setting the bits in the mask." 

429 " To have them interpolated make sure doSaturationInterpolation=True", 

430 default=True, 

431 ) 

432 saturatedMaskName = pexConfig.Field( 

433 dtype=str, 

434 doc="Name of mask plane to use in saturation detection and interpolation", 

435 default="SAT", 

436 ) 

437 saturation = pexConfig.Field( 

438 dtype=float, 

439 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

440 default=float("NaN"), 

441 ) 

442 growSaturationFootprintSize = pexConfig.Field( 

443 dtype=int, 

444 doc="Number of pixels by which to grow the saturation footprints", 

445 default=1, 

446 ) 

447 

448 # Suspect pixel handling. 

449 doSuspect = pexConfig.Field( 

450 dtype=bool, 

451 doc="Mask suspect pixels?", 

452 default=False, 

453 ) 

454 suspectMaskName = pexConfig.Field( 

455 dtype=str, 

456 doc="Name of mask plane to use for suspect pixels", 

457 default="SUSPECT", 

458 ) 

459 numEdgeSuspect = pexConfig.Field( 

460 dtype=int, 

461 doc="Number of edge pixels to be flagged as untrustworthy.", 

462 default=0, 

463 ) 

464 edgeMaskLevel = pexConfig.ChoiceField( 

465 dtype=str, 

466 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

467 default="DETECTOR", 

468 allowed={ 

469 'DETECTOR': 'Mask only the edges of the full detector.', 

470 'AMP': 'Mask edges of each amplifier.', 

471 }, 

472 ) 

473 

474 # Initial masking options. 

475 doSetBadRegions = pexConfig.Field( 

476 dtype=bool, 

477 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

478 default=True, 

479 ) 

480 badStatistic = pexConfig.ChoiceField( 

481 dtype=str, 

482 doc="How to estimate the average value for BAD regions.", 

483 default='MEANCLIP', 

484 allowed={ 

485 "MEANCLIP": "Correct using the (clipped) mean of good data", 

486 "MEDIAN": "Correct using the median of the good data", 

487 }, 

488 ) 

489 

490 # Overscan subtraction configuration. 

491 doOverscan = pexConfig.Field( 

492 dtype=bool, 

493 doc="Do overscan subtraction?", 

494 default=True, 

495 ) 

496 overscan = pexConfig.ConfigurableField( 

497 target=OverscanCorrectionTask, 

498 doc="Overscan subtraction task for image segments.", 

499 ) 

500 

501 # Amplifier to CCD assembly configuration 

502 doAssembleCcd = pexConfig.Field( 

503 dtype=bool, 

504 default=True, 

505 doc="Assemble amp-level exposures into a ccd-level exposure?" 

506 ) 

507 assembleCcd = pexConfig.ConfigurableField( 

508 target=AssembleCcdTask, 

509 doc="CCD assembly task", 

510 ) 

511 

512 # General calibration configuration. 

513 doAssembleIsrExposures = pexConfig.Field( 

514 dtype=bool, 

515 default=False, 

516 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

517 ) 

518 doTrimToMatchCalib = pexConfig.Field( 

519 dtype=bool, 

520 default=False, 

521 doc="Trim raw data to match calibration bounding boxes?" 

522 ) 

523 

524 # Bias subtraction. 

525 doBias = pexConfig.Field( 

526 dtype=bool, 

527 doc="Apply bias frame correction?", 

528 default=True, 

529 ) 

530 biasDataProductName = pexConfig.Field( 

531 dtype=str, 

532 doc="Name of the bias data product", 

533 default="bias", 

534 ) 

535 doBiasBeforeOverscan = pexConfig.Field( 

536 dtype=bool, 

537 doc="Reverse order of overscan and bias correction.", 

538 default=False 

539 ) 

540 

541 # Deferred charge correction. 

542 doDeferredCharge = pexConfig.Field( 

543 dtype=bool, 

544 doc="Apply deferred charge correction?", 

545 default=False, 

546 ) 

547 deferredChargeCorrection = pexConfig.ConfigurableField( 

548 target=DeferredChargeTask, 

549 doc="Deferred charge correction task.", 

550 ) 

551 

552 # Variance construction 

553 doVariance = pexConfig.Field( 

554 dtype=bool, 

555 doc="Calculate variance?", 

556 default=True 

557 ) 

558 gain = pexConfig.Field( 

559 dtype=float, 

560 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

561 default=float("NaN"), 

562 ) 

563 readNoise = pexConfig.Field( 

564 dtype=float, 

565 doc="The read noise to use if no Detector is present in the Exposure", 

566 default=0.0, 

567 ) 

568 doEmpiricalReadNoise = pexConfig.Field( 

569 dtype=bool, 

570 default=False, 

571 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

572 ) 

573 usePtcReadNoise = pexConfig.Field( 

574 dtype=bool, 

575 default=False, 

576 doc="Use readnoise values from the Photon Transfer Curve?" 

577 ) 

578 maskNegativeVariance = pexConfig.Field( 

579 dtype=bool, 

580 default=True, 

581 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

582 "in the measurement of the overscan at an edge due to the data falling off faster " 

583 "than the overscan model can account for it." 

584 ) 

585 negativeVarianceMaskName = pexConfig.Field( 

586 dtype=str, 

587 default="BAD", 

588 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

589 ) 

590 # Linearization. 

591 doLinearize = pexConfig.Field( 

592 dtype=bool, 

593 doc="Correct for nonlinearity of the detector's response?", 

594 default=True, 

595 ) 

596 

597 # Crosstalk. 

598 doCrosstalk = pexConfig.Field( 

599 dtype=bool, 

600 doc="Apply intra-CCD crosstalk correction?", 

601 default=False, 

602 ) 

603 doCrosstalkBeforeAssemble = pexConfig.Field( 

604 dtype=bool, 

605 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

606 default=False, 

607 ) 

608 crosstalk = pexConfig.ConfigurableField( 

609 target=CrosstalkTask, 

610 doc="Intra-CCD crosstalk correction", 

611 ) 

612 

613 # Masking options. 

614 doDefect = pexConfig.Field( 

615 dtype=bool, 

616 doc="Apply correction for CCD defects, e.g. hot pixels?", 

617 default=True, 

618 ) 

619 doNanMasking = pexConfig.Field( 

620 dtype=bool, 

621 doc="Mask non-finite (NAN, inf) pixels?", 

622 default=True, 

623 ) 

624 doWidenSaturationTrails = pexConfig.Field( 

625 dtype=bool, 

626 doc="Widen bleed trails based on their width?", 

627 default=True 

628 ) 

629 

630 # Brighter-Fatter correction. 

631 doBrighterFatter = pexConfig.Field( 

632 dtype=bool, 

633 default=False, 

634 doc="Apply the brighter-fatter correction?" 

635 ) 

636 doFluxConservingBrighterFatterCorrection = pexConfig.Field( 

637 dtype=bool, 

638 default=False, 

639 doc="Apply the flux-conserving BFE correction by Miller et al.?" 

640 ) 

641 brighterFatterLevel = pexConfig.ChoiceField( 

642 dtype=str, 

643 default="DETECTOR", 

644 doc="The level at which to correct for brighter-fatter.", 

645 allowed={ 

646 "AMP": "Every amplifier treated separately.", 

647 "DETECTOR": "One kernel per detector", 

648 } 

649 ) 

650 brighterFatterMaxIter = pexConfig.Field( 

651 dtype=int, 

652 default=10, 

653 doc="Maximum number of iterations for the brighter-fatter correction" 

654 ) 

655 brighterFatterThreshold = pexConfig.Field( 

656 dtype=float, 

657 default=1000, 

658 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

659 "absolute value of the difference between the current corrected image and the one " 

660 "from the previous iteration summed over all the pixels." 

661 ) 

662 brighterFatterApplyGain = pexConfig.Field( 

663 dtype=bool, 

664 default=True, 

665 doc="Should the gain be applied when applying the brighter-fatter correction?" 

666 ) 

667 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

668 dtype=str, 

669 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

670 "correction.", 

671 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

672 ) 

673 brighterFatterMaskGrowSize = pexConfig.Field( 

674 dtype=int, 

675 default=0, 

676 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

677 "when brighter-fatter correction is applied." 

678 ) 

679 

680 # Dark subtraction. 

681 doDark = pexConfig.Field( 

682 dtype=bool, 

683 doc="Apply dark frame correction?", 

684 default=True, 

685 ) 

686 darkDataProductName = pexConfig.Field( 

687 dtype=str, 

688 doc="Name of the dark data product", 

689 default="dark", 

690 ) 

691 

692 # Camera-specific stray light removal. 

693 doStrayLight = pexConfig.Field( 

694 dtype=bool, 

695 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

696 default=False, 

697 ) 

698 strayLight = pexConfig.ConfigurableField( 

699 target=StrayLightTask, 

700 doc="y-band stray light correction" 

701 ) 

702 

703 # Flat correction. 

704 doFlat = pexConfig.Field( 

705 dtype=bool, 

706 doc="Apply flat field correction?", 

707 default=True, 

708 ) 

709 flatDataProductName = pexConfig.Field( 

710 dtype=str, 

711 doc="Name of the flat data product", 

712 default="flat", 

713 ) 

714 flatScalingType = pexConfig.ChoiceField( 

715 dtype=str, 

716 doc="The method for scaling the flat on the fly.", 

717 default='USER', 

718 allowed={ 

719 "USER": "Scale by flatUserScale", 

720 "MEAN": "Scale by the inverse of the mean", 

721 "MEDIAN": "Scale by the inverse of the median", 

722 }, 

723 ) 

724 flatUserScale = pexConfig.Field( 

725 dtype=float, 

726 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

727 default=1.0, 

728 ) 

729 doTweakFlat = pexConfig.Field( 

730 dtype=bool, 

731 doc="Tweak flats to match observed amplifier ratios?", 

732 default=False 

733 ) 

734 

735 # Amplifier normalization based on gains instead of using flats 

736 # configuration. 

737 doApplyGains = pexConfig.Field( 

738 dtype=bool, 

739 doc="Correct the amplifiers for their gains instead of applying flat correction", 

740 default=False, 

741 ) 

742 usePtcGains = pexConfig.Field( 

743 dtype=bool, 

744 doc="Use the gain values from the input Photon Transfer Curve?", 

745 default=False, 

746 ) 

747 normalizeGains = pexConfig.Field( 

748 dtype=bool, 

749 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

750 default=False, 

751 ) 

752 

753 # Fringe correction. 

754 doFringe = pexConfig.Field( 

755 dtype=bool, 

756 doc="Apply fringe correction?", 

757 default=True, 

758 ) 

759 fringe = pexConfig.ConfigurableField( 

760 target=FringeTask, 

761 doc="Fringe subtraction task", 

762 ) 

763 fringeAfterFlat = pexConfig.Field( 

764 dtype=bool, 

765 doc="Do fringe subtraction after flat-fielding?", 

766 default=True, 

767 ) 

768 

769 # Amp offset correction. 

770 doAmpOffset = pexConfig.Field( 

771 doc="Calculate and apply amp offset corrections?", 

772 dtype=bool, 

773 default=False, 

774 ) 

775 ampOffset = pexConfig.ConfigurableField( 

776 doc="Amp offset correction task.", 

777 target=AmpOffsetTask, 

778 ) 

779 

780 # Initial CCD-level background statistics options. 

781 doMeasureBackground = pexConfig.Field( 

782 dtype=bool, 

783 doc="Measure the background level on the reduced image?", 

784 default=False, 

785 ) 

786 

787 # Camera-specific masking configuration. 

788 doCameraSpecificMasking = pexConfig.Field( 

789 dtype=bool, 

790 doc="Mask camera-specific bad regions?", 

791 default=False, 

792 ) 

793 masking = pexConfig.ConfigurableField( 

794 target=MaskingTask, 

795 doc="Masking task." 

796 ) 

797 

798 # Interpolation options. 

799 doInterpolate = pexConfig.Field( 

800 dtype=bool, 

801 doc="Interpolate masked pixels?", 

802 default=True, 

803 ) 

804 doSaturationInterpolation = pexConfig.Field( 

805 dtype=bool, 

806 doc="Perform interpolation over pixels masked as saturated?" 

807 " NB: This is independent of doSaturation; if that is False this plane" 

808 " will likely be blank, resulting in a no-op here.", 

809 default=True, 

810 ) 

811 doNanInterpolation = pexConfig.Field( 

812 dtype=bool, 

813 doc="Perform interpolation over pixels masked as NaN?" 

814 " NB: This is independent of doNanMasking; if that is False this plane" 

815 " will likely be blank, resulting in a no-op here.", 

816 default=True, 

817 ) 

818 doNanInterpAfterFlat = pexConfig.Field( 

819 dtype=bool, 

820 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

821 "also have to interpolate them before flat-fielding."), 

822 default=False, 

823 ) 

824 maskListToInterpolate = pexConfig.ListField( 

825 dtype=str, 

826 doc="List of mask planes that should be interpolated.", 

827 default=['SAT', 'BAD'], 

828 ) 

829 doSaveInterpPixels = pexConfig.Field( 

830 dtype=bool, 

831 doc="Save a copy of the pre-interpolated pixel values?", 

832 default=False, 

833 ) 

834 

835 # Default photometric calibration options. 

836 fluxMag0T1 = pexConfig.DictField( 

837 keytype=str, 

838 itemtype=float, 

839 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

840 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

841 )) 

842 ) 

843 defaultFluxMag0T1 = pexConfig.Field( 

844 dtype=float, 

845 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

846 default=pow(10.0, 0.4*28.0) 

847 ) 

848 

849 # Vignette correction configuration. 

850 doVignette = pexConfig.Field( 

851 dtype=bool, 

852 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

853 "according to vignetting parameters?"), 

854 default=False, 

855 ) 

856 doMaskVignettePolygon = pexConfig.Field( 

857 dtype=bool, 

858 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

859 "is False"), 

860 default=True, 

861 ) 

862 vignetteValue = pexConfig.Field( 

863 dtype=float, 

864 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

865 optional=True, 

866 default=None, 

867 ) 

868 vignette = pexConfig.ConfigurableField( 

869 target=VignetteTask, 

870 doc="Vignetting task.", 

871 ) 

872 

873 # Transmission curve configuration. 

874 doAttachTransmissionCurve = pexConfig.Field( 

875 dtype=bool, 

876 default=False, 

877 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

878 ) 

879 doUseOpticsTransmission = pexConfig.Field( 

880 dtype=bool, 

881 default=True, 

882 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

883 ) 

884 doUseFilterTransmission = pexConfig.Field( 

885 dtype=bool, 

886 default=True, 

887 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

888 ) 

889 doUseSensorTransmission = pexConfig.Field( 

890 dtype=bool, 

891 default=True, 

892 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

893 ) 

894 doUseAtmosphereTransmission = pexConfig.Field( 

895 dtype=bool, 

896 default=True, 

897 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

898 ) 

899 

900 # Illumination correction. 

901 doIlluminationCorrection = pexConfig.Field( 

902 dtype=bool, 

903 default=False, 

904 doc="Perform illumination correction?" 

905 ) 

906 illuminationCorrectionDataProductName = pexConfig.Field( 

907 dtype=str, 

908 doc="Name of the illumination correction data product.", 

909 default="illumcor", 

910 ) 

911 illumScale = pexConfig.Field( 

912 dtype=float, 

913 doc="Scale factor for the illumination correction.", 

914 default=1.0, 

915 ) 

916 illumFilters = pexConfig.ListField( 

917 dtype=str, 

918 default=[], 

919 doc="Only perform illumination correction for these filters." 

920 ) 

921 

922 # Calculate image quality statistics? 

923 doStandardStatistics = pexConfig.Field( 

924 dtype=bool, 

925 doc="Should standard image quality statistics be calculated?", 

926 default=True, 

927 ) 

928 # Calculate additional statistics? 

929 doCalculateStatistics = pexConfig.Field( 

930 dtype=bool, 

931 doc="Should additional ISR statistics be calculated?", 

932 default=False, 

933 ) 

934 isrStats = pexConfig.ConfigurableField( 

935 target=IsrStatisticsTask, 

936 doc="Task to calculate additional statistics.", 

937 ) 

938 

939 # Make binned images? 

940 doBinnedExposures = pexConfig.Field( 

941 dtype=bool, 

942 doc="Should binned exposures be calculated?", 

943 default=False, 

944 ) 

945 binFactor1 = pexConfig.Field( 945 ↛ exitline 945 didn't jump to the function exit

946 dtype=int, 

947 doc="Binning factor for first binned exposure. This is intended for a finely binned output.", 

948 default=8, 

949 check=lambda x: x > 1, 

950 ) 

951 binFactor2 = pexConfig.Field( 951 ↛ exitline 951 didn't jump to the function exit

952 dtype=int, 

953 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.", 

954 default=64, 

955 check=lambda x: x > 1, 

956 ) 

957 

958 # Write the outputs to disk. If ISR is run as a subtask, this may not 

959 # be needed. 

960 doWrite = pexConfig.Field( 

961 dtype=bool, 

962 doc="Persist postISRCCD?", 

963 default=True, 

964 ) 

965 

966 def validate(self): 

967 super().validate() 

968 if self.doFlat and self.doApplyGains: 

969 raise ValueError("You may not specify both doFlat and doApplyGains") 

970 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

971 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

972 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

973 self.maskListToInterpolate.append(self.saturatedMaskName) 

974 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

975 self.maskListToInterpolate.remove(self.saturatedMaskName) 

976 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

977 self.maskListToInterpolate.append("UNMASKEDNAN") 

978 if self.doCalculateStatistics and self.isrStats.doCtiStatistics: 

979 if self.doApplyGains != self.isrStats.doApplyGainsForCtiStatistics: 

980 raise ValueError("doApplyGains must match isrStats.applyGainForCtiStatistics.") 

981 

982 

983class IsrTask(pipeBase.PipelineTask): 

984 """Apply common instrument signature correction algorithms to a raw frame. 

985 

986 The process for correcting imaging data is very similar from 

987 camera to camera. This task provides a vanilla implementation of 

988 doing these corrections, including the ability to turn certain 

989 corrections off if they are not needed. The inputs to the primary 

990 method, `run()`, are a raw exposure to be corrected and the 

991 calibration data products. The raw input is a single chip sized 

992 mosaic of all amps including overscans and other non-science 

993 pixels. 

994 

995 The __init__ method sets up the subtasks for ISR processing, using 

996 the defaults from `lsst.ip.isr`. 

997 

998 Parameters 

999 ---------- 

1000 args : `list` 

1001 Positional arguments passed to the Task constructor. 

1002 None used at this time. 

1003 kwargs : `dict`, optional 

1004 Keyword arguments passed on to the Task constructor. 

1005 None used at this time. 

1006 """ 

1007 ConfigClass = IsrTaskConfig 

1008 _DefaultName = "isr" 

1009 

1010 def __init__(self, **kwargs): 

1011 super().__init__(**kwargs) 

1012 self.makeSubtask("assembleCcd") 

1013 self.makeSubtask("crosstalk") 

1014 self.makeSubtask("strayLight") 

1015 self.makeSubtask("fringe") 

1016 self.makeSubtask("masking") 

1017 self.makeSubtask("overscan") 

1018 self.makeSubtask("vignette") 

1019 self.makeSubtask("ampOffset") 

1020 self.makeSubtask("deferredChargeCorrection") 

1021 self.makeSubtask("isrStats") 

1022 

1023 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1024 inputs = butlerQC.get(inputRefs) 

1025 

1026 try: 

1027 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

1028 except Exception as e: 

1029 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

1030 (inputRefs, e)) 

1031 

1032 detector = inputs['ccdExposure'].getDetector() 

1033 

1034 # This is use for header provenance. 

1035 additionalInputDates = {} 

1036 

1037 if self.config.doCrosstalk is True: 

1038 # Crosstalk sources need to be defined by the pipeline 

1039 # yaml if they exist. 

1040 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

1041 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

1042 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

1043 else: 

1044 coeffVector = (self.config.crosstalk.crosstalkValues 

1045 if self.config.crosstalk.useConfigCoefficients else None) 

1046 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1047 inputs['crosstalk'] = crosstalkCalib 

1048 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1049 if 'crosstalkSources' not in inputs: 

1050 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1051 

1052 if self.doLinearize(detector) is True: 

1053 if 'linearizer' in inputs: 

1054 if isinstance(inputs['linearizer'], dict): 

1055 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1056 linearizer.fromYaml(inputs['linearizer']) 

1057 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1058 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1059 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1060 detector=detector, 

1061 log=self.log) 

1062 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1063 else: 

1064 linearizer = inputs['linearizer'] 

1065 linearizer.log = self.log 

1066 inputs['linearizer'] = linearizer 

1067 else: 

1068 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1069 self.log.warning("Constructing linearizer from cameraGeom information.") 

1070 

1071 if self.config.doDefect is True: 

1072 if "defects" in inputs and inputs['defects'] is not None: 

1073 # defects is loaded as a BaseCatalog with columns 

1074 # x0, y0, width, height. Masking expects a list of defects 

1075 # defined by their bounding box 

1076 if not isinstance(inputs["defects"], Defects): 

1077 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1078 

1079 # Load the correct style of brighter-fatter kernel, and repack 

1080 # the information as a numpy array. 

1081 brighterFatterSource = None 

1082 if self.config.doBrighterFatter: 

1083 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1084 if brighterFatterKernel is None: 

1085 # This type of kernel must be in (y, x) index 

1086 # ordering, as it used directly as the .array 

1087 # component of the afwImage kernel. 

1088 brighterFatterKernel = inputs.get('bfKernel', None) 

1089 brighterFatterSource = 'bfKernel' 

1090 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1091 

1092 if brighterFatterKernel is None: 

1093 # This was requested by the config, but none were found. 

1094 raise RuntimeError("No brighter-fatter kernel was supplied.") 

1095 elif not isinstance(brighterFatterKernel, numpy.ndarray): 

1096 # This is a ISR calib kernel. These kernels are 

1097 # generated in (x, y) index ordering, and need to be 

1098 # transposed to be used directly as the .array 

1099 # component of the afwImage kernel. This is done 

1100 # explicitly below when setting the ``bfKernel`` 

1101 # input. 

1102 brighterFatterSource = 'newBFKernel' 

1103 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1104 

1105 detName = detector.getName() 

1106 level = brighterFatterKernel.level 

1107 

1108 # This is expected to be a dictionary of amp-wise gains. 

1109 inputs['bfGains'] = brighterFatterKernel.gain 

1110 if self.config.brighterFatterLevel == 'DETECTOR': 

1111 kernel = None 

1112 if level == 'DETECTOR': 

1113 if detName in brighterFatterKernel.detKernels: 

1114 kernel = brighterFatterKernel.detKernels[detName] 

1115 else: 

1116 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1117 elif level == 'AMP': 

1118 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1119 "fatter kernels.") 

1120 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1121 kernel = brighterFatterKernel.detKernels[detName] 

1122 if kernel is None: 

1123 raise RuntimeError("Could not identify brighter-fatter kernel!") 

1124 # Do the one single transpose here so the kernel 

1125 # can be directly loaded into the afwImage .array 

1126 # component. 

1127 inputs['bfKernel'] = numpy.transpose(kernel) 

1128 elif self.config.brighterFatterLevel == 'AMP': 

1129 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1130 

1131 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1132 expId = inputs['ccdExposure'].info.id 

1133 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1134 expId=expId, 

1135 assembler=self.assembleCcd 

1136 if self.config.doAssembleIsrExposures else None) 

1137 else: 

1138 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1139 

1140 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1141 if 'strayLightData' not in inputs: 

1142 inputs['strayLightData'] = None 

1143 

1144 if self.config.doHeaderProvenance: 

1145 # Add calibration provenanace info to header. 

1146 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1147 

1148 # These inputs change name during this step. These should 

1149 # have matching entries in the additionalInputDates dict. 

1150 additionalInputs = [] 

1151 if self.config.doBrighterFatter: 

1152 additionalInputs.append(brighterFatterSource) 

1153 

1154 for inputName in sorted(list(inputs.keys()) + additionalInputs): 

1155 reference = getattr(inputRefs, inputName, None) 

1156 if reference is not None and hasattr(reference, "run"): 

1157 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1158 runValue = reference.run 

1159 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1160 idValue = str(reference.id) 

1161 dateKey = f"LSST CALIB DATE {inputName.upper()}" 

1162 

1163 if inputName in additionalInputDates: 

1164 dateValue = additionalInputDates[inputName] 

1165 else: 

1166 dateValue = self.extractCalibDate(inputs[inputName]) 

1167 

1168 exposureMetadata[runKey] = runValue 

1169 exposureMetadata[idKey] = idValue 

1170 exposureMetadata[dateKey] = dateValue 

1171 

1172 outputs = self.run(**inputs) 

1173 butlerQC.put(outputs, outputRefs) 

1174 

1175 @timeMethod 

1176 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1177 crosstalk=None, crosstalkSources=None, 

1178 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1179 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1180 sensorTransmission=None, atmosphereTransmission=None, 

1181 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1182 deferredChargeCalib=None, 

1183 ): 

1184 """Perform instrument signature removal on an exposure. 

1185 

1186 Steps included in the ISR processing, in order performed, are: 

1187 

1188 - saturation and suspect pixel masking 

1189 - overscan subtraction 

1190 - CCD assembly of individual amplifiers 

1191 - bias subtraction 

1192 - variance image construction 

1193 - linearization of non-linear response 

1194 - crosstalk masking 

1195 - brighter-fatter correction 

1196 - dark subtraction 

1197 - fringe correction 

1198 - stray light subtraction 

1199 - flat correction 

1200 - masking of known defects and camera specific features 

1201 - vignette calculation 

1202 - appending transmission curve and distortion model 

1203 

1204 Parameters 

1205 ---------- 

1206 ccdExposure : `lsst.afw.image.Exposure` 

1207 The raw exposure that is to be run through ISR. The 

1208 exposure is modified by this method. 

1209 camera : `lsst.afw.cameraGeom.Camera`, optional 

1210 The camera geometry for this exposure. Required if 

1211 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1212 ``flat`` does not have an associated detector. 

1213 bias : `lsst.afw.image.Exposure`, optional 

1214 Bias calibration frame. 

1215 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1216 Functor for linearization. 

1217 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1218 Calibration for crosstalk. 

1219 crosstalkSources : `list`, optional 

1220 List of possible crosstalk sources. 

1221 dark : `lsst.afw.image.Exposure`, optional 

1222 Dark calibration frame. 

1223 flat : `lsst.afw.image.Exposure`, optional 

1224 Flat calibration frame. 

1225 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1226 Photon transfer curve dataset, with, e.g., gains 

1227 and read noise. 

1228 bfKernel : `numpy.ndarray`, optional 

1229 Brighter-fatter kernel. 

1230 bfGains : `dict` of `float`, optional 

1231 Gains used to override the detector's nominal gains for the 

1232 brighter-fatter correction. A dict keyed by amplifier name for 

1233 the detector in question. 

1234 defects : `lsst.ip.isr.Defects`, optional 

1235 List of defects. 

1236 fringes : `lsst.pipe.base.Struct`, optional 

1237 Struct containing the fringe correction data, with 

1238 elements: 

1239 

1240 ``fringes`` 

1241 fringe calibration frame (`lsst.afw.image.Exposure`) 

1242 ``seed`` 

1243 random seed derived from the ``ccdExposureId`` for random 

1244 number generator (`numpy.uint32`) 

1245 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1246 A ``TransmissionCurve`` that represents the throughput of the, 

1247 optics, to be evaluated in focal-plane coordinates. 

1248 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1249 A ``TransmissionCurve`` that represents the throughput of the 

1250 filter itself, to be evaluated in focal-plane coordinates. 

1251 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1252 A ``TransmissionCurve`` that represents the throughput of the 

1253 sensor itself, to be evaluated in post-assembly trimmed detector 

1254 coordinates. 

1255 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1256 A ``TransmissionCurve`` that represents the throughput of the 

1257 atmosphere, assumed to be spatially constant. 

1258 detectorNum : `int`, optional 

1259 The integer number for the detector to process. 

1260 strayLightData : `object`, optional 

1261 Opaque object containing calibration information for stray-light 

1262 correction. If `None`, no correction will be performed. 

1263 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1264 Illumination correction image. 

1265 

1266 Returns 

1267 ------- 

1268 result : `lsst.pipe.base.Struct` 

1269 Result struct with component: 

1270 

1271 ``exposure`` 

1272 The fully ISR corrected exposure. 

1273 (`lsst.afw.image.Exposure`) 

1274 ``outputExposure`` 

1275 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1276 ``ossThumb`` 

1277 Thumbnail image of the exposure after overscan subtraction. 

1278 (`numpy.ndarray`) 

1279 ``flattenedThumb`` 

1280 Thumbnail image of the exposure after flat-field correction. 

1281 (`numpy.ndarray`) 

1282 ``outputStatistics`` 

1283 Values of the additional statistics calculated. 

1284 

1285 Raises 

1286 ------ 

1287 RuntimeError 

1288 Raised if a configuration option is set to `True`, but the 

1289 required calibration data has not been specified. 

1290 

1291 Notes 

1292 ----- 

1293 The current processed exposure can be viewed by setting the 

1294 appropriate `lsstDebug` entries in the ``debug.display`` 

1295 dictionary. The names of these entries correspond to some of 

1296 the `IsrTaskConfig` Boolean options, with the value denoting the 

1297 frame to use. The exposure is shown inside the matching 

1298 option check and after the processing of that step has 

1299 finished. The steps with debug points are: 

1300 

1301 * doAssembleCcd 

1302 * doBias 

1303 * doCrosstalk 

1304 * doBrighterFatter 

1305 * doDark 

1306 * doFringe 

1307 * doStrayLight 

1308 * doFlat 

1309 

1310 In addition, setting the ``postISRCCD`` entry displays the 

1311 exposure after all ISR processing has finished. 

1312 """ 

1313 

1314 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1315 bias = self.ensureExposure(bias, camera, detectorNum) 

1316 dark = self.ensureExposure(dark, camera, detectorNum) 

1317 flat = self.ensureExposure(flat, camera, detectorNum) 

1318 

1319 ccd = ccdExposure.getDetector() 

1320 filterLabel = ccdExposure.getFilter() 

1321 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1322 

1323 if not ccd: 

1324 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1325 ccd = [FakeAmp(ccdExposure, self.config)] 

1326 

1327 # Validate Input 

1328 if self.config.doBias and bias is None: 

1329 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1330 if self.doLinearize(ccd) and linearizer is None: 

1331 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1332 if self.config.doBrighterFatter and bfKernel is None: 

1333 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1334 if self.config.doDark and dark is None: 

1335 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1336 if self.config.doFlat and flat is None: 

1337 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1338 if self.config.doDefect and defects is None: 

1339 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1340 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1341 and fringes.fringes is None): 

1342 # The `fringes` object needs to be a pipeBase.Struct, as 

1343 # we use it as a `dict` for the parameters of 

1344 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1345 # not be `None` if `doFringe=True`. Otherwise, raise. 

1346 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1347 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1348 and illumMaskedImage is None): 

1349 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1350 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1351 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1352 if (self.config.usePtcGains and ptc is None): 

1353 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

1354 if (self.config.usePtcReadNoise and ptc is None): 

1355 raise RuntimeError("No ptcDataset provided to use PTC read noise.") 

1356 

1357 # Validate that the inputs match the exposure configuration. 

1358 exposureMetadata = ccdExposure.getMetadata() 

1359 if self.config.doBias: 

1360 self.compareCameraKeywords(exposureMetadata, bias, "bias") 

1361 if self.config.doBrighterFatter: 

1362 self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter") 

1363 if self.config.doCrosstalk: 

1364 self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk") 

1365 if self.config.doDark: 

1366 self.compareCameraKeywords(exposureMetadata, dark, "dark") 

1367 if self.config.doDefect: 

1368 self.compareCameraKeywords(exposureMetadata, defects, "defects") 

1369 if self.config.doDeferredCharge: 

1370 self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI") 

1371 if self.config.doFlat: 

1372 self.compareCameraKeywords(exposureMetadata, flat, "flat") 

1373 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1374 self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe") 

1375 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1376 self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination") 

1377 if self.doLinearize(ccd): 

1378 self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer") 

1379 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1380 self.compareCameraKeywords(exposureMetadata, ptc, "PTC") 

1381 if self.config.doStrayLight: 

1382 self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight") 

1383 

1384 # Start in ADU. Update units to electrons when gain is applied: 

1385 # updateVariance, applyGains 

1386 # Check if needed during/after BFE correction, CTI correction. 

1387 exposureMetadata["LSST ISR UNITS"] = "ADU" 

1388 

1389 # Begin ISR processing. 

1390 if self.config.doConvertIntToFloat: 

1391 self.log.info("Converting exposure to floating point values.") 

1392 ccdExposure = self.convertIntToFloat(ccdExposure) 

1393 

1394 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1395 self.log.info("Applying bias correction.") 

1396 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1397 trimToFit=self.config.doTrimToMatchCalib) 

1398 self.debugView(ccdExposure, "doBias") 

1399 

1400 # Amplifier level processing. 

1401 overscans = [] 

1402 

1403 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1404 # This will attempt to mask bleed pixels across all amplifiers. 

1405 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1406 

1407 for amp in ccd: 

1408 # if ccdExposure is one amp, 

1409 # check for coverage to prevent performing ops multiple times 

1410 if ccdExposure.getBBox().contains(amp.getBBox()): 

1411 # Check for fully masked bad amplifiers, 

1412 # and generate masks for SUSPECT and SATURATED values. 

1413 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1414 

1415 if self.config.doOverscan and not badAmp: 

1416 # Overscan correction on amp-by-amp basis. 

1417 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1418 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1419 if overscanResults is not None and \ 

1420 self.config.qa is not None and self.config.qa.saveStats is True: 

1421 if isinstance(overscanResults.overscanMean, float): 

1422 # Only serial overscan was run 

1423 mean = overscanResults.overscanMean 

1424 sigma = overscanResults.overscanSigma 

1425 residMean = overscanResults.residualMean 

1426 residSigma = overscanResults.residualSigma 

1427 else: 

1428 # Both serial and parallel overscan were 

1429 # run. Only report serial here. 

1430 mean = overscanResults.overscanMean[0] 

1431 sigma = overscanResults.overscanSigma[0] 

1432 residMean = overscanResults.residualMean[0] 

1433 residSigma = overscanResults.residualSigma[0] 

1434 

1435 self.metadata[f"FIT MEDIAN {amp.getName()}"] = mean 

1436 self.metadata[f"FIT STDEV {amp.getName()}"] = sigma 

1437 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1438 amp.getName(), mean, sigma) 

1439 

1440 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = residMean 

1441 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = residSigma 

1442 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1443 amp.getName(), residMean, residSigma) 

1444 

1445 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1446 else: 

1447 if badAmp: 

1448 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1449 overscanResults = None 

1450 

1451 overscans.append(overscanResults if overscanResults is not None else None) 

1452 else: 

1453 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1454 

1455 # Define an effective PTC that will contain the gain and readout 

1456 # noise to be used throughout the ISR task. 

1457 ptc = self.defineEffectivePtc(ptc, ccd, bfGains, overscans, exposureMetadata) 

1458 

1459 if self.config.doDeferredCharge: 

1460 self.log.info("Applying deferred charge/CTI correction.") 

1461 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1462 self.debugView(ccdExposure, "doDeferredCharge") 

1463 

1464 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1465 self.log.info("Applying crosstalk correction.") 

1466 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1467 crosstalkSources=crosstalkSources, camera=camera) 

1468 self.debugView(ccdExposure, "doCrosstalk") 

1469 

1470 if self.config.doAssembleCcd: 

1471 self.log.info("Assembling CCD from amplifiers.") 

1472 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1473 

1474 if self.config.expectWcs and not ccdExposure.getWcs(): 

1475 self.log.warning("No WCS found in input exposure.") 

1476 self.debugView(ccdExposure, "doAssembleCcd") 

1477 

1478 ossThumb = None 

1479 if self.config.qa.doThumbnailOss: 

1480 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1481 

1482 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1483 self.log.info("Applying bias correction.") 

1484 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1485 trimToFit=self.config.doTrimToMatchCalib) 

1486 self.debugView(ccdExposure, "doBias") 

1487 

1488 if self.config.doVariance: 

1489 for amp in ccd: 

1490 if ccdExposure.getBBox().contains(amp.getBBox()): 

1491 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1492 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1493 self.updateVariance(ampExposure, amp, ptc) 

1494 

1495 if self.config.qa is not None and self.config.qa.saveStats is True: 

1496 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1497 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1498 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1499 qaStats.getValue(afwMath.MEDIAN) 

1500 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1501 qaStats.getValue(afwMath.STDEVCLIP) 

1502 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1503 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1504 qaStats.getValue(afwMath.STDEVCLIP)) 

1505 if self.config.maskNegativeVariance: 

1506 self.maskNegativeVariance(ccdExposure) 

1507 

1508 if self.doLinearize(ccd): 

1509 self.log.info("Applying linearizer.") 

1510 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1511 detector=ccd, log=self.log) 

1512 

1513 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1514 self.log.info("Applying crosstalk correction.") 

1515 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1516 crosstalkSources=crosstalkSources, isTrimmed=True) 

1517 self.debugView(ccdExposure, "doCrosstalk") 

1518 

1519 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1520 # widen trails, and do anything else the camera needs. Saturated and 

1521 # suspect pixels have already been masked. 

1522 if self.config.doDefect: 

1523 self.log.info("Masking defects.") 

1524 self.maskDefect(ccdExposure, defects) 

1525 

1526 if self.config.numEdgeSuspect > 0: 

1527 self.log.info("Masking edges as SUSPECT.") 

1528 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1529 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1530 

1531 if self.config.doNanMasking: 

1532 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1533 self.maskNan(ccdExposure) 

1534 

1535 if self.config.doWidenSaturationTrails: 

1536 self.log.info("Widening saturation trails.") 

1537 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1538 

1539 if self.config.doCameraSpecificMasking: 

1540 self.log.info("Masking regions for camera specific reasons.") 

1541 self.masking.run(ccdExposure) 

1542 

1543 if self.config.doBrighterFatter: 

1544 # We need to apply flats and darks before we can interpolate, and 

1545 # we need to interpolate before we do B-F, but we do B-F without 

1546 # the flats and darks applied so we can work in units of electrons 

1547 # or holes. This context manager applies and then removes the darks 

1548 # and flats. 

1549 # 

1550 # We also do not want to interpolate values here, so operate on 

1551 # temporary images so we can apply only the BF-correction and roll 

1552 # back the interpolation. 

1553 interpExp = ccdExposure.clone() 

1554 with self.flatContext(interpExp, flat, dark): 

1555 isrFunctions.interpolateFromMask( 

1556 maskedImage=interpExp.getMaskedImage(), 

1557 fwhm=self.config.fwhm, 

1558 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1559 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1560 ) 

1561 bfExp = interpExp.clone() 

1562 

1563 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1564 type(bfKernel), type(bfGains)) 

1565 if self.config.doFluxConservingBrighterFatterCorrection: 

1566 bfResults = isrFunctions.fluxConservingBrighterFatterCorrection( 

1567 bfExp, 

1568 bfKernel, 

1569 self.config.brighterFatterMaxIter, 

1570 self.config.brighterFatterThreshold, 

1571 self.config.brighterFatterApplyGain, 

1572 bfGains 

1573 ) 

1574 else: 

1575 bfResults = isrFunctions.brighterFatterCorrection( 

1576 bfExp, 

1577 bfKernel, 

1578 self.config.brighterFatterMaxIter, 

1579 self.config.brighterFatterThreshold, 

1580 self.config.brighterFatterApplyGain, 

1581 bfGains 

1582 ) 

1583 if bfResults[1] == self.config.brighterFatterMaxIter - 1: 

1584 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1585 bfResults[0]) 

1586 else: 

1587 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1588 bfResults[1]) 

1589 image = ccdExposure.getMaskedImage().getImage() 

1590 bfCorr = bfExp.getMaskedImage().getImage() 

1591 bfCorr -= interpExp.getMaskedImage().getImage() 

1592 image += bfCorr 

1593 

1594 # Applying the brighter-fatter correction applies a 

1595 # convolution to the science image. At the edges this 

1596 # convolution may not have sufficient valid pixels to 

1597 # produce a valid correction. Mark pixels within the size 

1598 # of the brighter-fatter kernel as EDGE to warn of this 

1599 # fact. 

1600 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1601 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1602 maskPlane="EDGE") 

1603 

1604 if self.config.brighterFatterMaskGrowSize > 0: 

1605 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1606 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1607 isrFunctions.growMasks(ccdExposure.getMask(), 

1608 radius=self.config.brighterFatterMaskGrowSize, 

1609 maskNameList=maskPlane, 

1610 maskValue=maskPlane) 

1611 

1612 self.debugView(ccdExposure, "doBrighterFatter") 

1613 

1614 if self.config.doDark: 

1615 self.log.info("Applying dark correction.") 

1616 self.darkCorrection(ccdExposure, dark) 

1617 self.debugView(ccdExposure, "doDark") 

1618 

1619 if self.config.doFringe and not self.config.fringeAfterFlat: 

1620 self.log.info("Applying fringe correction before flat.") 

1621 self.fringe.run(ccdExposure, **fringes.getDict()) 

1622 self.debugView(ccdExposure, "doFringe") 

1623 

1624 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1625 self.log.info("Checking strayLight correction.") 

1626 self.strayLight.run(ccdExposure, strayLightData) 

1627 self.debugView(ccdExposure, "doStrayLight") 

1628 

1629 if self.config.doFlat: 

1630 self.log.info("Applying flat correction.") 

1631 self.flatCorrection(ccdExposure, flat) 

1632 self.debugView(ccdExposure, "doFlat") 

1633 

1634 if self.config.doApplyGains: 

1635 self.log.info("Applying gain correction instead of flat.") 

1636 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1637 ptcGains=ptc.gain) 

1638 exposureMetadata["LSST ISR UNITS"] = "electrons" 

1639 

1640 if self.config.doFringe and self.config.fringeAfterFlat: 

1641 self.log.info("Applying fringe correction after flat.") 

1642 self.fringe.run(ccdExposure, **fringes.getDict()) 

1643 

1644 if self.config.doVignette: 

1645 if self.config.doMaskVignettePolygon: 

1646 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1647 else: 

1648 self.log.info("Constructing and attaching vignette polygon.") 

1649 self.vignettePolygon = self.vignette.run( 

1650 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1651 vignetteValue=self.config.vignetteValue, log=self.log) 

1652 

1653 if self.config.doAttachTransmissionCurve: 

1654 self.log.info("Adding transmission curves.") 

1655 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1656 filterTransmission=filterTransmission, 

1657 sensorTransmission=sensorTransmission, 

1658 atmosphereTransmission=atmosphereTransmission) 

1659 

1660 flattenedThumb = None 

1661 if self.config.qa.doThumbnailFlattened: 

1662 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1663 

1664 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1665 self.log.info("Performing illumination correction.") 

1666 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1667 illumMaskedImage, illumScale=self.config.illumScale, 

1668 trimToFit=self.config.doTrimToMatchCalib) 

1669 

1670 preInterpExp = None 

1671 if self.config.doSaveInterpPixels: 

1672 preInterpExp = ccdExposure.clone() 

1673 

1674 # Reset and interpolate bad pixels. 

1675 # 

1676 # Large contiguous bad regions (which should have the BAD mask 

1677 # bit set) should have their values set to the image median. 

1678 # This group should include defects and bad amplifiers. As the 

1679 # area covered by these defects are large, there's little 

1680 # reason to expect that interpolation would provide a more 

1681 # useful value. 

1682 # 

1683 # Smaller defects can be safely interpolated after the larger 

1684 # regions have had their pixel values reset. This ensures 

1685 # that the remaining defects adjacent to bad amplifiers (as an 

1686 # example) do not attempt to interpolate extreme values. 

1687 if self.config.doSetBadRegions: 

1688 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1689 if badPixelCount > 0: 

1690 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1691 

1692 if self.config.doInterpolate: 

1693 self.log.info("Interpolating masked pixels.") 

1694 isrFunctions.interpolateFromMask( 

1695 maskedImage=ccdExposure.getMaskedImage(), 

1696 fwhm=self.config.fwhm, 

1697 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1698 maskNameList=list(self.config.maskListToInterpolate) 

1699 ) 

1700 

1701 self.roughZeroPoint(ccdExposure) 

1702 

1703 # correct for amp offsets within the CCD 

1704 if self.config.doAmpOffset: 

1705 self.log.info("Correcting amp offsets.") 

1706 self.ampOffset.run(ccdExposure) 

1707 

1708 if self.config.doMeasureBackground: 

1709 self.log.info("Measuring background level.") 

1710 self.measureBackground(ccdExposure, self.config.qa) 

1711 

1712 if self.config.qa is not None and self.config.qa.saveStats is True: 

1713 for amp in ccd: 

1714 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1715 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1716 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1717 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1718 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1719 qaStats.getValue(afwMath.STDEVCLIP) 

1720 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1721 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1722 qaStats.getValue(afwMath.STDEVCLIP)) 

1723 

1724 # Calculate standard image quality statistics 

1725 if self.config.doStandardStatistics: 

1726 metadata = ccdExposure.getMetadata() 

1727 for amp in ccd: 

1728 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1729 ampName = amp.getName() 

1730 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1731 ampExposure.getMaskedImage(), 

1732 [self.config.saturatedMaskName] 

1733 ) 

1734 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1735 ampExposure.getMaskedImage(), 

1736 ["BAD"] 

1737 ) 

1738 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1739 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1740 

1741 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1742 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1743 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1744 

1745 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1746 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1747 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1748 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1749 else: 

1750 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1751 

1752 # calculate additional statistics. 

1753 outputStatistics = None 

1754 if self.config.doCalculateStatistics: 

1755 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1756 bias=bias, dark=dark, flat=flat, ptc=ptc).results 

1757 

1758 # do any binning. 

1759 outputBin1Exposure = None 

1760 outputBin2Exposure = None 

1761 if self.config.doBinnedExposures: 

1762 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure) 

1763 

1764 self.debugView(ccdExposure, "postISRCCD") 

1765 

1766 return pipeBase.Struct( 

1767 exposure=ccdExposure, 

1768 ossThumb=ossThumb, 

1769 flattenedThumb=flattenedThumb, 

1770 

1771 outputBin1Exposure=outputBin1Exposure, 

1772 outputBin2Exposure=outputBin2Exposure, 

1773 

1774 preInterpExposure=preInterpExp, 

1775 outputExposure=ccdExposure, 

1776 outputOssThumbnail=ossThumb, 

1777 outputFlattenedThumbnail=flattenedThumb, 

1778 outputStatistics=outputStatistics, 

1779 ) 

1780 

1781 def defineEffectivePtc(self, ptcDataset, detector, bfGains, overScans, metadata): 

1782 """Define an effective Photon Transfer Curve dataset 

1783 with nominal gains and noise. 

1784 

1785 Parameters 

1786 ------ 

1787 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset` 

1788 Input Photon Transfer Curve dataset. 

1789 detector : `lsst.afw.cameraGeom.Detector` 

1790 Detector object. 

1791 bfGains : `dict` 

1792 Gains from running the brighter-fatter code. 

1793 A dict keyed by amplifier name for the detector 

1794 in question. 

1795 ovserScans : `list` [`lsst.pipe.base.Struct`] 

1796 List of overscanResults structures 

1797 metadata : `lsst.daf.base.PropertyList` 

1798 Exposure metadata to update gain and noise provenance. 

1799 

1800 Returns 

1801 ------- 

1802 effectivePtc : `lsst.ip.isr.PhotonTransferCurveDataset` 

1803 PTC dataset containing gains and readout noise 

1804 values to be used throughout 

1805 Instrument Signature Removal. 

1806 """ 

1807 amps = detector.getAmplifiers() 

1808 ampNames = [amp.getName() for amp in amps] 

1809 detName = detector.getName() 

1810 effectivePtc = PhotonTransferCurveDataset(ampNames, 'EFFECTIVE_PTC', 1) 

1811 boolGainMismatch = False 

1812 doWarningPtcValidation = True 

1813 

1814 for amp, overscanResults in zip(amps, overScans): 

1815 ampName = amp.getName() 

1816 # Gain: 

1817 # Try first with the PTC gains. 

1818 gainProvenanceString = "amp" 

1819 if self.config.usePtcGains: 

1820 gain = ptcDataset.gain[ampName] 

1821 gainProvenanceString = "ptc" 

1822 self.log.debug("Using gain from Photon Transfer Curve.") 

1823 else: 

1824 # Try then with the amplifier gain. 

1825 # We already have a detector at this point. If there was no 

1826 # detector to begin with, one would have been created with 

1827 # self.config.gain and self.config.noise. Same comment 

1828 # applies for the noise block below. 

1829 gain = amp.getGain() 

1830 

1831 # Check if the gain up to this point differs from the 

1832 # gain in bfGains. If so, raise or warn, accordingly. 

1833 if not boolGainMismatch and bfGains is not None and ampName in bfGains: 

1834 bfGain = bfGains[ampName] 

1835 if not math.isclose(gain, bfGain, rel_tol=1e-4): 

1836 if self.config.doRaiseOnCalibMismatch: 

1837 raise RuntimeError("Gain mismatch for det %s amp %s: " 

1838 "(gain (%s): %s, bfGain: %s)", 

1839 detName, ampName, gainProvenanceString, 

1840 gain, bfGain) 

1841 else: 

1842 self.log.warning("Gain mismatch for det %s amp %s: " 

1843 "(gain (%s): %s, bfGain: %s)", 

1844 detName, ampName, gainProvenanceString, 

1845 gain, bfGain) 

1846 boolGainMismatch = True 

1847 

1848 # Noise: 

1849 # Try first with the empirical noise from the overscan. 

1850 noiseProvenanceString = "amp" 

1851 if self.config.doEmpiricalReadNoise and overscanResults is not None: 

1852 noiseProvenanceString = "serial overscan" 

1853 if isinstance(overscanResults.residualSigma, float): 

1854 # Only serial overscan was run 

1855 noise = overscanResults.residualSigma 

1856 else: 

1857 # Both serial and parallel overscan were 

1858 # run. Only report noise from serial here. 

1859 noise = overscanResults.residualSigma[0] 

1860 elif self.config.usePtcReadNoise: 

1861 # Try then with the PTC noise. 

1862 noise = ptcDataset.noise[amp.getName()] 

1863 noiseProvenanceString = "ptc" 

1864 self.log.debug("Using noise from Photon Transfer Curve.") 

1865 else: 

1866 # Finally, try with the amplifier noise. 

1867 # We already have a detector at this point. If there 

1868 # was no detector to begin with, one would have 

1869 # been created with self.config.gain and 

1870 # self.config.noise. 

1871 noise = amp.getReadNoise() 

1872 

1873 if math.isnan(gain): 

1874 gain = 1.0 

1875 self.log.warning("Gain for amp %s set to NAN! Updating to" 

1876 " 1.0 to generate Poisson variance.", ampName) 

1877 elif gain <= 0: 

1878 patchedGain = 1.0 

1879 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

1880 ampName, gain, patchedGain) 

1881 gain = patchedGain 

1882 

1883 effectivePtc.gain[ampName] = gain 

1884 effectivePtc.noise[ampName] = noise 

1885 # Make sure noise,turnoff, and gain make sense 

1886 effectivePtc.validateGainNoiseTurnoffValues(ampName, doWarn=doWarningPtcValidation) 

1887 doWarningPtcValidation = False 

1888 

1889 metadata[f"LSST GAIN {amp.getName()}"] = effectivePtc.gain[ampName] 

1890 metadata[f"LSST READNOISE {amp.getName()}"] = effectivePtc.noise[ampName] 

1891 

1892 self.log.info("Det: %s - Noise provenance: %s, Gain provenance: %s", 

1893 detName, 

1894 noiseProvenanceString, 

1895 gainProvenanceString) 

1896 metadata["LSST ISR GAIN SOURCE"] = gainProvenanceString 

1897 metadata["LSST ISR NOISE SOURCE"] = noiseProvenanceString 

1898 

1899 return effectivePtc 

1900 

1901 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1902 """Ensure that the data returned by Butler is a fully constructed exp. 

1903 

1904 ISR requires exposure-level image data for historical reasons, so if we 

1905 did not recieve that from Butler, construct it from what we have, 

1906 modifying the input in place. 

1907 

1908 Parameters 

1909 ---------- 

1910 inputExp : `lsst.afw.image` image-type. 

1911 The input data structure obtained from Butler. 

1912 Can be `lsst.afw.image.Exposure`, 

1913 `lsst.afw.image.DecoratedImageU`, 

1914 or `lsst.afw.image.ImageF` 

1915 camera : `lsst.afw.cameraGeom.camera`, optional 

1916 The camera associated with the image. Used to find the appropriate 

1917 detector if detector is not already set. 

1918 detectorNum : `int`, optional 

1919 The detector in the camera to attach, if the detector is not 

1920 already set. 

1921 

1922 Returns 

1923 ------- 

1924 inputExp : `lsst.afw.image.Exposure` 

1925 The re-constructed exposure, with appropriate detector parameters. 

1926 

1927 Raises 

1928 ------ 

1929 TypeError 

1930 Raised if the input data cannot be used to construct an exposure. 

1931 """ 

1932 if isinstance(inputExp, afwImage.DecoratedImageU): 

1933 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1934 elif isinstance(inputExp, afwImage.ImageF): 

1935 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1936 elif isinstance(inputExp, afwImage.MaskedImageF): 

1937 inputExp = afwImage.makeExposure(inputExp) 

1938 elif isinstance(inputExp, afwImage.Exposure): 

1939 pass 

1940 elif inputExp is None: 

1941 # Assume this will be caught by the setup if it is a problem. 

1942 return inputExp 

1943 else: 

1944 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1945 (type(inputExp), )) 

1946 

1947 if inputExp.getDetector() is None: 

1948 if camera is None or detectorNum is None: 

1949 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1950 'without a detector set.') 

1951 inputExp.setDetector(camera[detectorNum]) 

1952 

1953 return inputExp 

1954 

1955 @staticmethod 

1956 def extractCalibDate(calib): 

1957 """Extract common calibration metadata values that will be written to 

1958 output header. 

1959 

1960 Parameters 

1961 ---------- 

1962 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1963 Calibration to pull date information from. 

1964 

1965 Returns 

1966 ------- 

1967 dateString : `str` 

1968 Calibration creation date string to add to header. 

1969 """ 

1970 if hasattr(calib, "getMetadata"): 

1971 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1972 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1973 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1974 else: 

1975 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1976 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1977 else: 

1978 return "Unknown Unknown" 

1979 

1980 def compareCameraKeywords(self, exposureMetadata, calib, calibName): 

1981 """Compare header keywords to confirm camera states match. 

1982 

1983 Parameters 

1984 ---------- 

1985 exposureMetadata : `lsst.daf.base.PropertySet` 

1986 Header for the exposure being processed. 

1987 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1988 Calibration to be applied. 

1989 calibName : `str` 

1990 Calib type for log message. 

1991 """ 

1992 try: 

1993 calibMetadata = calib.getMetadata() 

1994 except AttributeError: 

1995 return 

1996 for keyword in self.config.cameraKeywordsToCompare: 

1997 if keyword in exposureMetadata and keyword in calibMetadata: 

1998 if exposureMetadata[keyword] != calibMetadata[keyword]: 

1999 if self.config.doRaiseOnCalibMismatch: 

2000 raise RuntimeError("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

2001 calibName, keyword, 

2002 exposureMetadata[keyword], calibMetadata[keyword]) 

2003 else: 

2004 self.log.warning("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

2005 calibName, keyword, 

2006 exposureMetadata[keyword], calibMetadata[keyword]) 

2007 else: 

2008 self.log.debug("Sequencer keyword %s not found.", keyword) 

2009 

2010 def convertIntToFloat(self, exposure): 

2011 """Convert exposure image from uint16 to float. 

2012 

2013 If the exposure does not need to be converted, the input is 

2014 immediately returned. For exposures that are converted to use 

2015 floating point pixels, the variance is set to unity and the 

2016 mask to zero. 

2017 

2018 Parameters 

2019 ---------- 

2020 exposure : `lsst.afw.image.Exposure` 

2021 The raw exposure to be converted. 

2022 

2023 Returns 

2024 ------- 

2025 newexposure : `lsst.afw.image.Exposure` 

2026 The input ``exposure``, converted to floating point pixels. 

2027 

2028 Raises 

2029 ------ 

2030 RuntimeError 

2031 Raised if the exposure type cannot be converted to float. 

2032 

2033 """ 

2034 if isinstance(exposure, afwImage.ExposureF): 

2035 # Nothing to be done 

2036 self.log.debug("Exposure already of type float.") 

2037 return exposure 

2038 if not hasattr(exposure, "convertF"): 

2039 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

2040 

2041 newexposure = exposure.convertF() 

2042 newexposure.variance[:] = 1 

2043 newexposure.mask[:] = 0x0 

2044 

2045 return newexposure 

2046 

2047 def maskAmplifier(self, ccdExposure, amp, defects): 

2048 """Identify bad amplifiers, saturated and suspect pixels. 

2049 

2050 Parameters 

2051 ---------- 

2052 ccdExposure : `lsst.afw.image.Exposure` 

2053 Input exposure to be masked. 

2054 amp : `lsst.afw.cameraGeom.Amplifier` 

2055 Catalog of parameters defining the amplifier on this 

2056 exposure to mask. 

2057 defects : `lsst.ip.isr.Defects` 

2058 List of defects. Used to determine if the entire 

2059 amplifier is bad. 

2060 

2061 Returns 

2062 ------- 

2063 badAmp : `Bool` 

2064 If this is true, the entire amplifier area is covered by 

2065 defects and unusable. 

2066 

2067 """ 

2068 maskedImage = ccdExposure.getMaskedImage() 

2069 

2070 badAmp = False 

2071 

2072 # Check if entire amp region is defined as a defect 

2073 # NB: need to use amp.getBBox() for correct comparison with current 

2074 # defects definition. 

2075 if defects is not None: 

2076 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

2077 

2078 # In the case of a bad amp, we will set mask to "BAD" 

2079 # (here use amp.getRawBBox() for correct association with pixels in 

2080 # current ccdExposure). 

2081 if badAmp: 

2082 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

2083 afwImage.PARENT) 

2084 maskView = dataView.getMask() 

2085 maskView |= maskView.getPlaneBitMask("BAD") 

2086 del maskView 

2087 return badAmp 

2088 

2089 # Mask remaining defects after assembleCcd() to allow for defects that 

2090 # cross amplifier boundaries. Saturation and suspect pixels can be 

2091 # masked now, though. 

2092 limits = dict() 

2093 if self.config.doSaturation and not badAmp: 

2094 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

2095 if self.config.doSuspect and not badAmp: 

2096 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

2097 if math.isfinite(self.config.saturation): 

2098 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

2099 

2100 for maskName, maskThreshold in limits.items(): 

2101 if not math.isnan(maskThreshold): 

2102 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2103 isrFunctions.makeThresholdMask( 

2104 maskedImage=dataView, 

2105 threshold=maskThreshold, 

2106 growFootprints=0, 

2107 maskName=maskName 

2108 ) 

2109 

2110 # Determine if we've fully masked this amplifier with SUSPECT and 

2111 # SAT pixels. 

2112 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

2113 afwImage.PARENT) 

2114 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

2115 self.config.suspectMaskName]) 

2116 if numpy.all(maskView.getArray() & maskVal > 0): 

2117 badAmp = True 

2118 maskView |= maskView.getPlaneBitMask("BAD") 

2119 

2120 return badAmp 

2121 

2122 def overscanCorrection(self, ccdExposure, amp): 

2123 """Apply overscan correction in place. 

2124 

2125 This method does initial pixel rejection of the overscan 

2126 region. The overscan can also be optionally segmented to 

2127 allow for discontinuous overscan responses to be fit 

2128 separately. The actual overscan subtraction is performed by 

2129 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

2130 after the amplifier is preprocessed. 

2131 

2132 Parameters 

2133 ---------- 

2134 ccdExposure : `lsst.afw.image.Exposure` 

2135 Exposure to have overscan correction performed. 

2136 amp : `lsst.afw.cameraGeom.Amplifer` 

2137 The amplifier to consider while correcting the overscan. 

2138 

2139 Returns 

2140 ------- 

2141 overscanResults : `lsst.pipe.base.Struct` 

2142 Result struct with components: 

2143 

2144 ``imageFit`` 

2145 Value or fit subtracted from the amplifier image data. 

2146 (scalar or `lsst.afw.image.Image`) 

2147 ``overscanFit`` 

2148 Value or fit subtracted from the overscan image data. 

2149 (scalar or `lsst.afw.image.Image`) 

2150 ``overscanImage`` 

2151 Image of the overscan region with the overscan 

2152 correction applied. This quantity is used to estimate 

2153 the amplifier read noise empirically. 

2154 (`lsst.afw.image.Image`) 

2155 ``edgeMask`` 

2156 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

2157 ``overscanMean`` 

2158 Median overscan fit value. (`float`) 

2159 ``overscanSigma`` 

2160 Clipped standard deviation of the overscan after 

2161 correction. (`float`) 

2162 

2163 Raises 

2164 ------ 

2165 RuntimeError 

2166 Raised if the ``amp`` does not contain raw pixel information. 

2167 

2168 See Also 

2169 -------- 

2170 lsst.ip.isr.overscan.OverscanTask 

2171 """ 

2172 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2173 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2174 return None 

2175 

2176 # Perform overscan correction on subregions. 

2177 overscanResults = self.overscan.run(ccdExposure, amp) 

2178 

2179 metadata = ccdExposure.getMetadata() 

2180 ampName = amp.getName() 

2181 

2182 keyBase = "LSST ISR OVERSCAN" 

2183 # Updated quantities 

2184 if isinstance(overscanResults.overscanMean, float): 

2185 # Serial overscan correction only: 

2186 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

2187 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

2188 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

2189 

2190 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

2191 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

2192 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

2193 elif isinstance(overscanResults.overscanMean, tuple): 

2194 # Both serial and parallel overscan have run: 

2195 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

2196 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

2197 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

2198 

2199 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

2200 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

2201 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

2202 

2203 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

2204 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

2205 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

2206 

2207 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

2208 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

2209 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

2210 else: 

2211 self.log.warning("Unexpected type for overscan values; none added to header.") 

2212 

2213 return overscanResults 

2214 

2215 def updateVariance(self, ampExposure, amp, ptcDataset): 

2216 """Set the variance plane using the gain and read noise 

2217 

2218 The read noise is calculated from the ``overscanImage`` if the 

2219 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2220 the value from the amplifier data is used. 

2221 

2222 Parameters 

2223 ---------- 

2224 ampExposure : `lsst.afw.image.Exposure` 

2225 Exposure to process. 

2226 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

2227 Amplifier detector data. 

2228 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset` 

2229 Effective PTC dataset containing the gains and read noise. 

2230 

2231 See also 

2232 -------- 

2233 lsst.ip.isr.isrFunctions.updateVariance 

2234 """ 

2235 ampName = amp.getName() 

2236 # At this point, the effective PTC should have 

2237 # gain and noise values. 

2238 gain = ptcDataset.gain[ampName] 

2239 readNoise = ptcDataset.noise[ampName] 

2240 

2241 isrFunctions.updateVariance( 

2242 maskedImage=ampExposure.getMaskedImage(), 

2243 gain=gain, 

2244 readNoise=readNoise, 

2245 ) 

2246 

2247 def maskNegativeVariance(self, exposure): 

2248 """Identify and mask pixels with negative variance values. 

2249 

2250 Parameters 

2251 ---------- 

2252 exposure : `lsst.afw.image.Exposure` 

2253 Exposure to process. 

2254 

2255 See Also 

2256 -------- 

2257 lsst.ip.isr.isrFunctions.updateVariance 

2258 """ 

2259 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2260 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2261 exposure.mask.array[bad] |= maskPlane 

2262 

2263 def darkCorrection(self, exposure, darkExposure, invert=False): 

2264 """Apply dark correction in place. 

2265 

2266 Parameters 

2267 ---------- 

2268 exposure : `lsst.afw.image.Exposure` 

2269 Exposure to process. 

2270 darkExposure : `lsst.afw.image.Exposure` 

2271 Dark exposure of the same size as ``exposure``. 

2272 invert : `Bool`, optional 

2273 If True, re-add the dark to an already corrected image. 

2274 

2275 Raises 

2276 ------ 

2277 RuntimeError 

2278 Raised if either ``exposure`` or ``darkExposure`` do not 

2279 have their dark time defined. 

2280 

2281 See Also 

2282 -------- 

2283 lsst.ip.isr.isrFunctions.darkCorrection 

2284 """ 

2285 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2286 if math.isnan(expScale): 

2287 raise RuntimeError("Exposure darktime is NAN.") 

2288 if darkExposure.getInfo().getVisitInfo() is not None \ 

2289 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2290 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2291 else: 

2292 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2293 # so getDarkTime() does not exist. 

2294 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2295 darkScale = 1.0 

2296 

2297 isrFunctions.darkCorrection( 

2298 maskedImage=exposure.getMaskedImage(), 

2299 darkMaskedImage=darkExposure.getMaskedImage(), 

2300 expScale=expScale, 

2301 darkScale=darkScale, 

2302 invert=invert, 

2303 trimToFit=self.config.doTrimToMatchCalib 

2304 ) 

2305 

2306 def doLinearize(self, detector): 

2307 """Check if linearization is needed for the detector cameraGeom. 

2308 

2309 Checks config.doLinearize and the linearity type of the first 

2310 amplifier. 

2311 

2312 Parameters 

2313 ---------- 

2314 detector : `lsst.afw.cameraGeom.Detector` 

2315 Detector to get linearity type from. 

2316 

2317 Returns 

2318 ------- 

2319 doLinearize : `Bool` 

2320 If True, linearization should be performed. 

2321 """ 

2322 return self.config.doLinearize and \ 

2323 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2324 

2325 def flatCorrection(self, exposure, flatExposure, invert=False): 

2326 """Apply flat correction in place. 

2327 

2328 Parameters 

2329 ---------- 

2330 exposure : `lsst.afw.image.Exposure` 

2331 Exposure to process. 

2332 flatExposure : `lsst.afw.image.Exposure` 

2333 Flat exposure of the same size as ``exposure``. 

2334 invert : `Bool`, optional 

2335 If True, unflatten an already flattened image. 

2336 

2337 See Also 

2338 -------- 

2339 lsst.ip.isr.isrFunctions.flatCorrection 

2340 """ 

2341 isrFunctions.flatCorrection( 

2342 maskedImage=exposure.getMaskedImage(), 

2343 flatMaskedImage=flatExposure.getMaskedImage(), 

2344 scalingType=self.config.flatScalingType, 

2345 userScale=self.config.flatUserScale, 

2346 invert=invert, 

2347 trimToFit=self.config.doTrimToMatchCalib 

2348 ) 

2349 

2350 def saturationDetection(self, exposure, amp): 

2351 """Detect and mask saturated pixels in config.saturatedMaskName. 

2352 

2353 Parameters 

2354 ---------- 

2355 exposure : `lsst.afw.image.Exposure` 

2356 Exposure to process. Only the amplifier DataSec is processed. 

2357 amp : `lsst.afw.cameraGeom.Amplifier` 

2358 Amplifier detector data. 

2359 

2360 See Also 

2361 -------- 

2362 lsst.ip.isr.isrFunctions.makeThresholdMask 

2363 """ 

2364 if not math.isnan(amp.getSaturation()): 

2365 maskedImage = exposure.getMaskedImage() 

2366 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2367 isrFunctions.makeThresholdMask( 

2368 maskedImage=dataView, 

2369 threshold=amp.getSaturation(), 

2370 growFootprints=0, 

2371 maskName=self.config.saturatedMaskName, 

2372 ) 

2373 

2374 def saturationInterpolation(self, exposure): 

2375 """Interpolate over saturated pixels, in place. 

2376 

2377 This method should be called after `saturationDetection`, to 

2378 ensure that the saturated pixels have been identified in the 

2379 SAT mask. It should also be called after `assembleCcd`, since 

2380 saturated regions may cross amplifier boundaries. 

2381 

2382 Parameters 

2383 ---------- 

2384 exposure : `lsst.afw.image.Exposure` 

2385 Exposure to process. 

2386 

2387 See Also 

2388 -------- 

2389 lsst.ip.isr.isrTask.saturationDetection 

2390 lsst.ip.isr.isrFunctions.interpolateFromMask 

2391 """ 

2392 isrFunctions.interpolateFromMask( 

2393 maskedImage=exposure.getMaskedImage(), 

2394 fwhm=self.config.fwhm, 

2395 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2396 maskNameList=list(self.config.saturatedMaskName), 

2397 ) 

2398 

2399 def suspectDetection(self, exposure, amp): 

2400 """Detect and mask suspect pixels in config.suspectMaskName. 

2401 

2402 Parameters 

2403 ---------- 

2404 exposure : `lsst.afw.image.Exposure` 

2405 Exposure to process. Only the amplifier DataSec is processed. 

2406 amp : `lsst.afw.cameraGeom.Amplifier` 

2407 Amplifier detector data. 

2408 

2409 See Also 

2410 -------- 

2411 lsst.ip.isr.isrFunctions.makeThresholdMask 

2412 

2413 Notes 

2414 ----- 

2415 Suspect pixels are pixels whose value is greater than 

2416 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2417 affected by unknown systematics; for example if non-linearity 

2418 corrections above a certain level are unstable then that would be a 

2419 useful value for suspectLevel. A value of `nan` indicates that no such 

2420 level exists and no pixels are to be masked as suspicious. 

2421 """ 

2422 suspectLevel = amp.getSuspectLevel() 

2423 if math.isnan(suspectLevel): 

2424 return 

2425 

2426 maskedImage = exposure.getMaskedImage() 

2427 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2428 isrFunctions.makeThresholdMask( 

2429 maskedImage=dataView, 

2430 threshold=suspectLevel, 

2431 growFootprints=0, 

2432 maskName=self.config.suspectMaskName, 

2433 ) 

2434 

2435 def maskDefect(self, exposure, defectBaseList): 

2436 """Mask defects using mask plane "BAD", in place. 

2437 

2438 Parameters 

2439 ---------- 

2440 exposure : `lsst.afw.image.Exposure` 

2441 Exposure to process. 

2442 defectBaseList : defect-type 

2443 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2444 or `list` of `lsst.afw.image.DefectBase`. 

2445 

2446 Notes 

2447 ----- 

2448 Call this after CCD assembly, since defects may cross amplifier 

2449 boundaries. 

2450 """ 

2451 maskedImage = exposure.getMaskedImage() 

2452 if not isinstance(defectBaseList, Defects): 

2453 # Promotes DefectBase to Defect 

2454 defectList = Defects(defectBaseList) 

2455 else: 

2456 defectList = defectBaseList 

2457 defectList.maskPixels(maskedImage, maskName="BAD") 

2458 

2459 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2460 """Mask edge pixels with applicable mask plane. 

2461 

2462 Parameters 

2463 ---------- 

2464 exposure : `lsst.afw.image.Exposure` 

2465 Exposure to process. 

2466 numEdgePixels : `int`, optional 

2467 Number of edge pixels to mask. 

2468 maskPlane : `str`, optional 

2469 Mask plane name to use. 

2470 level : `str`, optional 

2471 Level at which to mask edges. 

2472 """ 

2473 maskedImage = exposure.getMaskedImage() 

2474 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2475 

2476 if numEdgePixels > 0: 

2477 if level == 'DETECTOR': 

2478 boxes = [maskedImage.getBBox()] 

2479 elif level == 'AMP': 

2480 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2481 

2482 for box in boxes: 

2483 # This makes a bbox numEdgeSuspect pixels smaller than the 

2484 # image on each side 

2485 subImage = maskedImage[box] 

2486 box.grow(-numEdgePixels) 

2487 # Mask pixels outside box 

2488 SourceDetectionTask.setEdgeBits( 

2489 subImage, 

2490 box, 

2491 maskBitMask) 

2492 

2493 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2494 """Mask and interpolate defects using mask plane "BAD", in place. 

2495 

2496 Parameters 

2497 ---------- 

2498 exposure : `lsst.afw.image.Exposure` 

2499 Exposure to process. 

2500 defectBaseList : defects-like 

2501 List of defects to mask and interpolate. Can be 

2502 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2503 

2504 See Also 

2505 -------- 

2506 lsst.ip.isr.isrTask.maskDefect 

2507 """ 

2508 self.maskDefect(exposure, defectBaseList) 

2509 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2510 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2511 isrFunctions.interpolateFromMask( 

2512 maskedImage=exposure.getMaskedImage(), 

2513 fwhm=self.config.fwhm, 

2514 growSaturatedFootprints=0, 

2515 maskNameList=["BAD"], 

2516 ) 

2517 

2518 def maskNan(self, exposure): 

2519 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2520 

2521 Parameters 

2522 ---------- 

2523 exposure : `lsst.afw.image.Exposure` 

2524 Exposure to process. 

2525 

2526 Notes 

2527 ----- 

2528 We mask over all non-finite values (NaN, inf), including those 

2529 that are masked with other bits (because those may or may not be 

2530 interpolated over later, and we want to remove all NaN/infs). 

2531 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2532 preserve the historical name. 

2533 """ 

2534 maskedImage = exposure.getMaskedImage() 

2535 

2536 # Find and mask NaNs 

2537 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2538 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2539 numNans = maskNans(maskedImage, maskVal) 

2540 self.metadata["NUMNANS"] = numNans 

2541 if numNans > 0: 

2542 self.log.warning("There were %d unmasked NaNs.", numNans) 

2543 

2544 def maskAndInterpolateNan(self, exposure): 

2545 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2546 in place. 

2547 

2548 Parameters 

2549 ---------- 

2550 exposure : `lsst.afw.image.Exposure` 

2551 Exposure to process. 

2552 

2553 See Also 

2554 -------- 

2555 lsst.ip.isr.isrTask.maskNan 

2556 """ 

2557 self.maskNan(exposure) 

2558 isrFunctions.interpolateFromMask( 

2559 maskedImage=exposure.getMaskedImage(), 

2560 fwhm=self.config.fwhm, 

2561 growSaturatedFootprints=0, 

2562 maskNameList=["UNMASKEDNAN"], 

2563 ) 

2564 

2565 def measureBackground(self, exposure, IsrQaConfig=None): 

2566 """Measure the image background in subgrids, for quality control. 

2567 

2568 Parameters 

2569 ---------- 

2570 exposure : `lsst.afw.image.Exposure` 

2571 Exposure to process. 

2572 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2573 Configuration object containing parameters on which background 

2574 statistics and subgrids to use. 

2575 """ 

2576 if IsrQaConfig is not None: 

2577 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2578 IsrQaConfig.flatness.nIter) 

2579 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2580 statsControl.setAndMask(maskVal) 

2581 maskedImage = exposure.getMaskedImage() 

2582 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2583 skyLevel = stats.getValue(afwMath.MEDIAN) 

2584 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2585 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2586 metadata = exposure.getMetadata() 

2587 metadata["SKYLEVEL"] = skyLevel 

2588 metadata["SKYSIGMA"] = skySigma 

2589 

2590 # calcluating flatlevel over the subgrids 

2591 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2592 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2593 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2594 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2595 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2596 skyLevels = numpy.zeros((nX, nY)) 

2597 

2598 for j in range(nY): 

2599 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2600 for i in range(nX): 

2601 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2602 

2603 xLLC = xc - meshXHalf 

2604 yLLC = yc - meshYHalf 

2605 xURC = xc + meshXHalf - 1 

2606 yURC = yc + meshYHalf - 1 

2607 

2608 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2609 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2610 

2611 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2612 

2613 good = numpy.where(numpy.isfinite(skyLevels)) 

2614 skyMedian = numpy.median(skyLevels[good]) 

2615 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2616 flatness_rms = numpy.std(flatness) 

2617 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2618 

2619 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2620 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2621 nX, nY, flatness_pp, flatness_rms) 

2622 

2623 metadata["FLATNESS_PP"] = float(flatness_pp) 

2624 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2625 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2626 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2627 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2628 

2629 def roughZeroPoint(self, exposure): 

2630 """Set an approximate magnitude zero point for the exposure. 

2631 

2632 Parameters 

2633 ---------- 

2634 exposure : `lsst.afw.image.Exposure` 

2635 Exposure to process. 

2636 """ 

2637 filterLabel = exposure.getFilter() 

2638 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2639 

2640 if physicalFilter in self.config.fluxMag0T1: 

2641 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2642 else: 

2643 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2644 fluxMag0 = self.config.defaultFluxMag0T1 

2645 

2646 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2647 if not expTime > 0: # handle NaN as well as <= 0 

2648 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2649 return 

2650 

2651 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2652 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2653 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2654 

2655 @contextmanager 

2656 def flatContext(self, exp, flat, dark=None): 

2657 """Context manager that applies and removes flats and darks, 

2658 if the task is configured to apply them. 

2659 

2660 Parameters 

2661 ---------- 

2662 exp : `lsst.afw.image.Exposure` 

2663 Exposure to process. 

2664 flat : `lsst.afw.image.Exposure` 

2665 Flat exposure the same size as ``exp``. 

2666 dark : `lsst.afw.image.Exposure`, optional 

2667 Dark exposure the same size as ``exp``. 

2668 

2669 Yields 

2670 ------ 

2671 exp : `lsst.afw.image.Exposure` 

2672 The flat and dark corrected exposure. 

2673 """ 

2674 if self.config.doDark and dark is not None: 

2675 self.darkCorrection(exp, dark) 

2676 if self.config.doFlat: 

2677 self.flatCorrection(exp, flat) 

2678 try: 

2679 yield exp 

2680 finally: 

2681 if self.config.doFlat: 

2682 self.flatCorrection(exp, flat, invert=True) 

2683 if self.config.doDark and dark is not None: 

2684 self.darkCorrection(exp, dark, invert=True) 

2685 

2686 def makeBinnedImages(self, exposure): 

2687 """Make visualizeVisit style binned exposures. 

2688 

2689 Parameters 

2690 ---------- 

2691 exposure : `lsst.afw.image.Exposure` 

2692 Exposure to bin. 

2693 

2694 Returns 

2695 ------- 

2696 bin1 : `lsst.afw.image.Exposure` 

2697 Binned exposure using binFactor1. 

2698 bin2 : `lsst.afw.image.Exposure` 

2699 Binned exposure using binFactor2. 

2700 """ 

2701 mi = exposure.getMaskedImage() 

2702 

2703 bin1 = afwMath.binImage(mi, self.config.binFactor1) 

2704 bin2 = afwMath.binImage(mi, self.config.binFactor2) 

2705 

2706 return bin1, bin2 

2707 

2708 def debugView(self, exposure, stepname): 

2709 """Utility function to examine ISR exposure at different stages. 

2710 

2711 Parameters 

2712 ---------- 

2713 exposure : `lsst.afw.image.Exposure` 

2714 Exposure to view. 

2715 stepname : `str` 

2716 State of processing to view. 

2717 """ 

2718 frame = getDebugFrame(self._display, stepname) 

2719 if frame: 

2720 display = getDisplay(frame) 

2721 display.scale('asinh', 'zscale') 

2722 display.mtv(exposure) 

2723 prompt = "Press Enter to continue [c]... " 

2724 while True: 

2725 ans = input(prompt).lower() 

2726 if ans in ("", "c",): 

2727 break 

2728 

2729 

2730class FakeAmp(object): 

2731 """A Detector-like object that supports returning gain and saturation level 

2732 

2733 This is used when the input exposure does not have a detector. 

2734 

2735 Parameters 

2736 ---------- 

2737 exposure : `lsst.afw.image.Exposure` 

2738 Exposure to generate a fake amplifier for. 

2739 config : `lsst.ip.isr.isrTaskConfig` 

2740 Configuration to apply to the fake amplifier. 

2741 """ 

2742 

2743 def __init__(self, exposure, config): 

2744 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2745 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2746 self._gain = config.gain 

2747 self._readNoise = config.readNoise 

2748 self._saturation = config.saturation 

2749 

2750 def getBBox(self): 

2751 return self._bbox 

2752 

2753 def getRawBBox(self): 

2754 return self._bbox 

2755 

2756 def getRawHorizontalOverscanBBox(self): 

2757 return self._RawHorizontalOverscanBBox 

2758 

2759 def getGain(self): 

2760 return self._gain 

2761 

2762 def getReadNoise(self): 

2763 return self._readNoise 

2764 

2765 def getSaturation(self): 

2766 return self._saturation 

2767 

2768 def getSuspectLevel(self): 

2769 return float("NaN")