Coverage for python/lsst/ip/isr/isrTask.py: 16%

916 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-12 10:46 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from lsst.daf.butler import DimensionGraph 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

75 Data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

95 

96 

97class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

98 dimensions={"instrument", "exposure", "detector"}, 

99 defaultTemplates={}): 

100 ccdExposure = cT.Input( 

101 name="raw", 

102 doc="Input exposure to process.", 

103 storageClass="Exposure", 

104 dimensions=["instrument", "exposure", "detector"], 

105 ) 

106 camera = cT.PrerequisiteInput( 

107 name="camera", 

108 storageClass="Camera", 

109 doc="Input camera to construct complete exposures.", 

110 dimensions=["instrument"], 

111 isCalibration=True, 

112 ) 

113 

114 crosstalk = cT.PrerequisiteInput( 

115 name="crosstalk", 

116 doc="Input crosstalk object", 

117 storageClass="CrosstalkCalib", 

118 dimensions=["instrument", "detector"], 

119 isCalibration=True, 

120 minimum=0, # can fall back to cameraGeom 

121 ) 

122 crosstalkSources = cT.PrerequisiteInput( 

123 name="isrOverscanCorrected", 

124 doc="Overscan corrected input images.", 

125 storageClass="Exposure", 

126 dimensions=["instrument", "exposure", "detector"], 

127 deferLoad=True, 

128 multiple=True, 

129 lookupFunction=crosstalkSourceLookup, 

130 minimum=0, # not needed for all instruments, no config to control this 

131 ) 

132 bias = cT.PrerequisiteInput( 

133 name="bias", 

134 doc="Input bias calibration.", 

135 storageClass="ExposureF", 

136 dimensions=["instrument", "detector"], 

137 isCalibration=True, 

138 ) 

139 dark = cT.PrerequisiteInput( 

140 name='dark', 

141 doc="Input dark calibration.", 

142 storageClass="ExposureF", 

143 dimensions=["instrument", "detector"], 

144 isCalibration=True, 

145 ) 

146 flat = cT.PrerequisiteInput( 

147 name="flat", 

148 doc="Input flat calibration.", 

149 storageClass="ExposureF", 

150 dimensions=["instrument", "physical_filter", "detector"], 

151 isCalibration=True, 

152 ) 

153 ptc = cT.PrerequisiteInput( 

154 name="ptc", 

155 doc="Input Photon Transfer Curve dataset", 

156 storageClass="PhotonTransferCurveDataset", 

157 dimensions=["instrument", "detector"], 

158 isCalibration=True, 

159 ) 

160 fringes = cT.PrerequisiteInput( 

161 name="fringe", 

162 doc="Input fringe calibration.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "physical_filter", "detector"], 

165 isCalibration=True, 

166 minimum=0, # only needed for some bands, even when enabled 

167 ) 

168 strayLightData = cT.PrerequisiteInput( 

169 name='yBackground', 

170 doc="Input stray light calibration.", 

171 storageClass="StrayLightData", 

172 dimensions=["instrument", "physical_filter", "detector"], 

173 deferLoad=True, 

174 isCalibration=True, 

175 minimum=0, # only needed for some bands, even when enabled 

176 ) 

177 bfKernel = cT.PrerequisiteInput( 

178 name='bfKernel', 

179 doc="Input brighter-fatter kernel.", 

180 storageClass="NumpyArray", 

181 dimensions=["instrument"], 

182 isCalibration=True, 

183 minimum=0, # can use either bfKernel or newBFKernel 

184 ) 

185 newBFKernel = cT.PrerequisiteInput( 

186 name='brighterFatterKernel', 

187 doc="Newer complete kernel + gain solutions.", 

188 storageClass="BrighterFatterKernel", 

189 dimensions=["instrument", "detector"], 

190 isCalibration=True, 

191 minimum=0, # can use either bfKernel or newBFKernel 

192 ) 

193 defects = cT.PrerequisiteInput( 

194 name='defects', 

195 doc="Input defect tables.", 

196 storageClass="Defects", 

197 dimensions=["instrument", "detector"], 

198 isCalibration=True, 

199 ) 

200 linearizer = cT.PrerequisiteInput( 

201 name='linearizer', 

202 storageClass="Linearizer", 

203 doc="Linearity correction calibration.", 

204 dimensions=["instrument", "detector"], 

205 isCalibration=True, 

206 minimum=0, # can fall back to cameraGeom 

207 ) 

208 opticsTransmission = cT.PrerequisiteInput( 

209 name="transmission_optics", 

210 storageClass="TransmissionCurve", 

211 doc="Transmission curve due to the optics.", 

212 dimensions=["instrument"], 

213 isCalibration=True, 

214 ) 

215 filterTransmission = cT.PrerequisiteInput( 

216 name="transmission_filter", 

217 storageClass="TransmissionCurve", 

218 doc="Transmission curve due to the filter.", 

219 dimensions=["instrument", "physical_filter"], 

220 isCalibration=True, 

221 ) 

222 sensorTransmission = cT.PrerequisiteInput( 

223 name="transmission_sensor", 

224 storageClass="TransmissionCurve", 

225 doc="Transmission curve due to the sensor.", 

226 dimensions=["instrument", "detector"], 

227 isCalibration=True, 

228 ) 

229 atmosphereTransmission = cT.PrerequisiteInput( 

230 name="transmission_atmosphere", 

231 storageClass="TransmissionCurve", 

232 doc="Transmission curve due to the atmosphere.", 

233 dimensions=["instrument"], 

234 isCalibration=True, 

235 ) 

236 illumMaskedImage = cT.PrerequisiteInput( 

237 name="illum", 

238 doc="Input illumination correction.", 

239 storageClass="MaskedImageF", 

240 dimensions=["instrument", "physical_filter", "detector"], 

241 isCalibration=True, 

242 ) 

243 deferredChargeCalib = cT.PrerequisiteInput( 

244 name="cpCtiCalib", 

245 doc="Deferred charge/CTI correction dataset.", 

246 storageClass="IsrCalib", 

247 dimensions=["instrument", "detector"], 

248 isCalibration=True, 

249 ) 

250 

251 outputExposure = cT.Output( 

252 name='postISRCCD', 

253 doc="Output ISR processed exposure.", 

254 storageClass="Exposure", 

255 dimensions=["instrument", "exposure", "detector"], 

256 ) 

257 preInterpExposure = cT.Output( 

258 name='preInterpISRCCD', 

259 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

260 storageClass="ExposureF", 

261 dimensions=["instrument", "exposure", "detector"], 

262 ) 

263 outputBin1Exposure = cT.Output( 

264 name="postIsrBin1", 

265 doc="First binned image.", 

266 storageClass="ExposureF", 

267 dimensions=["instrument", "exposure", "detector"], 

268 ) 

269 outputBin2Exposure = cT.Output( 

270 name="postIsrBin2", 

271 doc="Second binned image.", 

272 storageClass="ExposureF", 

273 dimensions=["instrument", "exposure", "detector"], 

274 ) 

275 

276 outputOssThumbnail = cT.Output( 

277 name="OssThumb", 

278 doc="Output Overscan-subtracted thumbnail image.", 

279 storageClass="Thumbnail", 

280 dimensions=["instrument", "exposure", "detector"], 

281 ) 

282 outputFlattenedThumbnail = cT.Output( 

283 name="FlattenedThumb", 

284 doc="Output flat-corrected thumbnail image.", 

285 storageClass="Thumbnail", 

286 dimensions=["instrument", "exposure", "detector"], 

287 ) 

288 outputStatistics = cT.Output( 

289 name="isrStatistics", 

290 doc="Output of additional statistics table.", 

291 storageClass="StructuredDataDict", 

292 dimensions=["instrument", "exposure", "detector"], 

293 ) 

294 

295 def __init__(self, *, config=None): 

296 super().__init__(config=config) 

297 

298 if config.doBias is not True: 

299 self.prerequisiteInputs.remove("bias") 

300 if config.doLinearize is not True: 

301 self.prerequisiteInputs.remove("linearizer") 

302 if config.doCrosstalk is not True: 

303 self.prerequisiteInputs.remove("crosstalkSources") 

304 self.prerequisiteInputs.remove("crosstalk") 

305 if config.doBrighterFatter is not True: 

306 self.prerequisiteInputs.remove("bfKernel") 

307 self.prerequisiteInputs.remove("newBFKernel") 

308 if config.doDefect is not True: 

309 self.prerequisiteInputs.remove("defects") 

310 if config.doDark is not True: 

311 self.prerequisiteInputs.remove("dark") 

312 if config.doFlat is not True: 

313 self.prerequisiteInputs.remove("flat") 

314 if config.doFringe is not True: 

315 self.prerequisiteInputs.remove("fringes") 

316 if config.doStrayLight is not True: 

317 self.prerequisiteInputs.remove("strayLightData") 

318 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

319 self.prerequisiteInputs.remove("ptc") 

320 if config.doAttachTransmissionCurve is not True: 

321 self.prerequisiteInputs.remove("opticsTransmission") 

322 self.prerequisiteInputs.remove("filterTransmission") 

323 self.prerequisiteInputs.remove("sensorTransmission") 

324 self.prerequisiteInputs.remove("atmosphereTransmission") 

325 else: 

326 if config.doUseOpticsTransmission is not True: 

327 self.prerequisiteInputs.remove("opticsTransmission") 

328 if config.doUseFilterTransmission is not True: 

329 self.prerequisiteInputs.remove("filterTransmission") 

330 if config.doUseSensorTransmission is not True: 

331 self.prerequisiteInputs.remove("sensorTransmission") 

332 if config.doUseAtmosphereTransmission is not True: 

333 self.prerequisiteInputs.remove("atmosphereTransmission") 

334 if config.doIlluminationCorrection is not True: 

335 self.prerequisiteInputs.remove("illumMaskedImage") 

336 if config.doDeferredCharge is not True: 

337 self.prerequisiteInputs.remove("deferredChargeCalib") 

338 

339 if config.doWrite is not True: 

340 self.outputs.remove("outputExposure") 

341 self.outputs.remove("preInterpExposure") 

342 self.outputs.remove("outputFlattenedThumbnail") 

343 self.outputs.remove("outputOssThumbnail") 

344 self.outputs.remove("outputStatistics") 

345 self.outputs.remove("outputBin1Exposure") 

346 self.outputs.remove("outputBin2Exposure") 

347 

348 if config.doBinnedExposures is not True: 

349 self.outputs.remove("outputBin1Exposure") 

350 self.outputs.remove("outputBin2Exposure") 

351 if config.doSaveInterpPixels is not True: 

352 self.outputs.remove("preInterpExposure") 

353 if config.qa.doThumbnailOss is not True: 

354 self.outputs.remove("outputOssThumbnail") 

355 if config.qa.doThumbnailFlattened is not True: 

356 self.outputs.remove("outputFlattenedThumbnail") 

357 if config.doCalculateStatistics is not True: 

358 self.outputs.remove("outputStatistics") 

359 

360 

361class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

362 pipelineConnections=IsrTaskConnections): 

363 """Configuration parameters for IsrTask. 

364 

365 Items are grouped in the order in which they are executed by the task. 

366 """ 

367 datasetType = pexConfig.Field( 

368 dtype=str, 

369 doc="Dataset type for input data; users will typically leave this alone, " 

370 "but camera-specific ISR tasks will override it", 

371 default="raw", 

372 ) 

373 

374 fallbackFilterName = pexConfig.Field( 

375 dtype=str, 

376 doc="Fallback default filter name for calibrations.", 

377 optional=True 

378 ) 

379 useFallbackDate = pexConfig.Field( 

380 dtype=bool, 

381 doc="Pass observation date when using fallback filter.", 

382 default=False, 

383 ) 

384 expectWcs = pexConfig.Field( 

385 dtype=bool, 

386 default=True, 

387 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

388 ) 

389 fwhm = pexConfig.Field( 

390 dtype=float, 

391 doc="FWHM of PSF in arcseconds (currently unused).", 

392 default=1.0, 

393 ) 

394 qa = pexConfig.ConfigField( 

395 dtype=isrQa.IsrQaConfig, 

396 doc="QA related configuration options.", 

397 ) 

398 doHeaderProvenance = pexConfig.Field( 

399 dtype=bool, 

400 default=True, 

401 doc="Write calibration identifiers into output exposure header?", 

402 ) 

403 

404 # Calib checking configuration: 

405 doRaiseOnCalibMismatch = pexConfig.Field( 

406 dtype=bool, 

407 default=False, 

408 doc="Should IsrTask halt if exposure and calibration header values do not match?", 

409 ) 

410 cameraKeywordsToCompare = pexConfig.ListField( 

411 dtype=str, 

412 doc="List of header keywords to compare between exposure and calibrations.", 

413 default=[], 

414 ) 

415 

416 # Image conversion configuration 

417 doConvertIntToFloat = pexConfig.Field( 

418 dtype=bool, 

419 doc="Convert integer raw images to floating point values?", 

420 default=True, 

421 ) 

422 

423 # Saturated pixel handling. 

424 doSaturation = pexConfig.Field( 

425 dtype=bool, 

426 doc="Mask saturated pixels? NB: this is totally independent of the" 

427 " interpolation option - this is ONLY setting the bits in the mask." 

428 " To have them interpolated make sure doSaturationInterpolation=True", 

429 default=True, 

430 ) 

431 saturatedMaskName = pexConfig.Field( 

432 dtype=str, 

433 doc="Name of mask plane to use in saturation detection and interpolation", 

434 default="SAT", 

435 ) 

436 saturation = pexConfig.Field( 

437 dtype=float, 

438 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

439 default=float("NaN"), 

440 ) 

441 growSaturationFootprintSize = pexConfig.Field( 

442 dtype=int, 

443 doc="Number of pixels by which to grow the saturation footprints", 

444 default=1, 

445 ) 

446 

447 # Suspect pixel handling. 

448 doSuspect = pexConfig.Field( 

449 dtype=bool, 

450 doc="Mask suspect pixels?", 

451 default=False, 

452 ) 

453 suspectMaskName = pexConfig.Field( 

454 dtype=str, 

455 doc="Name of mask plane to use for suspect pixels", 

456 default="SUSPECT", 

457 ) 

458 numEdgeSuspect = pexConfig.Field( 

459 dtype=int, 

460 doc="Number of edge pixels to be flagged as untrustworthy.", 

461 default=0, 

462 ) 

463 edgeMaskLevel = pexConfig.ChoiceField( 

464 dtype=str, 

465 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

466 default="DETECTOR", 

467 allowed={ 

468 'DETECTOR': 'Mask only the edges of the full detector.', 

469 'AMP': 'Mask edges of each amplifier.', 

470 }, 

471 ) 

472 

473 # Initial masking options. 

474 doSetBadRegions = pexConfig.Field( 

475 dtype=bool, 

476 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

477 default=True, 

478 ) 

479 badStatistic = pexConfig.ChoiceField( 

480 dtype=str, 

481 doc="How to estimate the average value for BAD regions.", 

482 default='MEANCLIP', 

483 allowed={ 

484 "MEANCLIP": "Correct using the (clipped) mean of good data", 

485 "MEDIAN": "Correct using the median of the good data", 

486 }, 

487 ) 

488 

489 # Overscan subtraction configuration. 

490 doOverscan = pexConfig.Field( 

491 dtype=bool, 

492 doc="Do overscan subtraction?", 

493 default=True, 

494 ) 

495 overscan = pexConfig.ConfigurableField( 

496 target=OverscanCorrectionTask, 

497 doc="Overscan subtraction task for image segments.", 

498 ) 

499 

500 # Amplifier to CCD assembly configuration 

501 doAssembleCcd = pexConfig.Field( 

502 dtype=bool, 

503 default=True, 

504 doc="Assemble amp-level exposures into a ccd-level exposure?" 

505 ) 

506 assembleCcd = pexConfig.ConfigurableField( 

507 target=AssembleCcdTask, 

508 doc="CCD assembly task", 

509 ) 

510 

511 # General calibration configuration. 

512 doAssembleIsrExposures = pexConfig.Field( 

513 dtype=bool, 

514 default=False, 

515 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

516 ) 

517 doTrimToMatchCalib = pexConfig.Field( 

518 dtype=bool, 

519 default=False, 

520 doc="Trim raw data to match calibration bounding boxes?" 

521 ) 

522 

523 # Bias subtraction. 

524 doBias = pexConfig.Field( 

525 dtype=bool, 

526 doc="Apply bias frame correction?", 

527 default=True, 

528 ) 

529 biasDataProductName = pexConfig.Field( 

530 dtype=str, 

531 doc="Name of the bias data product", 

532 default="bias", 

533 ) 

534 doBiasBeforeOverscan = pexConfig.Field( 

535 dtype=bool, 

536 doc="Reverse order of overscan and bias correction.", 

537 default=False 

538 ) 

539 

540 # Deferred charge correction. 

541 doDeferredCharge = pexConfig.Field( 

542 dtype=bool, 

543 doc="Apply deferred charge correction?", 

544 default=False, 

545 ) 

546 deferredChargeCorrection = pexConfig.ConfigurableField( 

547 target=DeferredChargeTask, 

548 doc="Deferred charge correction task.", 

549 ) 

550 

551 # Variance construction 

552 doVariance = pexConfig.Field( 

553 dtype=bool, 

554 doc="Calculate variance?", 

555 default=True 

556 ) 

557 gain = pexConfig.Field( 

558 dtype=float, 

559 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

560 default=float("NaN"), 

561 ) 

562 readNoise = pexConfig.Field( 

563 dtype=float, 

564 doc="The read noise to use if no Detector is present in the Exposure", 

565 default=0.0, 

566 ) 

567 doEmpiricalReadNoise = pexConfig.Field( 

568 dtype=bool, 

569 default=False, 

570 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

571 ) 

572 usePtcReadNoise = pexConfig.Field( 

573 dtype=bool, 

574 default=False, 

575 doc="Use readnoise values from the Photon Transfer Curve?" 

576 ) 

577 maskNegativeVariance = pexConfig.Field( 

578 dtype=bool, 

579 default=True, 

580 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

581 "in the measurement of the overscan at an edge due to the data falling off faster " 

582 "than the overscan model can account for it." 

583 ) 

584 negativeVarianceMaskName = pexConfig.Field( 

585 dtype=str, 

586 default="BAD", 

587 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

588 ) 

589 # Linearization. 

590 doLinearize = pexConfig.Field( 

591 dtype=bool, 

592 doc="Correct for nonlinearity of the detector's response?", 

593 default=True, 

594 ) 

595 

596 # Crosstalk. 

597 doCrosstalk = pexConfig.Field( 

598 dtype=bool, 

599 doc="Apply intra-CCD crosstalk correction?", 

600 default=False, 

601 ) 

602 doCrosstalkBeforeAssemble = pexConfig.Field( 

603 dtype=bool, 

604 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

605 default=False, 

606 ) 

607 crosstalk = pexConfig.ConfigurableField( 

608 target=CrosstalkTask, 

609 doc="Intra-CCD crosstalk correction", 

610 ) 

611 

612 # Masking options. 

613 doDefect = pexConfig.Field( 

614 dtype=bool, 

615 doc="Apply correction for CCD defects, e.g. hot pixels?", 

616 default=True, 

617 ) 

618 doNanMasking = pexConfig.Field( 

619 dtype=bool, 

620 doc="Mask non-finite (NAN, inf) pixels?", 

621 default=True, 

622 ) 

623 doWidenSaturationTrails = pexConfig.Field( 

624 dtype=bool, 

625 doc="Widen bleed trails based on their width?", 

626 default=True 

627 ) 

628 

629 # Brighter-Fatter correction. 

630 doBrighterFatter = pexConfig.Field( 

631 dtype=bool, 

632 default=False, 

633 doc="Apply the brighter-fatter correction?" 

634 ) 

635 doFluxConservingBrighterFatterCorrection = pexConfig.Field( 

636 dtype=bool, 

637 default=False, 

638 doc="Apply the flux-conserving BFE correction by Miller et al.?" 

639 ) 

640 brighterFatterLevel = pexConfig.ChoiceField( 

641 dtype=str, 

642 default="DETECTOR", 

643 doc="The level at which to correct for brighter-fatter.", 

644 allowed={ 

645 "AMP": "Every amplifier treated separately.", 

646 "DETECTOR": "One kernel per detector", 

647 } 

648 ) 

649 brighterFatterMaxIter = pexConfig.Field( 

650 dtype=int, 

651 default=10, 

652 doc="Maximum number of iterations for the brighter-fatter correction" 

653 ) 

654 brighterFatterThreshold = pexConfig.Field( 

655 dtype=float, 

656 default=1000, 

657 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

658 "absolute value of the difference between the current corrected image and the one " 

659 "from the previous iteration summed over all the pixels." 

660 ) 

661 brighterFatterApplyGain = pexConfig.Field( 

662 dtype=bool, 

663 default=True, 

664 doc="Should the gain be applied when applying the brighter-fatter correction?" 

665 ) 

666 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

667 dtype=str, 

668 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

669 "correction.", 

670 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

671 ) 

672 brighterFatterMaskGrowSize = pexConfig.Field( 

673 dtype=int, 

674 default=0, 

675 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

676 "when brighter-fatter correction is applied." 

677 ) 

678 

679 # Dark subtraction. 

680 doDark = pexConfig.Field( 

681 dtype=bool, 

682 doc="Apply dark frame correction?", 

683 default=True, 

684 ) 

685 darkDataProductName = pexConfig.Field( 

686 dtype=str, 

687 doc="Name of the dark data product", 

688 default="dark", 

689 ) 

690 

691 # Camera-specific stray light removal. 

692 doStrayLight = pexConfig.Field( 

693 dtype=bool, 

694 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

695 default=False, 

696 ) 

697 strayLight = pexConfig.ConfigurableField( 

698 target=StrayLightTask, 

699 doc="y-band stray light correction" 

700 ) 

701 

702 # Flat correction. 

703 doFlat = pexConfig.Field( 

704 dtype=bool, 

705 doc="Apply flat field correction?", 

706 default=True, 

707 ) 

708 flatDataProductName = pexConfig.Field( 

709 dtype=str, 

710 doc="Name of the flat data product", 

711 default="flat", 

712 ) 

713 flatScalingType = pexConfig.ChoiceField( 

714 dtype=str, 

715 doc="The method for scaling the flat on the fly.", 

716 default='USER', 

717 allowed={ 

718 "USER": "Scale by flatUserScale", 

719 "MEAN": "Scale by the inverse of the mean", 

720 "MEDIAN": "Scale by the inverse of the median", 

721 }, 

722 ) 

723 flatUserScale = pexConfig.Field( 

724 dtype=float, 

725 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

726 default=1.0, 

727 ) 

728 doTweakFlat = pexConfig.Field( 

729 dtype=bool, 

730 doc="Tweak flats to match observed amplifier ratios?", 

731 default=False 

732 ) 

733 

734 # Amplifier normalization based on gains instead of using flats 

735 # configuration. 

736 doApplyGains = pexConfig.Field( 

737 dtype=bool, 

738 doc="Correct the amplifiers for their gains instead of applying flat correction", 

739 default=False, 

740 ) 

741 usePtcGains = pexConfig.Field( 

742 dtype=bool, 

743 doc="Use the gain values from the Photon Transfer Curve?", 

744 default=False, 

745 ) 

746 normalizeGains = pexConfig.Field( 

747 dtype=bool, 

748 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

749 default=False, 

750 ) 

751 

752 # Fringe correction. 

753 doFringe = pexConfig.Field( 

754 dtype=bool, 

755 doc="Apply fringe correction?", 

756 default=True, 

757 ) 

758 fringe = pexConfig.ConfigurableField( 

759 target=FringeTask, 

760 doc="Fringe subtraction task", 

761 ) 

762 fringeAfterFlat = pexConfig.Field( 

763 dtype=bool, 

764 doc="Do fringe subtraction after flat-fielding?", 

765 default=True, 

766 ) 

767 

768 # Amp offset correction. 

769 doAmpOffset = pexConfig.Field( 

770 doc="Calculate and apply amp offset corrections?", 

771 dtype=bool, 

772 default=False, 

773 ) 

774 ampOffset = pexConfig.ConfigurableField( 

775 doc="Amp offset correction task.", 

776 target=AmpOffsetTask, 

777 ) 

778 

779 # Initial CCD-level background statistics options. 

780 doMeasureBackground = pexConfig.Field( 

781 dtype=bool, 

782 doc="Measure the background level on the reduced image?", 

783 default=False, 

784 ) 

785 

786 # Camera-specific masking configuration. 

787 doCameraSpecificMasking = pexConfig.Field( 

788 dtype=bool, 

789 doc="Mask camera-specific bad regions?", 

790 default=False, 

791 ) 

792 masking = pexConfig.ConfigurableField( 

793 target=MaskingTask, 

794 doc="Masking task." 

795 ) 

796 

797 # Interpolation options. 

798 doInterpolate = pexConfig.Field( 

799 dtype=bool, 

800 doc="Interpolate masked pixels?", 

801 default=True, 

802 ) 

803 doSaturationInterpolation = pexConfig.Field( 

804 dtype=bool, 

805 doc="Perform interpolation over pixels masked as saturated?" 

806 " NB: This is independent of doSaturation; if that is False this plane" 

807 " will likely be blank, resulting in a no-op here.", 

808 default=True, 

809 ) 

810 doNanInterpolation = pexConfig.Field( 

811 dtype=bool, 

812 doc="Perform interpolation over pixels masked as NaN?" 

813 " NB: This is independent of doNanMasking; if that is False this plane" 

814 " will likely be blank, resulting in a no-op here.", 

815 default=True, 

816 ) 

817 doNanInterpAfterFlat = pexConfig.Field( 

818 dtype=bool, 

819 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

820 "also have to interpolate them before flat-fielding."), 

821 default=False, 

822 ) 

823 maskListToInterpolate = pexConfig.ListField( 

824 dtype=str, 

825 doc="List of mask planes that should be interpolated.", 

826 default=['SAT', 'BAD'], 

827 ) 

828 doSaveInterpPixels = pexConfig.Field( 

829 dtype=bool, 

830 doc="Save a copy of the pre-interpolated pixel values?", 

831 default=False, 

832 ) 

833 

834 # Default photometric calibration options. 

835 fluxMag0T1 = pexConfig.DictField( 

836 keytype=str, 

837 itemtype=float, 

838 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

839 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

840 )) 

841 ) 

842 defaultFluxMag0T1 = pexConfig.Field( 

843 dtype=float, 

844 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

845 default=pow(10.0, 0.4*28.0) 

846 ) 

847 

848 # Vignette correction configuration. 

849 doVignette = pexConfig.Field( 

850 dtype=bool, 

851 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

852 "according to vignetting parameters?"), 

853 default=False, 

854 ) 

855 doMaskVignettePolygon = pexConfig.Field( 

856 dtype=bool, 

857 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

858 "is False"), 

859 default=True, 

860 ) 

861 vignetteValue = pexConfig.Field( 

862 dtype=float, 

863 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

864 optional=True, 

865 default=None, 

866 ) 

867 vignette = pexConfig.ConfigurableField( 

868 target=VignetteTask, 

869 doc="Vignetting task.", 

870 ) 

871 

872 # Transmission curve configuration. 

873 doAttachTransmissionCurve = pexConfig.Field( 

874 dtype=bool, 

875 default=False, 

876 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

877 ) 

878 doUseOpticsTransmission = pexConfig.Field( 

879 dtype=bool, 

880 default=True, 

881 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

882 ) 

883 doUseFilterTransmission = pexConfig.Field( 

884 dtype=bool, 

885 default=True, 

886 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

887 ) 

888 doUseSensorTransmission = pexConfig.Field( 

889 dtype=bool, 

890 default=True, 

891 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

892 ) 

893 doUseAtmosphereTransmission = pexConfig.Field( 

894 dtype=bool, 

895 default=True, 

896 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

897 ) 

898 

899 # Illumination correction. 

900 doIlluminationCorrection = pexConfig.Field( 

901 dtype=bool, 

902 default=False, 

903 doc="Perform illumination correction?" 

904 ) 

905 illuminationCorrectionDataProductName = pexConfig.Field( 

906 dtype=str, 

907 doc="Name of the illumination correction data product.", 

908 default="illumcor", 

909 ) 

910 illumScale = pexConfig.Field( 

911 dtype=float, 

912 doc="Scale factor for the illumination correction.", 

913 default=1.0, 

914 ) 

915 illumFilters = pexConfig.ListField( 

916 dtype=str, 

917 default=[], 

918 doc="Only perform illumination correction for these filters." 

919 ) 

920 

921 # Calculate image quality statistics? 

922 doStandardStatistics = pexConfig.Field( 

923 dtype=bool, 

924 doc="Should standard image quality statistics be calculated?", 

925 default=True, 

926 ) 

927 # Calculate additional statistics? 

928 doCalculateStatistics = pexConfig.Field( 

929 dtype=bool, 

930 doc="Should additional ISR statistics be calculated?", 

931 default=False, 

932 ) 

933 isrStats = pexConfig.ConfigurableField( 

934 target=IsrStatisticsTask, 

935 doc="Task to calculate additional statistics.", 

936 ) 

937 

938 # Make binned images? 

939 doBinnedExposures = pexConfig.Field( 

940 dtype=bool, 

941 doc="Should binned exposures be calculated?", 

942 default=False, 

943 ) 

944 binFactor1 = pexConfig.Field( 944 ↛ exitline 944 didn't jump to the function exit

945 dtype=int, 

946 doc="Binning factor for first binned exposure. This is intended for a finely binned output.", 

947 default=8, 

948 check=lambda x: x > 1, 

949 ) 

950 binFactor2 = pexConfig.Field( 950 ↛ exitline 950 didn't jump to the function exit

951 dtype=int, 

952 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.", 

953 default=64, 

954 check=lambda x: x > 1, 

955 ) 

956 

957 # Write the outputs to disk. If ISR is run as a subtask, this may not 

958 # be needed. 

959 doWrite = pexConfig.Field( 

960 dtype=bool, 

961 doc="Persist postISRCCD?", 

962 default=True, 

963 ) 

964 

965 def validate(self): 

966 super().validate() 

967 if self.doFlat and self.doApplyGains: 

968 raise ValueError("You may not specify both doFlat and doApplyGains") 

969 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

970 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

971 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

972 self.maskListToInterpolate.append(self.saturatedMaskName) 

973 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

974 self.maskListToInterpolate.remove(self.saturatedMaskName) 

975 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

976 self.maskListToInterpolate.append("UNMASKEDNAN") 

977 

978 

979class IsrTask(pipeBase.PipelineTask): 

980 """Apply common instrument signature correction algorithms to a raw frame. 

981 

982 The process for correcting imaging data is very similar from 

983 camera to camera. This task provides a vanilla implementation of 

984 doing these corrections, including the ability to turn certain 

985 corrections off if they are not needed. The inputs to the primary 

986 method, `run()`, are a raw exposure to be corrected and the 

987 calibration data products. The raw input is a single chip sized 

988 mosaic of all amps including overscans and other non-science 

989 pixels. 

990 

991 The __init__ method sets up the subtasks for ISR processing, using 

992 the defaults from `lsst.ip.isr`. 

993 

994 Parameters 

995 ---------- 

996 args : `list` 

997 Positional arguments passed to the Task constructor. 

998 None used at this time. 

999 kwargs : `dict`, optional 

1000 Keyword arguments passed on to the Task constructor. 

1001 None used at this time. 

1002 """ 

1003 ConfigClass = IsrTaskConfig 

1004 _DefaultName = "isr" 

1005 

1006 def __init__(self, **kwargs): 

1007 super().__init__(**kwargs) 

1008 self.makeSubtask("assembleCcd") 

1009 self.makeSubtask("crosstalk") 

1010 self.makeSubtask("strayLight") 

1011 self.makeSubtask("fringe") 

1012 self.makeSubtask("masking") 

1013 self.makeSubtask("overscan") 

1014 self.makeSubtask("vignette") 

1015 self.makeSubtask("ampOffset") 

1016 self.makeSubtask("deferredChargeCorrection") 

1017 self.makeSubtask("isrStats") 

1018 

1019 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1020 inputs = butlerQC.get(inputRefs) 

1021 

1022 try: 

1023 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

1024 except Exception as e: 

1025 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

1026 (inputRefs, e)) 

1027 

1028 detector = inputs['ccdExposure'].getDetector() 

1029 

1030 # This is use for header provenance. 

1031 additionalInputDates = {} 

1032 

1033 if self.config.doCrosstalk is True: 

1034 # Crosstalk sources need to be defined by the pipeline 

1035 # yaml if they exist. 

1036 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

1037 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

1038 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

1039 else: 

1040 coeffVector = (self.config.crosstalk.crosstalkValues 

1041 if self.config.crosstalk.useConfigCoefficients else None) 

1042 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1043 inputs['crosstalk'] = crosstalkCalib 

1044 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1045 if 'crosstalkSources' not in inputs: 

1046 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1047 

1048 if self.doLinearize(detector) is True: 

1049 if 'linearizer' in inputs: 

1050 if isinstance(inputs['linearizer'], dict): 

1051 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1052 linearizer.fromYaml(inputs['linearizer']) 

1053 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1054 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1055 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1056 detector=detector, 

1057 log=self.log) 

1058 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1059 else: 

1060 linearizer = inputs['linearizer'] 

1061 linearizer.log = self.log 

1062 inputs['linearizer'] = linearizer 

1063 else: 

1064 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1065 self.log.warning("Constructing linearizer from cameraGeom information.") 

1066 

1067 if self.config.doDefect is True: 

1068 if "defects" in inputs and inputs['defects'] is not None: 

1069 # defects is loaded as a BaseCatalog with columns 

1070 # x0, y0, width, height. Masking expects a list of defects 

1071 # defined by their bounding box 

1072 if not isinstance(inputs["defects"], Defects): 

1073 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1074 

1075 # Load the correct style of brighter-fatter kernel, and repack 

1076 # the information as a numpy array. 

1077 brighterFatterSource = None 

1078 if self.config.doBrighterFatter: 

1079 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1080 if brighterFatterKernel is None: 

1081 # This type of kernel must be in (y, x) index 

1082 # ordering, as it used directly as the .array 

1083 # component of the afwImage kernel. 

1084 brighterFatterKernel = inputs.get('bfKernel', None) 

1085 brighterFatterSource = 'bfKernel' 

1086 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1087 

1088 if brighterFatterKernel is None: 

1089 # This was requested by the config, but none were found. 

1090 raise RuntimeError("No brighter-fatter kernel was supplied.") 

1091 elif not isinstance(brighterFatterKernel, numpy.ndarray): 

1092 # This is a ISR calib kernel. These kernels are 

1093 # generated in (x, y) index ordering, and need to be 

1094 # transposed to be used directly as the .array 

1095 # component of the afwImage kernel. This is done 

1096 # explicitly below when setting the ``bfKernel`` 

1097 # input. 

1098 brighterFatterSource = 'newBFKernel' 

1099 additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel) 

1100 

1101 detName = detector.getName() 

1102 level = brighterFatterKernel.level 

1103 

1104 # This is expected to be a dictionary of amp-wise gains. 

1105 inputs['bfGains'] = brighterFatterKernel.gain 

1106 if self.config.brighterFatterLevel == 'DETECTOR': 

1107 kernel = None 

1108 if level == 'DETECTOR': 

1109 if detName in brighterFatterKernel.detKernels: 

1110 kernel = brighterFatterKernel.detKernels[detName] 

1111 else: 

1112 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1113 elif level == 'AMP': 

1114 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1115 "fatter kernels.") 

1116 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1117 kernel = brighterFatterKernel.detKernels[detName] 

1118 if kernel is None: 

1119 raise RuntimeError("Could not identify brighter-fatter kernel!") 

1120 # Do the one single transpose here so the kernel 

1121 # can be directly loaded into the afwImage .array 

1122 # component. 

1123 inputs['bfKernel'] = numpy.transpose(kernel) 

1124 elif self.config.brighterFatterLevel == 'AMP': 

1125 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1126 

1127 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1128 expId = inputs['ccdExposure'].info.id 

1129 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1130 expId=expId, 

1131 assembler=self.assembleCcd 

1132 if self.config.doAssembleIsrExposures else None) 

1133 else: 

1134 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1135 

1136 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1137 if 'strayLightData' not in inputs: 

1138 inputs['strayLightData'] = None 

1139 

1140 if self.config.doHeaderProvenance: 

1141 # Add calibration provenanace info to header. 

1142 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1143 

1144 # These inputs change name during this step. These should 

1145 # have matching entries in the additionalInputDates dict. 

1146 additionalInputs = [] 

1147 if self.config.doBrighterFatter: 

1148 additionalInputs.append(brighterFatterSource) 

1149 

1150 for inputName in sorted(list(inputs.keys()) + additionalInputs): 

1151 reference = getattr(inputRefs, inputName, None) 

1152 if reference is not None and hasattr(reference, "run"): 

1153 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1154 runValue = reference.run 

1155 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1156 idValue = str(reference.id) 

1157 dateKey = f"LSST CALIB DATE {inputName.upper()}" 

1158 

1159 if inputName in additionalInputDates: 

1160 dateValue = additionalInputDates[inputName] 

1161 else: 

1162 dateValue = self.extractCalibDate(inputs[inputName]) 

1163 

1164 exposureMetadata[runKey] = runValue 

1165 exposureMetadata[idKey] = idValue 

1166 exposureMetadata[dateKey] = dateValue 

1167 

1168 outputs = self.run(**inputs) 

1169 butlerQC.put(outputs, outputRefs) 

1170 

1171 @timeMethod 

1172 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1173 crosstalk=None, crosstalkSources=None, 

1174 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1175 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1176 sensorTransmission=None, atmosphereTransmission=None, 

1177 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1178 deferredChargeCalib=None, 

1179 ): 

1180 """Perform instrument signature removal on an exposure. 

1181 

1182 Steps included in the ISR processing, in order performed, are: 

1183 

1184 - saturation and suspect pixel masking 

1185 - overscan subtraction 

1186 - CCD assembly of individual amplifiers 

1187 - bias subtraction 

1188 - variance image construction 

1189 - linearization of non-linear response 

1190 - crosstalk masking 

1191 - brighter-fatter correction 

1192 - dark subtraction 

1193 - fringe correction 

1194 - stray light subtraction 

1195 - flat correction 

1196 - masking of known defects and camera specific features 

1197 - vignette calculation 

1198 - appending transmission curve and distortion model 

1199 

1200 Parameters 

1201 ---------- 

1202 ccdExposure : `lsst.afw.image.Exposure` 

1203 The raw exposure that is to be run through ISR. The 

1204 exposure is modified by this method. 

1205 camera : `lsst.afw.cameraGeom.Camera`, optional 

1206 The camera geometry for this exposure. Required if 

1207 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1208 ``flat`` does not have an associated detector. 

1209 bias : `lsst.afw.image.Exposure`, optional 

1210 Bias calibration frame. 

1211 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1212 Functor for linearization. 

1213 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1214 Calibration for crosstalk. 

1215 crosstalkSources : `list`, optional 

1216 List of possible crosstalk sources. 

1217 dark : `lsst.afw.image.Exposure`, optional 

1218 Dark calibration frame. 

1219 flat : `lsst.afw.image.Exposure`, optional 

1220 Flat calibration frame. 

1221 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1222 Photon transfer curve dataset, with, e.g., gains 

1223 and read noise. 

1224 bfKernel : `numpy.ndarray`, optional 

1225 Brighter-fatter kernel. 

1226 bfGains : `dict` of `float`, optional 

1227 Gains used to override the detector's nominal gains for the 

1228 brighter-fatter correction. A dict keyed by amplifier name for 

1229 the detector in question. 

1230 defects : `lsst.ip.isr.Defects`, optional 

1231 List of defects. 

1232 fringes : `lsst.pipe.base.Struct`, optional 

1233 Struct containing the fringe correction data, with 

1234 elements: 

1235 

1236 ``fringes`` 

1237 fringe calibration frame (`lsst.afw.image.Exposure`) 

1238 ``seed`` 

1239 random seed derived from the ``ccdExposureId`` for random 

1240 number generator (`numpy.uint32`) 

1241 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1242 A ``TransmissionCurve`` that represents the throughput of the, 

1243 optics, to be evaluated in focal-plane coordinates. 

1244 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1245 A ``TransmissionCurve`` that represents the throughput of the 

1246 filter itself, to be evaluated in focal-plane coordinates. 

1247 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1248 A ``TransmissionCurve`` that represents the throughput of the 

1249 sensor itself, to be evaluated in post-assembly trimmed detector 

1250 coordinates. 

1251 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1252 A ``TransmissionCurve`` that represents the throughput of the 

1253 atmosphere, assumed to be spatially constant. 

1254 detectorNum : `int`, optional 

1255 The integer number for the detector to process. 

1256 strayLightData : `object`, optional 

1257 Opaque object containing calibration information for stray-light 

1258 correction. If `None`, no correction will be performed. 

1259 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1260 Illumination correction image. 

1261 

1262 Returns 

1263 ------- 

1264 result : `lsst.pipe.base.Struct` 

1265 Result struct with component: 

1266 

1267 ``exposure`` 

1268 The fully ISR corrected exposure. 

1269 (`lsst.afw.image.Exposure`) 

1270 ``outputExposure`` 

1271 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1272 ``ossThumb`` 

1273 Thumbnail image of the exposure after overscan subtraction. 

1274 (`numpy.ndarray`) 

1275 ``flattenedThumb`` 

1276 Thumbnail image of the exposure after flat-field correction. 

1277 (`numpy.ndarray`) 

1278 ``outputStatistics`` 

1279 Values of the additional statistics calculated. 

1280 

1281 Raises 

1282 ------ 

1283 RuntimeError 

1284 Raised if a configuration option is set to `True`, but the 

1285 required calibration data has not been specified. 

1286 

1287 Notes 

1288 ----- 

1289 The current processed exposure can be viewed by setting the 

1290 appropriate `lsstDebug` entries in the ``debug.display`` 

1291 dictionary. The names of these entries correspond to some of 

1292 the `IsrTaskConfig` Boolean options, with the value denoting the 

1293 frame to use. The exposure is shown inside the matching 

1294 option check and after the processing of that step has 

1295 finished. The steps with debug points are: 

1296 

1297 * doAssembleCcd 

1298 * doBias 

1299 * doCrosstalk 

1300 * doBrighterFatter 

1301 * doDark 

1302 * doFringe 

1303 * doStrayLight 

1304 * doFlat 

1305 

1306 In addition, setting the ``postISRCCD`` entry displays the 

1307 exposure after all ISR processing has finished. 

1308 """ 

1309 

1310 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1311 bias = self.ensureExposure(bias, camera, detectorNum) 

1312 dark = self.ensureExposure(dark, camera, detectorNum) 

1313 flat = self.ensureExposure(flat, camera, detectorNum) 

1314 

1315 ccd = ccdExposure.getDetector() 

1316 filterLabel = ccdExposure.getFilter() 

1317 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1318 

1319 if not ccd: 

1320 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1321 ccd = [FakeAmp(ccdExposure, self.config)] 

1322 

1323 # Validate Input 

1324 if self.config.doBias and bias is None: 

1325 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1326 if self.doLinearize(ccd) and linearizer is None: 

1327 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1328 if self.config.doBrighterFatter and bfKernel is None: 

1329 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1330 if self.config.doDark and dark is None: 

1331 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1332 if self.config.doFlat and flat is None: 

1333 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1334 if self.config.doDefect and defects is None: 

1335 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1336 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1337 and fringes.fringes is None): 

1338 # The `fringes` object needs to be a pipeBase.Struct, as 

1339 # we use it as a `dict` for the parameters of 

1340 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1341 # not be `None` if `doFringe=True`. Otherwise, raise. 

1342 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1343 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1344 and illumMaskedImage is None): 

1345 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1346 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1347 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1348 

1349 # Validate that the inputs match the exposure configuration. 

1350 exposureMetadata = ccdExposure.getMetadata() 

1351 if self.config.doBias: 

1352 self.compareCameraKeywords(exposureMetadata, bias, "bias") 

1353 if self.config.doBrighterFatter: 

1354 self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter") 

1355 if self.config.doCrosstalk: 

1356 self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk") 

1357 if self.config.doDark: 

1358 self.compareCameraKeywords(exposureMetadata, dark, "dark") 

1359 if self.config.doDefect: 

1360 self.compareCameraKeywords(exposureMetadata, defects, "defects") 

1361 if self.config.doDeferredCharge: 

1362 self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI") 

1363 if self.config.doFlat: 

1364 self.compareCameraKeywords(exposureMetadata, flat, "flat") 

1365 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1366 self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe") 

1367 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1368 self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination") 

1369 if self.doLinearize(ccd): 

1370 self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer") 

1371 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1372 self.compareCameraKeywords(exposureMetadata, ptc, "PTC") 

1373 if self.config.doStrayLight: 

1374 self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight") 

1375 

1376 # Begin ISR processing. 

1377 if self.config.doConvertIntToFloat: 

1378 self.log.info("Converting exposure to floating point values.") 

1379 ccdExposure = self.convertIntToFloat(ccdExposure) 

1380 

1381 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1382 self.log.info("Applying bias correction.") 

1383 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1384 trimToFit=self.config.doTrimToMatchCalib) 

1385 self.debugView(ccdExposure, "doBias") 

1386 

1387 # Amplifier level processing. 

1388 overscans = [] 

1389 

1390 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1391 # This will attempt to mask bleed pixels across all amplifiers. 

1392 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1393 

1394 for amp in ccd: 

1395 # if ccdExposure is one amp, 

1396 # check for coverage to prevent performing ops multiple times 

1397 if ccdExposure.getBBox().contains(amp.getBBox()): 

1398 # Check for fully masked bad amplifiers, 

1399 # and generate masks for SUSPECT and SATURATED values. 

1400 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1401 

1402 if self.config.doOverscan and not badAmp: 

1403 # Overscan correction on amp-by-amp basis. 

1404 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1405 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1406 if overscanResults is not None and \ 

1407 self.config.qa is not None and self.config.qa.saveStats is True: 

1408 if isinstance(overscanResults.overscanMean, float): 

1409 # Only serial overscan was run 

1410 mean = overscanResults.overscanMean 

1411 sigma = overscanResults.overscanSigma 

1412 residMean = overscanResults.residualMean 

1413 residSigma = overscanResults.residualSigma 

1414 else: 

1415 # Both serial and parallel overscan were 

1416 # run. Only report serial here. 

1417 mean = overscanResults.overscanMean[0] 

1418 sigma = overscanResults.overscanSigma[0] 

1419 residMean = overscanResults.residualMean[0] 

1420 residSigma = overscanResults.residualSigma[0] 

1421 

1422 self.metadata[f"FIT MEDIAN {amp.getName()}"] = mean 

1423 self.metadata[f"FIT STDEV {amp.getName()}"] = sigma 

1424 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1425 amp.getName(), mean, sigma) 

1426 

1427 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = residMean 

1428 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = residSigma 

1429 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1430 amp.getName(), residMean, residSigma) 

1431 

1432 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1433 else: 

1434 if badAmp: 

1435 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1436 overscanResults = None 

1437 

1438 overscans.append(overscanResults if overscanResults is not None else None) 

1439 else: 

1440 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1441 

1442 if self.config.doDeferredCharge: 

1443 self.log.info("Applying deferred charge/CTI correction.") 

1444 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1445 self.debugView(ccdExposure, "doDeferredCharge") 

1446 

1447 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1448 self.log.info("Applying crosstalk correction.") 

1449 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1450 crosstalkSources=crosstalkSources, camera=camera) 

1451 self.debugView(ccdExposure, "doCrosstalk") 

1452 

1453 if self.config.doAssembleCcd: 

1454 self.log.info("Assembling CCD from amplifiers.") 

1455 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1456 

1457 if self.config.expectWcs and not ccdExposure.getWcs(): 

1458 self.log.warning("No WCS found in input exposure.") 

1459 self.debugView(ccdExposure, "doAssembleCcd") 

1460 

1461 ossThumb = None 

1462 if self.config.qa.doThumbnailOss: 

1463 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1464 

1465 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1466 self.log.info("Applying bias correction.") 

1467 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1468 trimToFit=self.config.doTrimToMatchCalib) 

1469 self.debugView(ccdExposure, "doBias") 

1470 

1471 if self.config.doVariance: 

1472 for amp, overscanResults in zip(ccd, overscans): 

1473 if ccdExposure.getBBox().contains(amp.getBBox()): 

1474 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1475 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1476 if overscanResults is not None: 

1477 self.updateVariance(ampExposure, amp, 

1478 overscanImage=overscanResults.overscanImage, 

1479 ptcDataset=ptc) 

1480 else: 

1481 self.updateVariance(ampExposure, amp, 

1482 overscanImage=None, 

1483 ptcDataset=ptc) 

1484 if self.config.qa is not None and self.config.qa.saveStats is True: 

1485 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1486 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1487 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1488 qaStats.getValue(afwMath.MEDIAN) 

1489 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1490 qaStats.getValue(afwMath.STDEVCLIP) 

1491 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1492 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1493 qaStats.getValue(afwMath.STDEVCLIP)) 

1494 if self.config.maskNegativeVariance: 

1495 self.maskNegativeVariance(ccdExposure) 

1496 

1497 if self.doLinearize(ccd): 

1498 self.log.info("Applying linearizer.") 

1499 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1500 detector=ccd, log=self.log) 

1501 

1502 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1503 self.log.info("Applying crosstalk correction.") 

1504 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1505 crosstalkSources=crosstalkSources, isTrimmed=True) 

1506 self.debugView(ccdExposure, "doCrosstalk") 

1507 

1508 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1509 # widen trails, and do anything else the camera needs. Saturated and 

1510 # suspect pixels have already been masked. 

1511 if self.config.doDefect: 

1512 self.log.info("Masking defects.") 

1513 self.maskDefect(ccdExposure, defects) 

1514 

1515 if self.config.numEdgeSuspect > 0: 

1516 self.log.info("Masking edges as SUSPECT.") 

1517 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1518 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1519 

1520 if self.config.doNanMasking: 

1521 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1522 self.maskNan(ccdExposure) 

1523 

1524 if self.config.doWidenSaturationTrails: 

1525 self.log.info("Widening saturation trails.") 

1526 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1527 

1528 if self.config.doCameraSpecificMasking: 

1529 self.log.info("Masking regions for camera specific reasons.") 

1530 self.masking.run(ccdExposure) 

1531 

1532 if self.config.doBrighterFatter: 

1533 # We need to apply flats and darks before we can interpolate, and 

1534 # we need to interpolate before we do B-F, but we do B-F without 

1535 # the flats and darks applied so we can work in units of electrons 

1536 # or holes. This context manager applies and then removes the darks 

1537 # and flats. 

1538 # 

1539 # We also do not want to interpolate values here, so operate on 

1540 # temporary images so we can apply only the BF-correction and roll 

1541 # back the interpolation. 

1542 interpExp = ccdExposure.clone() 

1543 with self.flatContext(interpExp, flat, dark): 

1544 isrFunctions.interpolateFromMask( 

1545 maskedImage=interpExp.getMaskedImage(), 

1546 fwhm=self.config.fwhm, 

1547 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1548 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1549 ) 

1550 bfExp = interpExp.clone() 

1551 

1552 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1553 type(bfKernel), type(bfGains)) 

1554 if self.config.doFluxConservingBrighterFatterCorrection: 

1555 bfResults = isrFunctions.fluxConservingBrighterFatterCorrection( 

1556 bfExp, 

1557 bfKernel, 

1558 self.config.brighterFatterMaxIter, 

1559 self.config.brighterFatterThreshold, 

1560 self.config.brighterFatterApplyGain, 

1561 bfGains 

1562 ) 

1563 else: 

1564 bfResults = isrFunctions.brighterFatterCorrection( 

1565 bfExp, 

1566 bfKernel, 

1567 self.config.brighterFatterMaxIter, 

1568 self.config.brighterFatterThreshold, 

1569 self.config.brighterFatterApplyGain, 

1570 bfGains 

1571 ) 

1572 if bfResults[1] == self.config.brighterFatterMaxIter - 1: 

1573 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1574 bfResults[0]) 

1575 else: 

1576 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1577 bfResults[1]) 

1578 image = ccdExposure.getMaskedImage().getImage() 

1579 bfCorr = bfExp.getMaskedImage().getImage() 

1580 bfCorr -= interpExp.getMaskedImage().getImage() 

1581 image += bfCorr 

1582 

1583 # Applying the brighter-fatter correction applies a 

1584 # convolution to the science image. At the edges this 

1585 # convolution may not have sufficient valid pixels to 

1586 # produce a valid correction. Mark pixels within the size 

1587 # of the brighter-fatter kernel as EDGE to warn of this 

1588 # fact. 

1589 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1590 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1591 maskPlane="EDGE") 

1592 

1593 if self.config.brighterFatterMaskGrowSize > 0: 

1594 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1595 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1596 isrFunctions.growMasks(ccdExposure.getMask(), 

1597 radius=self.config.brighterFatterMaskGrowSize, 

1598 maskNameList=maskPlane, 

1599 maskValue=maskPlane) 

1600 

1601 self.debugView(ccdExposure, "doBrighterFatter") 

1602 

1603 if self.config.doDark: 

1604 self.log.info("Applying dark correction.") 

1605 self.darkCorrection(ccdExposure, dark) 

1606 self.debugView(ccdExposure, "doDark") 

1607 

1608 if self.config.doFringe and not self.config.fringeAfterFlat: 

1609 self.log.info("Applying fringe correction before flat.") 

1610 self.fringe.run(ccdExposure, **fringes.getDict()) 

1611 self.debugView(ccdExposure, "doFringe") 

1612 

1613 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1614 self.log.info("Checking strayLight correction.") 

1615 self.strayLight.run(ccdExposure, strayLightData) 

1616 self.debugView(ccdExposure, "doStrayLight") 

1617 

1618 if self.config.doFlat: 

1619 self.log.info("Applying flat correction.") 

1620 self.flatCorrection(ccdExposure, flat) 

1621 self.debugView(ccdExposure, "doFlat") 

1622 

1623 if self.config.doApplyGains: 

1624 self.log.info("Applying gain correction instead of flat.") 

1625 if self.config.usePtcGains: 

1626 self.log.info("Using gains from the Photon Transfer Curve.") 

1627 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1628 ptcGains=ptc.gain) 

1629 else: 

1630 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1631 

1632 if self.config.doFringe and self.config.fringeAfterFlat: 

1633 self.log.info("Applying fringe correction after flat.") 

1634 self.fringe.run(ccdExposure, **fringes.getDict()) 

1635 

1636 if self.config.doVignette: 

1637 if self.config.doMaskVignettePolygon: 

1638 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1639 else: 

1640 self.log.info("Constructing and attaching vignette polygon.") 

1641 self.vignettePolygon = self.vignette.run( 

1642 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1643 vignetteValue=self.config.vignetteValue, log=self.log) 

1644 

1645 if self.config.doAttachTransmissionCurve: 

1646 self.log.info("Adding transmission curves.") 

1647 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1648 filterTransmission=filterTransmission, 

1649 sensorTransmission=sensorTransmission, 

1650 atmosphereTransmission=atmosphereTransmission) 

1651 

1652 flattenedThumb = None 

1653 if self.config.qa.doThumbnailFlattened: 

1654 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1655 

1656 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1657 self.log.info("Performing illumination correction.") 

1658 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1659 illumMaskedImage, illumScale=self.config.illumScale, 

1660 trimToFit=self.config.doTrimToMatchCalib) 

1661 

1662 preInterpExp = None 

1663 if self.config.doSaveInterpPixels: 

1664 preInterpExp = ccdExposure.clone() 

1665 

1666 # Reset and interpolate bad pixels. 

1667 # 

1668 # Large contiguous bad regions (which should have the BAD mask 

1669 # bit set) should have their values set to the image median. 

1670 # This group should include defects and bad amplifiers. As the 

1671 # area covered by these defects are large, there's little 

1672 # reason to expect that interpolation would provide a more 

1673 # useful value. 

1674 # 

1675 # Smaller defects can be safely interpolated after the larger 

1676 # regions have had their pixel values reset. This ensures 

1677 # that the remaining defects adjacent to bad amplifiers (as an 

1678 # example) do not attempt to interpolate extreme values. 

1679 if self.config.doSetBadRegions: 

1680 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1681 if badPixelCount > 0: 

1682 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1683 

1684 if self.config.doInterpolate: 

1685 self.log.info("Interpolating masked pixels.") 

1686 isrFunctions.interpolateFromMask( 

1687 maskedImage=ccdExposure.getMaskedImage(), 

1688 fwhm=self.config.fwhm, 

1689 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1690 maskNameList=list(self.config.maskListToInterpolate) 

1691 ) 

1692 

1693 self.roughZeroPoint(ccdExposure) 

1694 

1695 # correct for amp offsets within the CCD 

1696 if self.config.doAmpOffset: 

1697 self.log.info("Correcting amp offsets.") 

1698 self.ampOffset.run(ccdExposure) 

1699 

1700 if self.config.doMeasureBackground: 

1701 self.log.info("Measuring background level.") 

1702 self.measureBackground(ccdExposure, self.config.qa) 

1703 

1704 if self.config.qa is not None and self.config.qa.saveStats is True: 

1705 for amp in ccd: 

1706 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1707 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1708 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1709 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1710 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1711 qaStats.getValue(afwMath.STDEVCLIP) 

1712 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1713 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1714 qaStats.getValue(afwMath.STDEVCLIP)) 

1715 

1716 # Calculate standard image quality statistics 

1717 if self.config.doStandardStatistics: 

1718 metadata = ccdExposure.getMetadata() 

1719 for amp in ccd: 

1720 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1721 ampName = amp.getName() 

1722 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1723 ampExposure.getMaskedImage(), 

1724 [self.config.saturatedMaskName] 

1725 ) 

1726 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1727 ampExposure.getMaskedImage(), 

1728 ["BAD"] 

1729 ) 

1730 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1731 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1732 

1733 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1734 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1735 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1736 

1737 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1738 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1739 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1740 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1741 else: 

1742 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1743 

1744 # calculate additional statistics. 

1745 outputStatistics = None 

1746 if self.config.doCalculateStatistics: 

1747 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1748 ptc=ptc).results 

1749 

1750 # do any binning. 

1751 outputBin1Exposure = None 

1752 outputBin2Exposure = None 

1753 if self.config.doBinnedExposures: 

1754 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure) 

1755 

1756 self.debugView(ccdExposure, "postISRCCD") 

1757 

1758 return pipeBase.Struct( 

1759 exposure=ccdExposure, 

1760 ossThumb=ossThumb, 

1761 flattenedThumb=flattenedThumb, 

1762 

1763 outputBin1Exposure=outputBin1Exposure, 

1764 outputBin2Exposure=outputBin2Exposure, 

1765 

1766 preInterpExposure=preInterpExp, 

1767 outputExposure=ccdExposure, 

1768 outputOssThumbnail=ossThumb, 

1769 outputFlattenedThumbnail=flattenedThumb, 

1770 outputStatistics=outputStatistics, 

1771 ) 

1772 

1773 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1774 """Ensure that the data returned by Butler is a fully constructed exp. 

1775 

1776 ISR requires exposure-level image data for historical reasons, so if we 

1777 did not recieve that from Butler, construct it from what we have, 

1778 modifying the input in place. 

1779 

1780 Parameters 

1781 ---------- 

1782 inputExp : `lsst.afw.image` image-type. 

1783 The input data structure obtained from Butler. 

1784 Can be `lsst.afw.image.Exposure`, 

1785 `lsst.afw.image.DecoratedImageU`, 

1786 or `lsst.afw.image.ImageF` 

1787 camera : `lsst.afw.cameraGeom.camera`, optional 

1788 The camera associated with the image. Used to find the appropriate 

1789 detector if detector is not already set. 

1790 detectorNum : `int`, optional 

1791 The detector in the camera to attach, if the detector is not 

1792 already set. 

1793 

1794 Returns 

1795 ------- 

1796 inputExp : `lsst.afw.image.Exposure` 

1797 The re-constructed exposure, with appropriate detector parameters. 

1798 

1799 Raises 

1800 ------ 

1801 TypeError 

1802 Raised if the input data cannot be used to construct an exposure. 

1803 """ 

1804 if isinstance(inputExp, afwImage.DecoratedImageU): 

1805 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1806 elif isinstance(inputExp, afwImage.ImageF): 

1807 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1808 elif isinstance(inputExp, afwImage.MaskedImageF): 

1809 inputExp = afwImage.makeExposure(inputExp) 

1810 elif isinstance(inputExp, afwImage.Exposure): 

1811 pass 

1812 elif inputExp is None: 

1813 # Assume this will be caught by the setup if it is a problem. 

1814 return inputExp 

1815 else: 

1816 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1817 (type(inputExp), )) 

1818 

1819 if inputExp.getDetector() is None: 

1820 if camera is None or detectorNum is None: 

1821 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1822 'without a detector set.') 

1823 inputExp.setDetector(camera[detectorNum]) 

1824 

1825 return inputExp 

1826 

1827 @staticmethod 

1828 def extractCalibDate(calib): 

1829 """Extract common calibration metadata values that will be written to 

1830 output header. 

1831 

1832 Parameters 

1833 ---------- 

1834 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1835 Calibration to pull date information from. 

1836 

1837 Returns 

1838 ------- 

1839 dateString : `str` 

1840 Calibration creation date string to add to header. 

1841 """ 

1842 if hasattr(calib, "getMetadata"): 

1843 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1844 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1845 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1846 else: 

1847 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1848 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1849 else: 

1850 return "Unknown Unknown" 

1851 

1852 def compareCameraKeywords(self, exposureMetadata, calib, calibName): 

1853 """Compare header keywords to confirm camera states match. 

1854 

1855 Parameters 

1856 ---------- 

1857 exposureMetadata : `lsst.daf.base.PropertySet` 

1858 Header for the exposure being processed. 

1859 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1860 Calibration to be applied. 

1861 calibName : `str` 

1862 Calib type for log message. 

1863 """ 

1864 try: 

1865 calibMetadata = calib.getMetadata() 

1866 except AttributeError: 

1867 return 

1868 for keyword in self.config.cameraKeywordsToCompare: 

1869 if keyword in exposureMetadata and keyword in calibMetadata: 

1870 if exposureMetadata[keyword] != calibMetadata[keyword]: 

1871 if self.config.doRaiseOnCalibMismatch: 

1872 raise RuntimeError("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1873 calibName, keyword, 

1874 exposureMetadata[keyword], calibMetadata[keyword]) 

1875 else: 

1876 self.log.warning("Sequencer mismatch for %s [%s]: exposure: %s calib: %s", 

1877 calibName, keyword, 

1878 exposureMetadata[keyword], calibMetadata[keyword]) 

1879 else: 

1880 self.log.debug("Sequencer keyword %s not found.", keyword) 

1881 

1882 def convertIntToFloat(self, exposure): 

1883 """Convert exposure image from uint16 to float. 

1884 

1885 If the exposure does not need to be converted, the input is 

1886 immediately returned. For exposures that are converted to use 

1887 floating point pixels, the variance is set to unity and the 

1888 mask to zero. 

1889 

1890 Parameters 

1891 ---------- 

1892 exposure : `lsst.afw.image.Exposure` 

1893 The raw exposure to be converted. 

1894 

1895 Returns 

1896 ------- 

1897 newexposure : `lsst.afw.image.Exposure` 

1898 The input ``exposure``, converted to floating point pixels. 

1899 

1900 Raises 

1901 ------ 

1902 RuntimeError 

1903 Raised if the exposure type cannot be converted to float. 

1904 

1905 """ 

1906 if isinstance(exposure, afwImage.ExposureF): 

1907 # Nothing to be done 

1908 self.log.debug("Exposure already of type float.") 

1909 return exposure 

1910 if not hasattr(exposure, "convertF"): 

1911 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1912 

1913 newexposure = exposure.convertF() 

1914 newexposure.variance[:] = 1 

1915 newexposure.mask[:] = 0x0 

1916 

1917 return newexposure 

1918 

1919 def maskAmplifier(self, ccdExposure, amp, defects): 

1920 """Identify bad amplifiers, saturated and suspect pixels. 

1921 

1922 Parameters 

1923 ---------- 

1924 ccdExposure : `lsst.afw.image.Exposure` 

1925 Input exposure to be masked. 

1926 amp : `lsst.afw.cameraGeom.Amplifier` 

1927 Catalog of parameters defining the amplifier on this 

1928 exposure to mask. 

1929 defects : `lsst.ip.isr.Defects` 

1930 List of defects. Used to determine if the entire 

1931 amplifier is bad. 

1932 

1933 Returns 

1934 ------- 

1935 badAmp : `Bool` 

1936 If this is true, the entire amplifier area is covered by 

1937 defects and unusable. 

1938 

1939 """ 

1940 maskedImage = ccdExposure.getMaskedImage() 

1941 

1942 badAmp = False 

1943 

1944 # Check if entire amp region is defined as a defect 

1945 # NB: need to use amp.getBBox() for correct comparison with current 

1946 # defects definition. 

1947 if defects is not None: 

1948 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1949 

1950 # In the case of a bad amp, we will set mask to "BAD" 

1951 # (here use amp.getRawBBox() for correct association with pixels in 

1952 # current ccdExposure). 

1953 if badAmp: 

1954 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1955 afwImage.PARENT) 

1956 maskView = dataView.getMask() 

1957 maskView |= maskView.getPlaneBitMask("BAD") 

1958 del maskView 

1959 return badAmp 

1960 

1961 # Mask remaining defects after assembleCcd() to allow for defects that 

1962 # cross amplifier boundaries. Saturation and suspect pixels can be 

1963 # masked now, though. 

1964 limits = dict() 

1965 if self.config.doSaturation and not badAmp: 

1966 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1967 if self.config.doSuspect and not badAmp: 

1968 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1969 if math.isfinite(self.config.saturation): 

1970 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1971 

1972 for maskName, maskThreshold in limits.items(): 

1973 if not math.isnan(maskThreshold): 

1974 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1975 isrFunctions.makeThresholdMask( 

1976 maskedImage=dataView, 

1977 threshold=maskThreshold, 

1978 growFootprints=0, 

1979 maskName=maskName 

1980 ) 

1981 

1982 # Determine if we've fully masked this amplifier with SUSPECT and 

1983 # SAT pixels. 

1984 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1985 afwImage.PARENT) 

1986 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1987 self.config.suspectMaskName]) 

1988 if numpy.all(maskView.getArray() & maskVal > 0): 

1989 badAmp = True 

1990 maskView |= maskView.getPlaneBitMask("BAD") 

1991 

1992 return badAmp 

1993 

1994 def overscanCorrection(self, ccdExposure, amp): 

1995 """Apply overscan correction in place. 

1996 

1997 This method does initial pixel rejection of the overscan 

1998 region. The overscan can also be optionally segmented to 

1999 allow for discontinuous overscan responses to be fit 

2000 separately. The actual overscan subtraction is performed by 

2001 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

2002 after the amplifier is preprocessed. 

2003 

2004 Parameters 

2005 ---------- 

2006 ccdExposure : `lsst.afw.image.Exposure` 

2007 Exposure to have overscan correction performed. 

2008 amp : `lsst.afw.cameraGeom.Amplifer` 

2009 The amplifier to consider while correcting the overscan. 

2010 

2011 Returns 

2012 ------- 

2013 overscanResults : `lsst.pipe.base.Struct` 

2014 Result struct with components: 

2015 

2016 ``imageFit`` 

2017 Value or fit subtracted from the amplifier image data. 

2018 (scalar or `lsst.afw.image.Image`) 

2019 ``overscanFit`` 

2020 Value or fit subtracted from the overscan image data. 

2021 (scalar or `lsst.afw.image.Image`) 

2022 ``overscanImage`` 

2023 Image of the overscan region with the overscan 

2024 correction applied. This quantity is used to estimate 

2025 the amplifier read noise empirically. 

2026 (`lsst.afw.image.Image`) 

2027 ``edgeMask`` 

2028 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

2029 ``overscanMean`` 

2030 Median overscan fit value. (`float`) 

2031 ``overscanSigma`` 

2032 Clipped standard deviation of the overscan after 

2033 correction. (`float`) 

2034 

2035 Raises 

2036 ------ 

2037 RuntimeError 

2038 Raised if the ``amp`` does not contain raw pixel information. 

2039 

2040 See Also 

2041 -------- 

2042 lsst.ip.isr.overscan.OverscanTask 

2043 """ 

2044 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2045 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2046 return None 

2047 

2048 # Perform overscan correction on subregions. 

2049 overscanResults = self.overscan.run(ccdExposure, amp) 

2050 

2051 metadata = ccdExposure.getMetadata() 

2052 ampName = amp.getName() 

2053 

2054 keyBase = "LSST ISR OVERSCAN" 

2055 # Updated quantities 

2056 if isinstance(overscanResults.overscanMean, float): 

2057 # Serial overscan correction only: 

2058 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

2059 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

2060 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

2061 

2062 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

2063 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

2064 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

2065 elif isinstance(overscanResults.overscanMean, tuple): 

2066 # Both serial and parallel overscan have run: 

2067 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

2068 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

2069 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

2070 

2071 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

2072 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

2073 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

2074 

2075 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

2076 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

2077 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

2078 

2079 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

2080 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

2081 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

2082 else: 

2083 self.log.warning("Unexpected type for overscan values; none added to header.") 

2084 

2085 return overscanResults 

2086 

2087 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2088 """Set the variance plane using the gain and read noise 

2089 

2090 The read noise is calculated from the ``overscanImage`` if the 

2091 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2092 the value from the amplifier data is used. 

2093 

2094 Parameters 

2095 ---------- 

2096 ampExposure : `lsst.afw.image.Exposure` 

2097 Exposure to process. 

2098 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

2099 Amplifier detector data. 

2100 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2101 Image of overscan, required only for empirical read noise. 

2102 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2103 PTC dataset containing the gains and read noise. 

2104 

2105 Raises 

2106 ------ 

2107 RuntimeError 

2108 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2109 are ``True``, but ptcDataset is not provided. 

2110 

2111 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2112 ``overscanImage`` is ``None``. 

2113 

2114 See also 

2115 -------- 

2116 lsst.ip.isr.isrFunctions.updateVariance 

2117 """ 

2118 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2119 if self.config.usePtcGains: 

2120 if ptcDataset is None: 

2121 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2122 else: 

2123 gain = ptcDataset.gain[amp.getName()] 

2124 self.log.info("Using gain from Photon Transfer Curve.") 

2125 else: 

2126 gain = amp.getGain() 

2127 

2128 if math.isnan(gain): 

2129 gain = 1.0 

2130 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2131 elif gain <= 0: 

2132 patchedGain = 1.0 

2133 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2134 amp.getName(), gain, patchedGain) 

2135 gain = patchedGain 

2136 

2137 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2138 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(), 

2139 [self.config.saturatedMaskName, 

2140 self.config.suspectMaskName, 

2141 "BAD", "NO_DATA"]) 

2142 allPixels = ampExposure.getWidth() * ampExposure.getHeight() 

2143 if allPixels == badPixels: 

2144 # If the image is bad, do not raise. 

2145 self.log.info("Skipping empirical read noise for amp %s. No good pixels.", 

2146 amp.getName()) 

2147 else: 

2148 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2149 

2150 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2151 stats = afwMath.StatisticsControl() 

2152 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2153 readNoise = afwMath.makeStatistics(overscanImage.getImage(), 

2154 afwMath.STDEVCLIP, stats).getValue() 

2155 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2156 amp.getName(), readNoise) 

2157 elif self.config.usePtcReadNoise: 

2158 if ptcDataset is None: 

2159 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2160 else: 

2161 readNoise = ptcDataset.noise[amp.getName()] 

2162 self.log.info("Using read noise from Photon Transfer Curve.") 

2163 else: 

2164 readNoise = amp.getReadNoise() 

2165 

2166 metadata = ampExposure.getMetadata() 

2167 metadata[f'LSST GAIN {amp.getName()}'] = gain 

2168 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise 

2169 

2170 isrFunctions.updateVariance( 

2171 maskedImage=ampExposure.getMaskedImage(), 

2172 gain=gain, 

2173 readNoise=readNoise, 

2174 ) 

2175 

2176 def maskNegativeVariance(self, exposure): 

2177 """Identify and mask pixels with negative variance values. 

2178 

2179 Parameters 

2180 ---------- 

2181 exposure : `lsst.afw.image.Exposure` 

2182 Exposure to process. 

2183 

2184 See Also 

2185 -------- 

2186 lsst.ip.isr.isrFunctions.updateVariance 

2187 """ 

2188 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2189 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2190 exposure.mask.array[bad] |= maskPlane 

2191 

2192 def darkCorrection(self, exposure, darkExposure, invert=False): 

2193 """Apply dark correction in place. 

2194 

2195 Parameters 

2196 ---------- 

2197 exposure : `lsst.afw.image.Exposure` 

2198 Exposure to process. 

2199 darkExposure : `lsst.afw.image.Exposure` 

2200 Dark exposure of the same size as ``exposure``. 

2201 invert : `Bool`, optional 

2202 If True, re-add the dark to an already corrected image. 

2203 

2204 Raises 

2205 ------ 

2206 RuntimeError 

2207 Raised if either ``exposure`` or ``darkExposure`` do not 

2208 have their dark time defined. 

2209 

2210 See Also 

2211 -------- 

2212 lsst.ip.isr.isrFunctions.darkCorrection 

2213 """ 

2214 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2215 if math.isnan(expScale): 

2216 raise RuntimeError("Exposure darktime is NAN.") 

2217 if darkExposure.getInfo().getVisitInfo() is not None \ 

2218 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2219 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2220 else: 

2221 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2222 # so getDarkTime() does not exist. 

2223 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2224 darkScale = 1.0 

2225 

2226 isrFunctions.darkCorrection( 

2227 maskedImage=exposure.getMaskedImage(), 

2228 darkMaskedImage=darkExposure.getMaskedImage(), 

2229 expScale=expScale, 

2230 darkScale=darkScale, 

2231 invert=invert, 

2232 trimToFit=self.config.doTrimToMatchCalib 

2233 ) 

2234 

2235 def doLinearize(self, detector): 

2236 """Check if linearization is needed for the detector cameraGeom. 

2237 

2238 Checks config.doLinearize and the linearity type of the first 

2239 amplifier. 

2240 

2241 Parameters 

2242 ---------- 

2243 detector : `lsst.afw.cameraGeom.Detector` 

2244 Detector to get linearity type from. 

2245 

2246 Returns 

2247 ------- 

2248 doLinearize : `Bool` 

2249 If True, linearization should be performed. 

2250 """ 

2251 return self.config.doLinearize and \ 

2252 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2253 

2254 def flatCorrection(self, exposure, flatExposure, invert=False): 

2255 """Apply flat correction in place. 

2256 

2257 Parameters 

2258 ---------- 

2259 exposure : `lsst.afw.image.Exposure` 

2260 Exposure to process. 

2261 flatExposure : `lsst.afw.image.Exposure` 

2262 Flat exposure of the same size as ``exposure``. 

2263 invert : `Bool`, optional 

2264 If True, unflatten an already flattened image. 

2265 

2266 See Also 

2267 -------- 

2268 lsst.ip.isr.isrFunctions.flatCorrection 

2269 """ 

2270 isrFunctions.flatCorrection( 

2271 maskedImage=exposure.getMaskedImage(), 

2272 flatMaskedImage=flatExposure.getMaskedImage(), 

2273 scalingType=self.config.flatScalingType, 

2274 userScale=self.config.flatUserScale, 

2275 invert=invert, 

2276 trimToFit=self.config.doTrimToMatchCalib 

2277 ) 

2278 

2279 def saturationDetection(self, exposure, amp): 

2280 """Detect and mask saturated pixels in config.saturatedMaskName. 

2281 

2282 Parameters 

2283 ---------- 

2284 exposure : `lsst.afw.image.Exposure` 

2285 Exposure to process. Only the amplifier DataSec is processed. 

2286 amp : `lsst.afw.cameraGeom.Amplifier` 

2287 Amplifier detector data. 

2288 

2289 See Also 

2290 -------- 

2291 lsst.ip.isr.isrFunctions.makeThresholdMask 

2292 """ 

2293 if not math.isnan(amp.getSaturation()): 

2294 maskedImage = exposure.getMaskedImage() 

2295 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2296 isrFunctions.makeThresholdMask( 

2297 maskedImage=dataView, 

2298 threshold=amp.getSaturation(), 

2299 growFootprints=0, 

2300 maskName=self.config.saturatedMaskName, 

2301 ) 

2302 

2303 def saturationInterpolation(self, exposure): 

2304 """Interpolate over saturated pixels, in place. 

2305 

2306 This method should be called after `saturationDetection`, to 

2307 ensure that the saturated pixels have been identified in the 

2308 SAT mask. It should also be called after `assembleCcd`, since 

2309 saturated regions may cross amplifier boundaries. 

2310 

2311 Parameters 

2312 ---------- 

2313 exposure : `lsst.afw.image.Exposure` 

2314 Exposure to process. 

2315 

2316 See Also 

2317 -------- 

2318 lsst.ip.isr.isrTask.saturationDetection 

2319 lsst.ip.isr.isrFunctions.interpolateFromMask 

2320 """ 

2321 isrFunctions.interpolateFromMask( 

2322 maskedImage=exposure.getMaskedImage(), 

2323 fwhm=self.config.fwhm, 

2324 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2325 maskNameList=list(self.config.saturatedMaskName), 

2326 ) 

2327 

2328 def suspectDetection(self, exposure, amp): 

2329 """Detect and mask suspect pixels in config.suspectMaskName. 

2330 

2331 Parameters 

2332 ---------- 

2333 exposure : `lsst.afw.image.Exposure` 

2334 Exposure to process. Only the amplifier DataSec is processed. 

2335 amp : `lsst.afw.cameraGeom.Amplifier` 

2336 Amplifier detector data. 

2337 

2338 See Also 

2339 -------- 

2340 lsst.ip.isr.isrFunctions.makeThresholdMask 

2341 

2342 Notes 

2343 ----- 

2344 Suspect pixels are pixels whose value is greater than 

2345 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2346 affected by unknown systematics; for example if non-linearity 

2347 corrections above a certain level are unstable then that would be a 

2348 useful value for suspectLevel. A value of `nan` indicates that no such 

2349 level exists and no pixels are to be masked as suspicious. 

2350 """ 

2351 suspectLevel = amp.getSuspectLevel() 

2352 if math.isnan(suspectLevel): 

2353 return 

2354 

2355 maskedImage = exposure.getMaskedImage() 

2356 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2357 isrFunctions.makeThresholdMask( 

2358 maskedImage=dataView, 

2359 threshold=suspectLevel, 

2360 growFootprints=0, 

2361 maskName=self.config.suspectMaskName, 

2362 ) 

2363 

2364 def maskDefect(self, exposure, defectBaseList): 

2365 """Mask defects using mask plane "BAD", in place. 

2366 

2367 Parameters 

2368 ---------- 

2369 exposure : `lsst.afw.image.Exposure` 

2370 Exposure to process. 

2371 defectBaseList : defect-type 

2372 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2373 or `list` of `lsst.afw.image.DefectBase`. 

2374 

2375 Notes 

2376 ----- 

2377 Call this after CCD assembly, since defects may cross amplifier 

2378 boundaries. 

2379 """ 

2380 maskedImage = exposure.getMaskedImage() 

2381 if not isinstance(defectBaseList, Defects): 

2382 # Promotes DefectBase to Defect 

2383 defectList = Defects(defectBaseList) 

2384 else: 

2385 defectList = defectBaseList 

2386 defectList.maskPixels(maskedImage, maskName="BAD") 

2387 

2388 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2389 """Mask edge pixels with applicable mask plane. 

2390 

2391 Parameters 

2392 ---------- 

2393 exposure : `lsst.afw.image.Exposure` 

2394 Exposure to process. 

2395 numEdgePixels : `int`, optional 

2396 Number of edge pixels to mask. 

2397 maskPlane : `str`, optional 

2398 Mask plane name to use. 

2399 level : `str`, optional 

2400 Level at which to mask edges. 

2401 """ 

2402 maskedImage = exposure.getMaskedImage() 

2403 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2404 

2405 if numEdgePixels > 0: 

2406 if level == 'DETECTOR': 

2407 boxes = [maskedImage.getBBox()] 

2408 elif level == 'AMP': 

2409 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2410 

2411 for box in boxes: 

2412 # This makes a bbox numEdgeSuspect pixels smaller than the 

2413 # image on each side 

2414 subImage = maskedImage[box] 

2415 box.grow(-numEdgePixels) 

2416 # Mask pixels outside box 

2417 SourceDetectionTask.setEdgeBits( 

2418 subImage, 

2419 box, 

2420 maskBitMask) 

2421 

2422 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2423 """Mask and interpolate defects using mask plane "BAD", in place. 

2424 

2425 Parameters 

2426 ---------- 

2427 exposure : `lsst.afw.image.Exposure` 

2428 Exposure to process. 

2429 defectBaseList : defects-like 

2430 List of defects to mask and interpolate. Can be 

2431 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2432 

2433 See Also 

2434 -------- 

2435 lsst.ip.isr.isrTask.maskDefect 

2436 """ 

2437 self.maskDefect(exposure, defectBaseList) 

2438 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2439 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2440 isrFunctions.interpolateFromMask( 

2441 maskedImage=exposure.getMaskedImage(), 

2442 fwhm=self.config.fwhm, 

2443 growSaturatedFootprints=0, 

2444 maskNameList=["BAD"], 

2445 ) 

2446 

2447 def maskNan(self, exposure): 

2448 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2449 

2450 Parameters 

2451 ---------- 

2452 exposure : `lsst.afw.image.Exposure` 

2453 Exposure to process. 

2454 

2455 Notes 

2456 ----- 

2457 We mask over all non-finite values (NaN, inf), including those 

2458 that are masked with other bits (because those may or may not be 

2459 interpolated over later, and we want to remove all NaN/infs). 

2460 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2461 preserve the historical name. 

2462 """ 

2463 maskedImage = exposure.getMaskedImage() 

2464 

2465 # Find and mask NaNs 

2466 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2467 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2468 numNans = maskNans(maskedImage, maskVal) 

2469 self.metadata["NUMNANS"] = numNans 

2470 if numNans > 0: 

2471 self.log.warning("There were %d unmasked NaNs.", numNans) 

2472 

2473 def maskAndInterpolateNan(self, exposure): 

2474 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2475 in place. 

2476 

2477 Parameters 

2478 ---------- 

2479 exposure : `lsst.afw.image.Exposure` 

2480 Exposure to process. 

2481 

2482 See Also 

2483 -------- 

2484 lsst.ip.isr.isrTask.maskNan 

2485 """ 

2486 self.maskNan(exposure) 

2487 isrFunctions.interpolateFromMask( 

2488 maskedImage=exposure.getMaskedImage(), 

2489 fwhm=self.config.fwhm, 

2490 growSaturatedFootprints=0, 

2491 maskNameList=["UNMASKEDNAN"], 

2492 ) 

2493 

2494 def measureBackground(self, exposure, IsrQaConfig=None): 

2495 """Measure the image background in subgrids, for quality control. 

2496 

2497 Parameters 

2498 ---------- 

2499 exposure : `lsst.afw.image.Exposure` 

2500 Exposure to process. 

2501 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2502 Configuration object containing parameters on which background 

2503 statistics and subgrids to use. 

2504 """ 

2505 if IsrQaConfig is not None: 

2506 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2507 IsrQaConfig.flatness.nIter) 

2508 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2509 statsControl.setAndMask(maskVal) 

2510 maskedImage = exposure.getMaskedImage() 

2511 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2512 skyLevel = stats.getValue(afwMath.MEDIAN) 

2513 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2514 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2515 metadata = exposure.getMetadata() 

2516 metadata["SKYLEVEL"] = skyLevel 

2517 metadata["SKYSIGMA"] = skySigma 

2518 

2519 # calcluating flatlevel over the subgrids 

2520 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2521 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2522 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2523 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2524 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2525 skyLevels = numpy.zeros((nX, nY)) 

2526 

2527 for j in range(nY): 

2528 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2529 for i in range(nX): 

2530 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2531 

2532 xLLC = xc - meshXHalf 

2533 yLLC = yc - meshYHalf 

2534 xURC = xc + meshXHalf - 1 

2535 yURC = yc + meshYHalf - 1 

2536 

2537 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2538 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2539 

2540 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2541 

2542 good = numpy.where(numpy.isfinite(skyLevels)) 

2543 skyMedian = numpy.median(skyLevels[good]) 

2544 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2545 flatness_rms = numpy.std(flatness) 

2546 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2547 

2548 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2549 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2550 nX, nY, flatness_pp, flatness_rms) 

2551 

2552 metadata["FLATNESS_PP"] = float(flatness_pp) 

2553 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2554 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2555 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2556 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2557 

2558 def roughZeroPoint(self, exposure): 

2559 """Set an approximate magnitude zero point for the exposure. 

2560 

2561 Parameters 

2562 ---------- 

2563 exposure : `lsst.afw.image.Exposure` 

2564 Exposure to process. 

2565 """ 

2566 filterLabel = exposure.getFilter() 

2567 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2568 

2569 if physicalFilter in self.config.fluxMag0T1: 

2570 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2571 else: 

2572 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2573 fluxMag0 = self.config.defaultFluxMag0T1 

2574 

2575 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2576 if not expTime > 0: # handle NaN as well as <= 0 

2577 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2578 return 

2579 

2580 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2581 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2582 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2583 

2584 @contextmanager 

2585 def flatContext(self, exp, flat, dark=None): 

2586 """Context manager that applies and removes flats and darks, 

2587 if the task is configured to apply them. 

2588 

2589 Parameters 

2590 ---------- 

2591 exp : `lsst.afw.image.Exposure` 

2592 Exposure to process. 

2593 flat : `lsst.afw.image.Exposure` 

2594 Flat exposure the same size as ``exp``. 

2595 dark : `lsst.afw.image.Exposure`, optional 

2596 Dark exposure the same size as ``exp``. 

2597 

2598 Yields 

2599 ------ 

2600 exp : `lsst.afw.image.Exposure` 

2601 The flat and dark corrected exposure. 

2602 """ 

2603 if self.config.doDark and dark is not None: 

2604 self.darkCorrection(exp, dark) 

2605 if self.config.doFlat: 

2606 self.flatCorrection(exp, flat) 

2607 try: 

2608 yield exp 

2609 finally: 

2610 if self.config.doFlat: 

2611 self.flatCorrection(exp, flat, invert=True) 

2612 if self.config.doDark and dark is not None: 

2613 self.darkCorrection(exp, dark, invert=True) 

2614 

2615 def makeBinnedImages(self, exposure): 

2616 """Make visualizeVisit style binned exposures. 

2617 

2618 Parameters 

2619 ---------- 

2620 exposure : `lsst.afw.image.Exposure` 

2621 Exposure to bin. 

2622 

2623 Returns 

2624 ------- 

2625 bin1 : `lsst.afw.image.Exposure` 

2626 Binned exposure using binFactor1. 

2627 bin2 : `lsst.afw.image.Exposure` 

2628 Binned exposure using binFactor2. 

2629 """ 

2630 mi = exposure.getMaskedImage() 

2631 

2632 bin1 = afwMath.binImage(mi, self.config.binFactor1) 

2633 bin2 = afwMath.binImage(mi, self.config.binFactor2) 

2634 

2635 return bin1, bin2 

2636 

2637 def debugView(self, exposure, stepname): 

2638 """Utility function to examine ISR exposure at different stages. 

2639 

2640 Parameters 

2641 ---------- 

2642 exposure : `lsst.afw.image.Exposure` 

2643 Exposure to view. 

2644 stepname : `str` 

2645 State of processing to view. 

2646 """ 

2647 frame = getDebugFrame(self._display, stepname) 

2648 if frame: 

2649 display = getDisplay(frame) 

2650 display.scale('asinh', 'zscale') 

2651 display.mtv(exposure) 

2652 prompt = "Press Enter to continue [c]... " 

2653 while True: 

2654 ans = input(prompt).lower() 

2655 if ans in ("", "c",): 

2656 break 

2657 

2658 

2659class FakeAmp(object): 

2660 """A Detector-like object that supports returning gain and saturation level 

2661 

2662 This is used when the input exposure does not have a detector. 

2663 

2664 Parameters 

2665 ---------- 

2666 exposure : `lsst.afw.image.Exposure` 

2667 Exposure to generate a fake amplifier for. 

2668 config : `lsst.ip.isr.isrTaskConfig` 

2669 Configuration to apply to the fake amplifier. 

2670 """ 

2671 

2672 def __init__(self, exposure, config): 

2673 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2674 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2675 self._gain = config.gain 

2676 self._readNoise = config.readNoise 

2677 self._saturation = config.saturation 

2678 

2679 def getBBox(self): 

2680 return self._bbox 

2681 

2682 def getRawBBox(self): 

2683 return self._bbox 

2684 

2685 def getRawHorizontalOverscanBBox(self): 

2686 return self._RawHorizontalOverscanBBox 

2687 

2688 def getGain(self): 

2689 return self._gain 

2690 

2691 def getReadNoise(self): 

2692 return self._readNoise 

2693 

2694 def getSaturation(self): 

2695 return self._saturation 

2696 

2697 def getSuspectLevel(self): 

2698 return float("NaN")