Coverage for python/lsst/ip/isr/isrTask.py: 17%

818 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-09 02:21 -0800

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from lsst.daf.butler import DimensionGraph 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

75 Data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

95 

96 

97class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

98 dimensions={"instrument", "exposure", "detector"}, 

99 defaultTemplates={}): 

100 ccdExposure = cT.Input( 

101 name="raw", 

102 doc="Input exposure to process.", 

103 storageClass="Exposure", 

104 dimensions=["instrument", "exposure", "detector"], 

105 ) 

106 camera = cT.PrerequisiteInput( 

107 name="camera", 

108 storageClass="Camera", 

109 doc="Input camera to construct complete exposures.", 

110 dimensions=["instrument"], 

111 isCalibration=True, 

112 ) 

113 

114 crosstalk = cT.PrerequisiteInput( 

115 name="crosstalk", 

116 doc="Input crosstalk object", 

117 storageClass="CrosstalkCalib", 

118 dimensions=["instrument", "detector"], 

119 isCalibration=True, 

120 minimum=0, # can fall back to cameraGeom 

121 ) 

122 crosstalkSources = cT.PrerequisiteInput( 

123 name="isrOverscanCorrected", 

124 doc="Overscan corrected input images.", 

125 storageClass="Exposure", 

126 dimensions=["instrument", "exposure", "detector"], 

127 deferLoad=True, 

128 multiple=True, 

129 lookupFunction=crosstalkSourceLookup, 

130 minimum=0, # not needed for all instruments, no config to control this 

131 ) 

132 bias = cT.PrerequisiteInput( 

133 name="bias", 

134 doc="Input bias calibration.", 

135 storageClass="ExposureF", 

136 dimensions=["instrument", "detector"], 

137 isCalibration=True, 

138 ) 

139 dark = cT.PrerequisiteInput( 

140 name='dark', 

141 doc="Input dark calibration.", 

142 storageClass="ExposureF", 

143 dimensions=["instrument", "detector"], 

144 isCalibration=True, 

145 ) 

146 flat = cT.PrerequisiteInput( 

147 name="flat", 

148 doc="Input flat calibration.", 

149 storageClass="ExposureF", 

150 dimensions=["instrument", "physical_filter", "detector"], 

151 isCalibration=True, 

152 ) 

153 ptc = cT.PrerequisiteInput( 

154 name="ptc", 

155 doc="Input Photon Transfer Curve dataset", 

156 storageClass="PhotonTransferCurveDataset", 

157 dimensions=["instrument", "detector"], 

158 isCalibration=True, 

159 ) 

160 fringes = cT.PrerequisiteInput( 

161 name="fringe", 

162 doc="Input fringe calibration.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "physical_filter", "detector"], 

165 isCalibration=True, 

166 minimum=0, # only needed for some bands, even when enabled 

167 ) 

168 strayLightData = cT.PrerequisiteInput( 

169 name='yBackground', 

170 doc="Input stray light calibration.", 

171 storageClass="StrayLightData", 

172 dimensions=["instrument", "physical_filter", "detector"], 

173 deferLoad=True, 

174 isCalibration=True, 

175 minimum=0, # only needed for some bands, even when enabled 

176 ) 

177 bfKernel = cT.PrerequisiteInput( 

178 name='bfKernel', 

179 doc="Input brighter-fatter kernel.", 

180 storageClass="NumpyArray", 

181 dimensions=["instrument"], 

182 isCalibration=True, 

183 minimum=0, # can use either bfKernel or newBFKernel 

184 ) 

185 newBFKernel = cT.PrerequisiteInput( 

186 name='brighterFatterKernel', 

187 doc="Newer complete kernel + gain solutions.", 

188 storageClass="BrighterFatterKernel", 

189 dimensions=["instrument", "detector"], 

190 isCalibration=True, 

191 minimum=0, # can use either bfKernel or newBFKernel 

192 ) 

193 defects = cT.PrerequisiteInput( 

194 name='defects', 

195 doc="Input defect tables.", 

196 storageClass="Defects", 

197 dimensions=["instrument", "detector"], 

198 isCalibration=True, 

199 ) 

200 linearizer = cT.PrerequisiteInput( 

201 name='linearizer', 

202 storageClass="Linearizer", 

203 doc="Linearity correction calibration.", 

204 dimensions=["instrument", "detector"], 

205 isCalibration=True, 

206 minimum=0, # can fall back to cameraGeom 

207 ) 

208 opticsTransmission = cT.PrerequisiteInput( 

209 name="transmission_optics", 

210 storageClass="TransmissionCurve", 

211 doc="Transmission curve due to the optics.", 

212 dimensions=["instrument"], 

213 isCalibration=True, 

214 ) 

215 filterTransmission = cT.PrerequisiteInput( 

216 name="transmission_filter", 

217 storageClass="TransmissionCurve", 

218 doc="Transmission curve due to the filter.", 

219 dimensions=["instrument", "physical_filter"], 

220 isCalibration=True, 

221 ) 

222 sensorTransmission = cT.PrerequisiteInput( 

223 name="transmission_sensor", 

224 storageClass="TransmissionCurve", 

225 doc="Transmission curve due to the sensor.", 

226 dimensions=["instrument", "detector"], 

227 isCalibration=True, 

228 ) 

229 atmosphereTransmission = cT.PrerequisiteInput( 

230 name="transmission_atmosphere", 

231 storageClass="TransmissionCurve", 

232 doc="Transmission curve due to the atmosphere.", 

233 dimensions=["instrument"], 

234 isCalibration=True, 

235 ) 

236 illumMaskedImage = cT.PrerequisiteInput( 

237 name="illum", 

238 doc="Input illumination correction.", 

239 storageClass="MaskedImageF", 

240 dimensions=["instrument", "physical_filter", "detector"], 

241 isCalibration=True, 

242 ) 

243 deferredChargeCalib = cT.PrerequisiteInput( 

244 name="cpCtiCalib", 

245 doc="Deferred charge/CTI correction dataset.", 

246 storageClass="IsrCalib", 

247 dimensions=["instrument", "detector"], 

248 isCalibration=True, 

249 ) 

250 

251 outputExposure = cT.Output( 

252 name='postISRCCD', 

253 doc="Output ISR processed exposure.", 

254 storageClass="Exposure", 

255 dimensions=["instrument", "exposure", "detector"], 

256 ) 

257 preInterpExposure = cT.Output( 

258 name='preInterpISRCCD', 

259 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

260 storageClass="ExposureF", 

261 dimensions=["instrument", "exposure", "detector"], 

262 ) 

263 outputOssThumbnail = cT.Output( 

264 name="OssThumb", 

265 doc="Output Overscan-subtracted thumbnail image.", 

266 storageClass="Thumbnail", 

267 dimensions=["instrument", "exposure", "detector"], 

268 ) 

269 outputFlattenedThumbnail = cT.Output( 

270 name="FlattenedThumb", 

271 doc="Output flat-corrected thumbnail image.", 

272 storageClass="Thumbnail", 

273 dimensions=["instrument", "exposure", "detector"], 

274 ) 

275 outputStatistics = cT.Output( 

276 name="isrStatistics", 

277 doc="Output of additional statistics table.", 

278 storageClass="StructuredDataDict", 

279 dimensions=["instrument", "exposure", "detector"], 

280 ) 

281 

282 def __init__(self, *, config=None): 

283 super().__init__(config=config) 

284 

285 if config.doBias is not True: 

286 self.prerequisiteInputs.remove("bias") 

287 if config.doLinearize is not True: 

288 self.prerequisiteInputs.remove("linearizer") 

289 if config.doCrosstalk is not True: 

290 self.prerequisiteInputs.remove("crosstalkSources") 

291 self.prerequisiteInputs.remove("crosstalk") 

292 if config.doBrighterFatter is not True: 

293 self.prerequisiteInputs.remove("bfKernel") 

294 self.prerequisiteInputs.remove("newBFKernel") 

295 if config.doDefect is not True: 

296 self.prerequisiteInputs.remove("defects") 

297 if config.doDark is not True: 

298 self.prerequisiteInputs.remove("dark") 

299 if config.doFlat is not True: 

300 self.prerequisiteInputs.remove("flat") 

301 if config.doFringe is not True: 

302 self.prerequisiteInputs.remove("fringes") 

303 if config.doStrayLight is not True: 

304 self.prerequisiteInputs.remove("strayLightData") 

305 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

306 self.prerequisiteInputs.remove("ptc") 

307 if config.doAttachTransmissionCurve is not True: 

308 self.prerequisiteInputs.remove("opticsTransmission") 

309 self.prerequisiteInputs.remove("filterTransmission") 

310 self.prerequisiteInputs.remove("sensorTransmission") 

311 self.prerequisiteInputs.remove("atmosphereTransmission") 

312 else: 

313 if config.doUseOpticsTransmission is not True: 

314 self.prerequisiteInputs.remove("opticsTransmission") 

315 if config.doUseFilterTransmission is not True: 

316 self.prerequisiteInputs.remove("filterTransmission") 

317 if config.doUseSensorTransmission is not True: 

318 self.prerequisiteInputs.remove("sensorTransmission") 

319 if config.doUseAtmosphereTransmission is not True: 

320 self.prerequisiteInputs.remove("atmosphereTransmission") 

321 if config.doIlluminationCorrection is not True: 

322 self.prerequisiteInputs.remove("illumMaskedImage") 

323 if config.doDeferredCharge is not True: 

324 self.prerequisiteInputs.remove("deferredChargeCalib") 

325 

326 if config.doWrite is not True: 

327 self.outputs.remove("outputExposure") 

328 self.outputs.remove("preInterpExposure") 

329 self.outputs.remove("outputFlattenedThumbnail") 

330 self.outputs.remove("outputOssThumbnail") 

331 self.outputs.remove("outputStatistics") 

332 

333 if config.doSaveInterpPixels is not True: 

334 self.outputs.remove("preInterpExposure") 

335 if config.qa.doThumbnailOss is not True: 

336 self.outputs.remove("outputOssThumbnail") 

337 if config.qa.doThumbnailFlattened is not True: 

338 self.outputs.remove("outputFlattenedThumbnail") 

339 if config.doCalculateStatistics is not True: 

340 self.outputs.remove("outputStatistics") 

341 

342 

343class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

344 pipelineConnections=IsrTaskConnections): 

345 """Configuration parameters for IsrTask. 

346 

347 Items are grouped in the order in which they are executed by the task. 

348 """ 

349 datasetType = pexConfig.Field( 

350 dtype=str, 

351 doc="Dataset type for input data; users will typically leave this alone, " 

352 "but camera-specific ISR tasks will override it", 

353 default="raw", 

354 ) 

355 

356 fallbackFilterName = pexConfig.Field( 

357 dtype=str, 

358 doc="Fallback default filter name for calibrations.", 

359 optional=True 

360 ) 

361 useFallbackDate = pexConfig.Field( 

362 dtype=bool, 

363 doc="Pass observation date when using fallback filter.", 

364 default=False, 

365 ) 

366 expectWcs = pexConfig.Field( 

367 dtype=bool, 

368 default=True, 

369 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

370 ) 

371 fwhm = pexConfig.Field( 

372 dtype=float, 

373 doc="FWHM of PSF in arcseconds.", 

374 default=1.0, 

375 ) 

376 qa = pexConfig.ConfigField( 

377 dtype=isrQa.IsrQaConfig, 

378 doc="QA related configuration options.", 

379 ) 

380 doHeaderProvenance = pexConfig.Field( 

381 dtype=bool, 

382 default=True, 

383 doc="Write calibration identifiers into output exposure header?", 

384 ) 

385 

386 # Image conversion configuration 

387 doConvertIntToFloat = pexConfig.Field( 

388 dtype=bool, 

389 doc="Convert integer raw images to floating point values?", 

390 default=True, 

391 ) 

392 

393 # Saturated pixel handling. 

394 doSaturation = pexConfig.Field( 

395 dtype=bool, 

396 doc="Mask saturated pixels? NB: this is totally independent of the" 

397 " interpolation option - this is ONLY setting the bits in the mask." 

398 " To have them interpolated make sure doSaturationInterpolation=True", 

399 default=True, 

400 ) 

401 saturatedMaskName = pexConfig.Field( 

402 dtype=str, 

403 doc="Name of mask plane to use in saturation detection and interpolation", 

404 default="SAT", 

405 ) 

406 saturation = pexConfig.Field( 

407 dtype=float, 

408 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

409 default=float("NaN"), 

410 ) 

411 growSaturationFootprintSize = pexConfig.Field( 

412 dtype=int, 

413 doc="Number of pixels by which to grow the saturation footprints", 

414 default=1, 

415 ) 

416 

417 # Suspect pixel handling. 

418 doSuspect = pexConfig.Field( 

419 dtype=bool, 

420 doc="Mask suspect pixels?", 

421 default=False, 

422 ) 

423 suspectMaskName = pexConfig.Field( 

424 dtype=str, 

425 doc="Name of mask plane to use for suspect pixels", 

426 default="SUSPECT", 

427 ) 

428 numEdgeSuspect = pexConfig.Field( 

429 dtype=int, 

430 doc="Number of edge pixels to be flagged as untrustworthy.", 

431 default=0, 

432 ) 

433 edgeMaskLevel = pexConfig.ChoiceField( 

434 dtype=str, 

435 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

436 default="DETECTOR", 

437 allowed={ 

438 'DETECTOR': 'Mask only the edges of the full detector.', 

439 'AMP': 'Mask edges of each amplifier.', 

440 }, 

441 ) 

442 

443 # Initial masking options. 

444 doSetBadRegions = pexConfig.Field( 

445 dtype=bool, 

446 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

447 default=True, 

448 ) 

449 badStatistic = pexConfig.ChoiceField( 

450 dtype=str, 

451 doc="How to estimate the average value for BAD regions.", 

452 default='MEANCLIP', 

453 allowed={ 

454 "MEANCLIP": "Correct using the (clipped) mean of good data", 

455 "MEDIAN": "Correct using the median of the good data", 

456 }, 

457 ) 

458 

459 # Overscan subtraction configuration. 

460 doOverscan = pexConfig.Field( 

461 dtype=bool, 

462 doc="Do overscan subtraction?", 

463 default=True, 

464 ) 

465 overscan = pexConfig.ConfigurableField( 

466 target=OverscanCorrectionTask, 

467 doc="Overscan subtraction task for image segments.", 

468 ) 

469 

470 # Amplifier to CCD assembly configuration 

471 doAssembleCcd = pexConfig.Field( 

472 dtype=bool, 

473 default=True, 

474 doc="Assemble amp-level exposures into a ccd-level exposure?" 

475 ) 

476 assembleCcd = pexConfig.ConfigurableField( 

477 target=AssembleCcdTask, 

478 doc="CCD assembly task", 

479 ) 

480 

481 # General calibration configuration. 

482 doAssembleIsrExposures = pexConfig.Field( 

483 dtype=bool, 

484 default=False, 

485 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

486 ) 

487 doTrimToMatchCalib = pexConfig.Field( 

488 dtype=bool, 

489 default=False, 

490 doc="Trim raw data to match calibration bounding boxes?" 

491 ) 

492 

493 # Bias subtraction. 

494 doBias = pexConfig.Field( 

495 dtype=bool, 

496 doc="Apply bias frame correction?", 

497 default=True, 

498 ) 

499 biasDataProductName = pexConfig.Field( 

500 dtype=str, 

501 doc="Name of the bias data product", 

502 default="bias", 

503 ) 

504 doBiasBeforeOverscan = pexConfig.Field( 

505 dtype=bool, 

506 doc="Reverse order of overscan and bias correction.", 

507 default=False 

508 ) 

509 

510 # Deferred charge correction. 

511 doDeferredCharge = pexConfig.Field( 

512 dtype=bool, 

513 doc="Apply deferred charge correction?", 

514 default=False, 

515 ) 

516 deferredChargeCorrection = pexConfig.ConfigurableField( 

517 target=DeferredChargeTask, 

518 doc="Deferred charge correction task.", 

519 ) 

520 

521 # Variance construction 

522 doVariance = pexConfig.Field( 

523 dtype=bool, 

524 doc="Calculate variance?", 

525 default=True 

526 ) 

527 gain = pexConfig.Field( 

528 dtype=float, 

529 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

530 default=float("NaN"), 

531 ) 

532 readNoise = pexConfig.Field( 

533 dtype=float, 

534 doc="The read noise to use if no Detector is present in the Exposure", 

535 default=0.0, 

536 ) 

537 doEmpiricalReadNoise = pexConfig.Field( 

538 dtype=bool, 

539 default=False, 

540 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

541 ) 

542 usePtcReadNoise = pexConfig.Field( 

543 dtype=bool, 

544 default=False, 

545 doc="Use readnoise values from the Photon Transfer Curve?" 

546 ) 

547 maskNegativeVariance = pexConfig.Field( 

548 dtype=bool, 

549 default=True, 

550 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

551 "in the measurement of the overscan at an edge due to the data falling off faster " 

552 "than the overscan model can account for it." 

553 ) 

554 negativeVarianceMaskName = pexConfig.Field( 

555 dtype=str, 

556 default="BAD", 

557 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

558 ) 

559 # Linearization. 

560 doLinearize = pexConfig.Field( 

561 dtype=bool, 

562 doc="Correct for nonlinearity of the detector's response?", 

563 default=True, 

564 ) 

565 

566 # Crosstalk. 

567 doCrosstalk = pexConfig.Field( 

568 dtype=bool, 

569 doc="Apply intra-CCD crosstalk correction?", 

570 default=False, 

571 ) 

572 doCrosstalkBeforeAssemble = pexConfig.Field( 

573 dtype=bool, 

574 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

575 default=False, 

576 ) 

577 crosstalk = pexConfig.ConfigurableField( 

578 target=CrosstalkTask, 

579 doc="Intra-CCD crosstalk correction", 

580 ) 

581 

582 # Masking options. 

583 doDefect = pexConfig.Field( 

584 dtype=bool, 

585 doc="Apply correction for CCD defects, e.g. hot pixels?", 

586 default=True, 

587 ) 

588 doNanMasking = pexConfig.Field( 

589 dtype=bool, 

590 doc="Mask non-finite (NAN, inf) pixels?", 

591 default=True, 

592 ) 

593 doWidenSaturationTrails = pexConfig.Field( 

594 dtype=bool, 

595 doc="Widen bleed trails based on their width?", 

596 default=True 

597 ) 

598 

599 # Brighter-Fatter correction. 

600 doBrighterFatter = pexConfig.Field( 

601 dtype=bool, 

602 default=False, 

603 doc="Apply the brighter-fatter correction?" 

604 ) 

605 brighterFatterLevel = pexConfig.ChoiceField( 

606 dtype=str, 

607 default="DETECTOR", 

608 doc="The level at which to correct for brighter-fatter.", 

609 allowed={ 

610 "AMP": "Every amplifier treated separately.", 

611 "DETECTOR": "One kernel per detector", 

612 } 

613 ) 

614 brighterFatterMaxIter = pexConfig.Field( 

615 dtype=int, 

616 default=10, 

617 doc="Maximum number of iterations for the brighter-fatter correction" 

618 ) 

619 brighterFatterThreshold = pexConfig.Field( 

620 dtype=float, 

621 default=1000, 

622 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

623 "absolute value of the difference between the current corrected image and the one " 

624 "from the previous iteration summed over all the pixels." 

625 ) 

626 brighterFatterApplyGain = pexConfig.Field( 

627 dtype=bool, 

628 default=True, 

629 doc="Should the gain be applied when applying the brighter-fatter correction?" 

630 ) 

631 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

632 dtype=str, 

633 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

634 "correction.", 

635 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

636 ) 

637 brighterFatterMaskGrowSize = pexConfig.Field( 

638 dtype=int, 

639 default=0, 

640 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

641 "when brighter-fatter correction is applied." 

642 ) 

643 

644 # Dark subtraction. 

645 doDark = pexConfig.Field( 

646 dtype=bool, 

647 doc="Apply dark frame correction?", 

648 default=True, 

649 ) 

650 darkDataProductName = pexConfig.Field( 

651 dtype=str, 

652 doc="Name of the dark data product", 

653 default="dark", 

654 ) 

655 

656 # Camera-specific stray light removal. 

657 doStrayLight = pexConfig.Field( 

658 dtype=bool, 

659 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

660 default=False, 

661 ) 

662 strayLight = pexConfig.ConfigurableField( 

663 target=StrayLightTask, 

664 doc="y-band stray light correction" 

665 ) 

666 

667 # Flat correction. 

668 doFlat = pexConfig.Field( 

669 dtype=bool, 

670 doc="Apply flat field correction?", 

671 default=True, 

672 ) 

673 flatDataProductName = pexConfig.Field( 

674 dtype=str, 

675 doc="Name of the flat data product", 

676 default="flat", 

677 ) 

678 flatScalingType = pexConfig.ChoiceField( 

679 dtype=str, 

680 doc="The method for scaling the flat on the fly.", 

681 default='USER', 

682 allowed={ 

683 "USER": "Scale by flatUserScale", 

684 "MEAN": "Scale by the inverse of the mean", 

685 "MEDIAN": "Scale by the inverse of the median", 

686 }, 

687 ) 

688 flatUserScale = pexConfig.Field( 

689 dtype=float, 

690 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

691 default=1.0, 

692 ) 

693 doTweakFlat = pexConfig.Field( 

694 dtype=bool, 

695 doc="Tweak flats to match observed amplifier ratios?", 

696 default=False 

697 ) 

698 

699 # Amplifier normalization based on gains instead of using flats 

700 # configuration. 

701 doApplyGains = pexConfig.Field( 

702 dtype=bool, 

703 doc="Correct the amplifiers for their gains instead of applying flat correction", 

704 default=False, 

705 ) 

706 usePtcGains = pexConfig.Field( 

707 dtype=bool, 

708 doc="Use the gain values from the Photon Transfer Curve?", 

709 default=False, 

710 ) 

711 normalizeGains = pexConfig.Field( 

712 dtype=bool, 

713 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

714 default=False, 

715 ) 

716 

717 # Fringe correction. 

718 doFringe = pexConfig.Field( 

719 dtype=bool, 

720 doc="Apply fringe correction?", 

721 default=True, 

722 ) 

723 fringe = pexConfig.ConfigurableField( 

724 target=FringeTask, 

725 doc="Fringe subtraction task", 

726 ) 

727 fringeAfterFlat = pexConfig.Field( 

728 dtype=bool, 

729 doc="Do fringe subtraction after flat-fielding?", 

730 default=True, 

731 ) 

732 

733 # Amp offset correction. 

734 doAmpOffset = pexConfig.Field( 

735 doc="Calculate and apply amp offset corrections?", 

736 dtype=bool, 

737 default=False, 

738 ) 

739 ampOffset = pexConfig.ConfigurableField( 

740 doc="Amp offset correction task.", 

741 target=AmpOffsetTask, 

742 ) 

743 

744 # Initial CCD-level background statistics options. 

745 doMeasureBackground = pexConfig.Field( 

746 dtype=bool, 

747 doc="Measure the background level on the reduced image?", 

748 default=False, 

749 ) 

750 

751 # Camera-specific masking configuration. 

752 doCameraSpecificMasking = pexConfig.Field( 

753 dtype=bool, 

754 doc="Mask camera-specific bad regions?", 

755 default=False, 

756 ) 

757 masking = pexConfig.ConfigurableField( 

758 target=MaskingTask, 

759 doc="Masking task." 

760 ) 

761 

762 # Interpolation options. 

763 doInterpolate = pexConfig.Field( 

764 dtype=bool, 

765 doc="Interpolate masked pixels?", 

766 default=True, 

767 ) 

768 doSaturationInterpolation = pexConfig.Field( 

769 dtype=bool, 

770 doc="Perform interpolation over pixels masked as saturated?" 

771 " NB: This is independent of doSaturation; if that is False this plane" 

772 " will likely be blank, resulting in a no-op here.", 

773 default=True, 

774 ) 

775 doNanInterpolation = pexConfig.Field( 

776 dtype=bool, 

777 doc="Perform interpolation over pixels masked as NaN?" 

778 " NB: This is independent of doNanMasking; if that is False this plane" 

779 " will likely be blank, resulting in a no-op here.", 

780 default=True, 

781 ) 

782 doNanInterpAfterFlat = pexConfig.Field( 

783 dtype=bool, 

784 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

785 "also have to interpolate them before flat-fielding."), 

786 default=False, 

787 ) 

788 maskListToInterpolate = pexConfig.ListField( 

789 dtype=str, 

790 doc="List of mask planes that should be interpolated.", 

791 default=['SAT', 'BAD'], 

792 ) 

793 doSaveInterpPixels = pexConfig.Field( 

794 dtype=bool, 

795 doc="Save a copy of the pre-interpolated pixel values?", 

796 default=False, 

797 ) 

798 

799 # Default photometric calibration options. 

800 fluxMag0T1 = pexConfig.DictField( 

801 keytype=str, 

802 itemtype=float, 

803 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

804 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

805 )) 

806 ) 

807 defaultFluxMag0T1 = pexConfig.Field( 

808 dtype=float, 

809 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

810 default=pow(10.0, 0.4*28.0) 

811 ) 

812 

813 # Vignette correction configuration. 

814 doVignette = pexConfig.Field( 

815 dtype=bool, 

816 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

817 "according to vignetting parameters?"), 

818 default=False, 

819 ) 

820 doMaskVignettePolygon = pexConfig.Field( 

821 dtype=bool, 

822 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

823 "is False"), 

824 default=True, 

825 ) 

826 vignetteValue = pexConfig.Field( 

827 dtype=float, 

828 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

829 optional=True, 

830 default=None, 

831 ) 

832 vignette = pexConfig.ConfigurableField( 

833 target=VignetteTask, 

834 doc="Vignetting task.", 

835 ) 

836 

837 # Transmission curve configuration. 

838 doAttachTransmissionCurve = pexConfig.Field( 

839 dtype=bool, 

840 default=False, 

841 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

842 ) 

843 doUseOpticsTransmission = pexConfig.Field( 

844 dtype=bool, 

845 default=True, 

846 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

847 ) 

848 doUseFilterTransmission = pexConfig.Field( 

849 dtype=bool, 

850 default=True, 

851 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

852 ) 

853 doUseSensorTransmission = pexConfig.Field( 

854 dtype=bool, 

855 default=True, 

856 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

857 ) 

858 doUseAtmosphereTransmission = pexConfig.Field( 

859 dtype=bool, 

860 default=True, 

861 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

862 ) 

863 

864 # Illumination correction. 

865 doIlluminationCorrection = pexConfig.Field( 

866 dtype=bool, 

867 default=False, 

868 doc="Perform illumination correction?" 

869 ) 

870 illuminationCorrectionDataProductName = pexConfig.Field( 

871 dtype=str, 

872 doc="Name of the illumination correction data product.", 

873 default="illumcor", 

874 ) 

875 illumScale = pexConfig.Field( 

876 dtype=float, 

877 doc="Scale factor for the illumination correction.", 

878 default=1.0, 

879 ) 

880 illumFilters = pexConfig.ListField( 

881 dtype=str, 

882 default=[], 

883 doc="Only perform illumination correction for these filters." 

884 ) 

885 

886 # Calculate additional statistics? 

887 doCalculateStatistics = pexConfig.Field( 

888 dtype=bool, 

889 doc="Should additional ISR statistics be calculated?", 

890 default=False, 

891 ) 

892 isrStats = pexConfig.ConfigurableField( 

893 target=IsrStatisticsTask, 

894 doc="Task to calculate additional statistics.", 

895 ) 

896 

897 # Write the outputs to disk. If ISR is run as a subtask, this may not 

898 # be needed. 

899 doWrite = pexConfig.Field( 

900 dtype=bool, 

901 doc="Persist postISRCCD?", 

902 default=True, 

903 ) 

904 

905 def validate(self): 

906 super().validate() 

907 if self.doFlat and self.doApplyGains: 

908 raise ValueError("You may not specify both doFlat and doApplyGains") 

909 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

910 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

911 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

912 self.maskListToInterpolate.append(self.saturatedMaskName) 

913 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

914 self.maskListToInterpolate.remove(self.saturatedMaskName) 

915 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

916 self.maskListToInterpolate.append("UNMASKEDNAN") 

917 

918 

919class IsrTask(pipeBase.PipelineTask): 

920 """Apply common instrument signature correction algorithms to a raw frame. 

921 

922 The process for correcting imaging data is very similar from 

923 camera to camera. This task provides a vanilla implementation of 

924 doing these corrections, including the ability to turn certain 

925 corrections off if they are not needed. The inputs to the primary 

926 method, `run()`, are a raw exposure to be corrected and the 

927 calibration data products. The raw input is a single chip sized 

928 mosaic of all amps including overscans and other non-science 

929 pixels. 

930 

931 The __init__ method sets up the subtasks for ISR processing, using 

932 the defaults from `lsst.ip.isr`. 

933 

934 Parameters 

935 ---------- 

936 args : `list` 

937 Positional arguments passed to the Task constructor. 

938 None used at this time. 

939 kwargs : `dict`, optional 

940 Keyword arguments passed on to the Task constructor. 

941 None used at this time. 

942 """ 

943 ConfigClass = IsrTaskConfig 

944 _DefaultName = "isr" 

945 

946 def __init__(self, **kwargs): 

947 super().__init__(**kwargs) 

948 self.makeSubtask("assembleCcd") 

949 self.makeSubtask("crosstalk") 

950 self.makeSubtask("strayLight") 

951 self.makeSubtask("fringe") 

952 self.makeSubtask("masking") 

953 self.makeSubtask("overscan") 

954 self.makeSubtask("vignette") 

955 self.makeSubtask("ampOffset") 

956 self.makeSubtask("deferredChargeCorrection") 

957 self.makeSubtask("isrStats") 

958 

959 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

960 inputs = butlerQC.get(inputRefs) 

961 

962 try: 

963 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

964 except Exception as e: 

965 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

966 (inputRefs, e)) 

967 

968 detector = inputs['ccdExposure'].getDetector() 

969 

970 if self.config.doCrosstalk is True: 

971 # Crosstalk sources need to be defined by the pipeline 

972 # yaml if they exist. 

973 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

974 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

975 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

976 else: 

977 coeffVector = (self.config.crosstalk.crosstalkValues 

978 if self.config.crosstalk.useConfigCoefficients else None) 

979 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

980 inputs['crosstalk'] = crosstalkCalib 

981 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

982 if 'crosstalkSources' not in inputs: 

983 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

984 

985 if self.doLinearize(detector) is True: 

986 if 'linearizer' in inputs: 

987 if isinstance(inputs['linearizer'], dict): 

988 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

989 linearizer.fromYaml(inputs['linearizer']) 

990 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

991 elif isinstance(inputs['linearizer'], numpy.ndarray): 

992 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

993 detector=detector, 

994 log=self.log) 

995 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

996 else: 

997 linearizer = inputs['linearizer'] 

998 linearizer.log = self.log 

999 inputs['linearizer'] = linearizer 

1000 else: 

1001 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1002 self.log.warning("Constructing linearizer from cameraGeom information.") 

1003 

1004 if self.config.doDefect is True: 

1005 if "defects" in inputs and inputs['defects'] is not None: 

1006 # defects is loaded as a BaseCatalog with columns 

1007 # x0, y0, width, height. Masking expects a list of defects 

1008 # defined by their bounding box 

1009 if not isinstance(inputs["defects"], Defects): 

1010 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1011 

1012 # Load the correct style of brighter-fatter kernel, and repack 

1013 # the information as a numpy array. 

1014 if self.config.doBrighterFatter: 

1015 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1016 if brighterFatterKernel is None: 

1017 brighterFatterKernel = inputs.get('bfKernel', None) 

1018 

1019 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1020 # This is a ISR calib kernel 

1021 detName = detector.getName() 

1022 level = brighterFatterKernel.level 

1023 

1024 # This is expected to be a dictionary of amp-wise gains. 

1025 inputs['bfGains'] = brighterFatterKernel.gain 

1026 if self.config.brighterFatterLevel == 'DETECTOR': 

1027 if level == 'DETECTOR': 

1028 if detName in brighterFatterKernel.detKernels: 

1029 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1030 else: 

1031 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1032 elif level == 'AMP': 

1033 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1034 "fatter kernels.") 

1035 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1036 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1037 elif self.config.brighterFatterLevel == 'AMP': 

1038 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1039 

1040 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1041 expId = inputs['ccdExposure'].info.id 

1042 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1043 expId=expId, 

1044 assembler=self.assembleCcd 

1045 if self.config.doAssembleIsrExposures else None) 

1046 else: 

1047 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1048 

1049 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1050 if 'strayLightData' not in inputs: 

1051 inputs['strayLightData'] = None 

1052 

1053 if self.config.doHeaderProvenance: 

1054 # Add calibration provenanace info to header. 

1055 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1056 for inputName in sorted(inputs.keys()): 

1057 reference = getattr(inputRefs, inputName, None) 

1058 if reference is not None and hasattr(reference, "run"): 

1059 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1060 runValue = reference.run 

1061 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1062 idValue = str(reference.id) 

1063 

1064 exposureMetadata[runKey] = runValue 

1065 exposureMetadata[idKey] = idValue 

1066 

1067 outputs = self.run(**inputs) 

1068 butlerQC.put(outputs, outputRefs) 

1069 

1070 @timeMethod 

1071 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1072 crosstalk=None, crosstalkSources=None, 

1073 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1074 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1075 sensorTransmission=None, atmosphereTransmission=None, 

1076 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1077 deferredChargeCalib=None, 

1078 ): 

1079 """Perform instrument signature removal on an exposure. 

1080 

1081 Steps included in the ISR processing, in order performed, are: 

1082 

1083 - saturation and suspect pixel masking 

1084 - overscan subtraction 

1085 - CCD assembly of individual amplifiers 

1086 - bias subtraction 

1087 - variance image construction 

1088 - linearization of non-linear response 

1089 - crosstalk masking 

1090 - brighter-fatter correction 

1091 - dark subtraction 

1092 - fringe correction 

1093 - stray light subtraction 

1094 - flat correction 

1095 - masking of known defects and camera specific features 

1096 - vignette calculation 

1097 - appending transmission curve and distortion model 

1098 

1099 Parameters 

1100 ---------- 

1101 ccdExposure : `lsst.afw.image.Exposure` 

1102 The raw exposure that is to be run through ISR. The 

1103 exposure is modified by this method. 

1104 camera : `lsst.afw.cameraGeom.Camera`, optional 

1105 The camera geometry for this exposure. Required if 

1106 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1107 ``flat`` does not have an associated detector. 

1108 bias : `lsst.afw.image.Exposure`, optional 

1109 Bias calibration frame. 

1110 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1111 Functor for linearization. 

1112 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1113 Calibration for crosstalk. 

1114 crosstalkSources : `list`, optional 

1115 List of possible crosstalk sources. 

1116 dark : `lsst.afw.image.Exposure`, optional 

1117 Dark calibration frame. 

1118 flat : `lsst.afw.image.Exposure`, optional 

1119 Flat calibration frame. 

1120 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1121 Photon transfer curve dataset, with, e.g., gains 

1122 and read noise. 

1123 bfKernel : `numpy.ndarray`, optional 

1124 Brighter-fatter kernel. 

1125 bfGains : `dict` of `float`, optional 

1126 Gains used to override the detector's nominal gains for the 

1127 brighter-fatter correction. A dict keyed by amplifier name for 

1128 the detector in question. 

1129 defects : `lsst.ip.isr.Defects`, optional 

1130 List of defects. 

1131 fringes : `lsst.pipe.base.Struct`, optional 

1132 Struct containing the fringe correction data, with 

1133 elements: 

1134 

1135 ``fringes`` 

1136 fringe calibration frame (`lsst.afw.image.Exposure`) 

1137 ``seed`` 

1138 random seed derived from the ``ccdExposureId`` for random 

1139 number generator (`numpy.uint32`) 

1140 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1141 A ``TransmissionCurve`` that represents the throughput of the, 

1142 optics, to be evaluated in focal-plane coordinates. 

1143 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1144 A ``TransmissionCurve`` that represents the throughput of the 

1145 filter itself, to be evaluated in focal-plane coordinates. 

1146 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1147 A ``TransmissionCurve`` that represents the throughput of the 

1148 sensor itself, to be evaluated in post-assembly trimmed detector 

1149 coordinates. 

1150 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1151 A ``TransmissionCurve`` that represents the throughput of the 

1152 atmosphere, assumed to be spatially constant. 

1153 detectorNum : `int`, optional 

1154 The integer number for the detector to process. 

1155 strayLightData : `object`, optional 

1156 Opaque object containing calibration information for stray-light 

1157 correction. If `None`, no correction will be performed. 

1158 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1159 Illumination correction image. 

1160 

1161 Returns 

1162 ------- 

1163 result : `lsst.pipe.base.Struct` 

1164 Result struct with component: 

1165 

1166 ``exposure`` 

1167 The fully ISR corrected exposure. 

1168 (`lsst.afw.image.Exposure`) 

1169 ``outputExposure`` 

1170 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1171 ``ossThumb`` 

1172 Thumbnail image of the exposure after overscan subtraction. 

1173 (`numpy.ndarray`) 

1174 ``flattenedThumb`` 

1175 Thumbnail image of the exposure after flat-field correction. 

1176 (`numpy.ndarray`) 

1177 ``outputStatistics`` 

1178 Values of the additional statistics calculated. 

1179 

1180 Raises 

1181 ------ 

1182 RuntimeError 

1183 Raised if a configuration option is set to `True`, but the 

1184 required calibration data has not been specified. 

1185 

1186 Notes 

1187 ----- 

1188 The current processed exposure can be viewed by setting the 

1189 appropriate `lsstDebug` entries in the ``debug.display`` 

1190 dictionary. The names of these entries correspond to some of 

1191 the `IsrTaskConfig` Boolean options, with the value denoting the 

1192 frame to use. The exposure is shown inside the matching 

1193 option check and after the processing of that step has 

1194 finished. The steps with debug points are: 

1195 

1196 * doAssembleCcd 

1197 * doBias 

1198 * doCrosstalk 

1199 * doBrighterFatter 

1200 * doDark 

1201 * doFringe 

1202 * doStrayLight 

1203 * doFlat 

1204 

1205 In addition, setting the ``postISRCCD`` entry displays the 

1206 exposure after all ISR processing has finished. 

1207 """ 

1208 

1209 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1210 bias = self.ensureExposure(bias, camera, detectorNum) 

1211 dark = self.ensureExposure(dark, camera, detectorNum) 

1212 flat = self.ensureExposure(flat, camera, detectorNum) 

1213 

1214 ccd = ccdExposure.getDetector() 

1215 filterLabel = ccdExposure.getFilter() 

1216 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1217 

1218 if not ccd: 

1219 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1220 ccd = [FakeAmp(ccdExposure, self.config)] 

1221 

1222 # Validate Input 

1223 if self.config.doBias and bias is None: 

1224 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1225 if self.doLinearize(ccd) and linearizer is None: 

1226 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1227 if self.config.doBrighterFatter and bfKernel is None: 

1228 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1229 if self.config.doDark and dark is None: 

1230 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1231 if self.config.doFlat and flat is None: 

1232 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1233 if self.config.doDefect and defects is None: 

1234 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1235 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1236 and fringes.fringes is None): 

1237 # The `fringes` object needs to be a pipeBase.Struct, as 

1238 # we use it as a `dict` for the parameters of 

1239 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1240 # not be `None` if `doFringe=True`. Otherwise, raise. 

1241 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1242 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1243 and illumMaskedImage is None): 

1244 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1245 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1246 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1247 

1248 if self.config.doHeaderProvenance: 

1249 # Inputs have been validated, so we can add their date 

1250 # information to the output header. 

1251 exposureMetadata = ccdExposure.getMetadata() 

1252 if self.config.doBias: 

1253 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias) 

1254 if self.config.doBrighterFatter: 

1255 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel) 

1256 if self.config.doCrosstalk: 

1257 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk) 

1258 if self.config.doDark: 

1259 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark) 

1260 if self.config.doDefect: 

1261 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects) 

1262 if self.config.doDeferredCharge: 

1263 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib) 

1264 if self.config.doFlat: 

1265 exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat) 

1266 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1267 exposureMetadata["LSST CALIB DATE FRINGE"] = self.extractCalibDate(fringes.fringes) 

1268 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1269 exposureMetadata["LSST CALIB DATE ILLUMINATION"] = self.extractCalibDate(illumMaskedImage) 

1270 if self.doLinearize(ccd): 

1271 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer) 

1272 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1273 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc) 

1274 if self.config.doStrayLight: 

1275 exposureMetadata["LSST CALIB DATE STRAYLIGHT"] = self.extractCalibDate(strayLightData) 

1276 if self.config.doAttachTransmissionCurve: 

1277 exposureMetadata["LSST CALIB DATE OPTICS_TR"] = self.extractCalibDate(opticsTransmission) 

1278 exposureMetadata["LSST CALIB DATE FILTER_TR"] = self.extractCalibDate(filterTransmission) 

1279 exposureMetadata["LSST CALIB DATE SENSOR_TR"] = self.extractCalibDate(sensorTransmission) 

1280 exposureMetadata["LSST CALIB DATE ATMOSP_TR"] = self.extractCalibDate(atmosphereTransmission) 

1281 

1282 # Begin ISR processing. 

1283 if self.config.doConvertIntToFloat: 

1284 self.log.info("Converting exposure to floating point values.") 

1285 ccdExposure = self.convertIntToFloat(ccdExposure) 

1286 

1287 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1288 self.log.info("Applying bias correction.") 

1289 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1290 trimToFit=self.config.doTrimToMatchCalib) 

1291 self.debugView(ccdExposure, "doBias") 

1292 

1293 # Amplifier level processing. 

1294 overscans = [] 

1295 for amp in ccd: 

1296 # if ccdExposure is one amp, 

1297 # check for coverage to prevent performing ops multiple times 

1298 if ccdExposure.getBBox().contains(amp.getBBox()): 

1299 # Check for fully masked bad amplifiers, 

1300 # and generate masks for SUSPECT and SATURATED values. 

1301 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1302 

1303 if self.config.doOverscan and not badAmp: 

1304 # Overscan correction on amp-by-amp basis. 

1305 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1306 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1307 if overscanResults is not None and \ 

1308 self.config.qa is not None and self.config.qa.saveStats is True: 

1309 

1310 self.metadata[f"FIT MEDIAN {amp.getName()}"] = overscanResults.overscanMean 

1311 self.metadata[f"FIT STDEV {amp.getName()}"] = overscanResults.overscanSigma 

1312 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1313 amp.getName(), overscanResults.overscanMean, 

1314 overscanResults.overscanSigma) 

1315 

1316 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = overscanResults.residualMean 

1317 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = overscanResults.residualSigma 

1318 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1319 amp.getName(), overscanResults.residualMean, 

1320 overscanResults.residualSigma) 

1321 

1322 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1323 else: 

1324 if badAmp: 

1325 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1326 overscanResults = None 

1327 

1328 overscans.append(overscanResults if overscanResults is not None else None) 

1329 else: 

1330 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1331 

1332 if self.config.doDeferredCharge: 

1333 self.log.info("Applying deferred charge/CTI correction.") 

1334 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1335 self.debugView(ccdExposure, "doDeferredCharge") 

1336 

1337 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1338 self.log.info("Applying crosstalk correction.") 

1339 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1340 crosstalkSources=crosstalkSources, camera=camera) 

1341 self.debugView(ccdExposure, "doCrosstalk") 

1342 

1343 if self.config.doAssembleCcd: 

1344 self.log.info("Assembling CCD from amplifiers.") 

1345 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1346 

1347 if self.config.expectWcs and not ccdExposure.getWcs(): 

1348 self.log.warning("No WCS found in input exposure.") 

1349 self.debugView(ccdExposure, "doAssembleCcd") 

1350 

1351 ossThumb = None 

1352 if self.config.qa.doThumbnailOss: 

1353 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1354 

1355 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1356 self.log.info("Applying bias correction.") 

1357 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1358 trimToFit=self.config.doTrimToMatchCalib) 

1359 self.debugView(ccdExposure, "doBias") 

1360 

1361 if self.config.doVariance: 

1362 for amp, overscanResults in zip(ccd, overscans): 

1363 if ccdExposure.getBBox().contains(amp.getBBox()): 

1364 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1365 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1366 if overscanResults is not None: 

1367 self.updateVariance(ampExposure, amp, 

1368 overscanImage=overscanResults.overscanImage, 

1369 ptcDataset=ptc) 

1370 else: 

1371 self.updateVariance(ampExposure, amp, 

1372 overscanImage=None, 

1373 ptcDataset=ptc) 

1374 if self.config.qa is not None and self.config.qa.saveStats is True: 

1375 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1376 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1377 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1378 qaStats.getValue(afwMath.MEDIAN) 

1379 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1380 qaStats.getValue(afwMath.STDEVCLIP) 

1381 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1382 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1383 qaStats.getValue(afwMath.STDEVCLIP)) 

1384 if self.config.maskNegativeVariance: 

1385 self.maskNegativeVariance(ccdExposure) 

1386 

1387 if self.doLinearize(ccd): 

1388 self.log.info("Applying linearizer.") 

1389 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1390 detector=ccd, log=self.log) 

1391 

1392 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1393 self.log.info("Applying crosstalk correction.") 

1394 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1395 crosstalkSources=crosstalkSources, isTrimmed=True) 

1396 self.debugView(ccdExposure, "doCrosstalk") 

1397 

1398 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1399 # widen trails, and do anything else the camera needs. Saturated and 

1400 # suspect pixels have already been masked. 

1401 if self.config.doDefect: 

1402 self.log.info("Masking defects.") 

1403 self.maskDefect(ccdExposure, defects) 

1404 

1405 if self.config.numEdgeSuspect > 0: 

1406 self.log.info("Masking edges as SUSPECT.") 

1407 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1408 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1409 

1410 if self.config.doNanMasking: 

1411 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1412 self.maskNan(ccdExposure) 

1413 

1414 if self.config.doWidenSaturationTrails: 

1415 self.log.info("Widening saturation trails.") 

1416 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1417 

1418 if self.config.doCameraSpecificMasking: 

1419 self.log.info("Masking regions for camera specific reasons.") 

1420 self.masking.run(ccdExposure) 

1421 

1422 if self.config.doBrighterFatter: 

1423 # We need to apply flats and darks before we can interpolate, and 

1424 # we need to interpolate before we do B-F, but we do B-F without 

1425 # the flats and darks applied so we can work in units of electrons 

1426 # or holes. This context manager applies and then removes the darks 

1427 # and flats. 

1428 # 

1429 # We also do not want to interpolate values here, so operate on 

1430 # temporary images so we can apply only the BF-correction and roll 

1431 # back the interpolation. 

1432 interpExp = ccdExposure.clone() 

1433 with self.flatContext(interpExp, flat, dark): 

1434 isrFunctions.interpolateFromMask( 

1435 maskedImage=interpExp.getMaskedImage(), 

1436 fwhm=self.config.fwhm, 

1437 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1438 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1439 ) 

1440 bfExp = interpExp.clone() 

1441 

1442 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1443 type(bfKernel), type(bfGains)) 

1444 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1445 self.config.brighterFatterMaxIter, 

1446 self.config.brighterFatterThreshold, 

1447 self.config.brighterFatterApplyGain, 

1448 bfGains) 

1449 if bfResults[1] == self.config.brighterFatterMaxIter: 

1450 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1451 bfResults[0]) 

1452 else: 

1453 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1454 bfResults[1]) 

1455 image = ccdExposure.getMaskedImage().getImage() 

1456 bfCorr = bfExp.getMaskedImage().getImage() 

1457 bfCorr -= interpExp.getMaskedImage().getImage() 

1458 image += bfCorr 

1459 

1460 # Applying the brighter-fatter correction applies a 

1461 # convolution to the science image. At the edges this 

1462 # convolution may not have sufficient valid pixels to 

1463 # produce a valid correction. Mark pixels within the size 

1464 # of the brighter-fatter kernel as EDGE to warn of this 

1465 # fact. 

1466 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1467 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1468 maskPlane="EDGE") 

1469 

1470 if self.config.brighterFatterMaskGrowSize > 0: 

1471 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1472 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1473 isrFunctions.growMasks(ccdExposure.getMask(), 

1474 radius=self.config.brighterFatterMaskGrowSize, 

1475 maskNameList=maskPlane, 

1476 maskValue=maskPlane) 

1477 

1478 self.debugView(ccdExposure, "doBrighterFatter") 

1479 

1480 if self.config.doDark: 

1481 self.log.info("Applying dark correction.") 

1482 self.darkCorrection(ccdExposure, dark) 

1483 self.debugView(ccdExposure, "doDark") 

1484 

1485 if self.config.doFringe and not self.config.fringeAfterFlat: 

1486 self.log.info("Applying fringe correction before flat.") 

1487 self.fringe.run(ccdExposure, **fringes.getDict()) 

1488 self.debugView(ccdExposure, "doFringe") 

1489 

1490 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1491 self.log.info("Checking strayLight correction.") 

1492 self.strayLight.run(ccdExposure, strayLightData) 

1493 self.debugView(ccdExposure, "doStrayLight") 

1494 

1495 if self.config.doFlat: 

1496 self.log.info("Applying flat correction.") 

1497 self.flatCorrection(ccdExposure, flat) 

1498 self.debugView(ccdExposure, "doFlat") 

1499 

1500 if self.config.doApplyGains: 

1501 self.log.info("Applying gain correction instead of flat.") 

1502 if self.config.usePtcGains: 

1503 self.log.info("Using gains from the Photon Transfer Curve.") 

1504 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1505 ptcGains=ptc.gain) 

1506 else: 

1507 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1508 

1509 if self.config.doFringe and self.config.fringeAfterFlat: 

1510 self.log.info("Applying fringe correction after flat.") 

1511 self.fringe.run(ccdExposure, **fringes.getDict()) 

1512 

1513 if self.config.doVignette: 

1514 if self.config.doMaskVignettePolygon: 

1515 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1516 else: 

1517 self.log.info("Constructing and attaching vignette polygon.") 

1518 self.vignettePolygon = self.vignette.run( 

1519 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1520 vignetteValue=self.config.vignetteValue, log=self.log) 

1521 

1522 if self.config.doAttachTransmissionCurve: 

1523 self.log.info("Adding transmission curves.") 

1524 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1525 filterTransmission=filterTransmission, 

1526 sensorTransmission=sensorTransmission, 

1527 atmosphereTransmission=atmosphereTransmission) 

1528 

1529 flattenedThumb = None 

1530 if self.config.qa.doThumbnailFlattened: 

1531 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1532 

1533 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1534 self.log.info("Performing illumination correction.") 

1535 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1536 illumMaskedImage, illumScale=self.config.illumScale, 

1537 trimToFit=self.config.doTrimToMatchCalib) 

1538 

1539 preInterpExp = None 

1540 if self.config.doSaveInterpPixels: 

1541 preInterpExp = ccdExposure.clone() 

1542 

1543 # Reset and interpolate bad pixels. 

1544 # 

1545 # Large contiguous bad regions (which should have the BAD mask 

1546 # bit set) should have their values set to the image median. 

1547 # This group should include defects and bad amplifiers. As the 

1548 # area covered by these defects are large, there's little 

1549 # reason to expect that interpolation would provide a more 

1550 # useful value. 

1551 # 

1552 # Smaller defects can be safely interpolated after the larger 

1553 # regions have had their pixel values reset. This ensures 

1554 # that the remaining defects adjacent to bad amplifiers (as an 

1555 # example) do not attempt to interpolate extreme values. 

1556 if self.config.doSetBadRegions: 

1557 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1558 if badPixelCount > 0: 

1559 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1560 

1561 if self.config.doInterpolate: 

1562 self.log.info("Interpolating masked pixels.") 

1563 isrFunctions.interpolateFromMask( 

1564 maskedImage=ccdExposure.getMaskedImage(), 

1565 fwhm=self.config.fwhm, 

1566 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1567 maskNameList=list(self.config.maskListToInterpolate) 

1568 ) 

1569 

1570 self.roughZeroPoint(ccdExposure) 

1571 

1572 # correct for amp offsets within the CCD 

1573 if self.config.doAmpOffset: 

1574 self.log.info("Correcting amp offsets.") 

1575 self.ampOffset.run(ccdExposure) 

1576 

1577 if self.config.doMeasureBackground: 

1578 self.log.info("Measuring background level.") 

1579 self.measureBackground(ccdExposure, self.config.qa) 

1580 

1581 if self.config.qa is not None and self.config.qa.saveStats is True: 

1582 for amp in ccd: 

1583 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1584 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1585 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1586 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1587 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1588 qaStats.getValue(afwMath.STDEVCLIP) 

1589 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1590 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1591 qaStats.getValue(afwMath.STDEVCLIP)) 

1592 

1593 # calculate additional statistics. 

1594 outputStatistics = None 

1595 if self.config.doCalculateStatistics: 

1596 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1597 ptc=ptc).results 

1598 

1599 self.debugView(ccdExposure, "postISRCCD") 

1600 

1601 return pipeBase.Struct( 

1602 exposure=ccdExposure, 

1603 ossThumb=ossThumb, 

1604 flattenedThumb=flattenedThumb, 

1605 

1606 preInterpExposure=preInterpExp, 

1607 outputExposure=ccdExposure, 

1608 outputOssThumbnail=ossThumb, 

1609 outputFlattenedThumbnail=flattenedThumb, 

1610 outputStatistics=outputStatistics, 

1611 ) 

1612 

1613 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1614 """Ensure that the data returned by Butler is a fully constructed exp. 

1615 

1616 ISR requires exposure-level image data for historical reasons, so if we 

1617 did not recieve that from Butler, construct it from what we have, 

1618 modifying the input in place. 

1619 

1620 Parameters 

1621 ---------- 

1622 inputExp : `lsst.afw.image` image-type. 

1623 The input data structure obtained from Butler. 

1624 Can be `lsst.afw.image.Exposure`, 

1625 `lsst.afw.image.DecoratedImageU`, 

1626 or `lsst.afw.image.ImageF` 

1627 camera : `lsst.afw.cameraGeom.camera`, optional 

1628 The camera associated with the image. Used to find the appropriate 

1629 detector if detector is not already set. 

1630 detectorNum : `int`, optional 

1631 The detector in the camera to attach, if the detector is not 

1632 already set. 

1633 

1634 Returns 

1635 ------- 

1636 inputExp : `lsst.afw.image.Exposure` 

1637 The re-constructed exposure, with appropriate detector parameters. 

1638 

1639 Raises 

1640 ------ 

1641 TypeError 

1642 Raised if the input data cannot be used to construct an exposure. 

1643 """ 

1644 if isinstance(inputExp, afwImage.DecoratedImageU): 

1645 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1646 elif isinstance(inputExp, afwImage.ImageF): 

1647 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1648 elif isinstance(inputExp, afwImage.MaskedImageF): 

1649 inputExp = afwImage.makeExposure(inputExp) 

1650 elif isinstance(inputExp, afwImage.Exposure): 

1651 pass 

1652 elif inputExp is None: 

1653 # Assume this will be caught by the setup if it is a problem. 

1654 return inputExp 

1655 else: 

1656 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1657 (type(inputExp), )) 

1658 

1659 if inputExp.getDetector() is None: 

1660 if camera is None or detectorNum is None: 

1661 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1662 'without a detector set.') 

1663 inputExp.setDetector(camera[detectorNum]) 

1664 

1665 return inputExp 

1666 

1667 @staticmethod 

1668 def extractCalibDate(calib): 

1669 """Extract common calibration metadata values that will be written to 

1670 output header. 

1671 

1672 Parameters 

1673 ---------- 

1674 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1675 Calibration to pull date information from. 

1676 

1677 Returns 

1678 ------- 

1679 dateString : `str` 

1680 Calibration creation date string to add to header. 

1681 """ 

1682 if hasattr(calib, "getMetadata"): 

1683 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1684 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1685 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1686 else: 

1687 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1688 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1689 else: 

1690 return "Unknown Unknown" 

1691 

1692 def convertIntToFloat(self, exposure): 

1693 """Convert exposure image from uint16 to float. 

1694 

1695 If the exposure does not need to be converted, the input is 

1696 immediately returned. For exposures that are converted to use 

1697 floating point pixels, the variance is set to unity and the 

1698 mask to zero. 

1699 

1700 Parameters 

1701 ---------- 

1702 exposure : `lsst.afw.image.Exposure` 

1703 The raw exposure to be converted. 

1704 

1705 Returns 

1706 ------- 

1707 newexposure : `lsst.afw.image.Exposure` 

1708 The input ``exposure``, converted to floating point pixels. 

1709 

1710 Raises 

1711 ------ 

1712 RuntimeError 

1713 Raised if the exposure type cannot be converted to float. 

1714 

1715 """ 

1716 if isinstance(exposure, afwImage.ExposureF): 

1717 # Nothing to be done 

1718 self.log.debug("Exposure already of type float.") 

1719 return exposure 

1720 if not hasattr(exposure, "convertF"): 

1721 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1722 

1723 newexposure = exposure.convertF() 

1724 newexposure.variance[:] = 1 

1725 newexposure.mask[:] = 0x0 

1726 

1727 return newexposure 

1728 

1729 def maskAmplifier(self, ccdExposure, amp, defects): 

1730 """Identify bad amplifiers, saturated and suspect pixels. 

1731 

1732 Parameters 

1733 ---------- 

1734 ccdExposure : `lsst.afw.image.Exposure` 

1735 Input exposure to be masked. 

1736 amp : `lsst.afw.cameraGeom.Amplifier` 

1737 Catalog of parameters defining the amplifier on this 

1738 exposure to mask. 

1739 defects : `lsst.ip.isr.Defects` 

1740 List of defects. Used to determine if the entire 

1741 amplifier is bad. 

1742 

1743 Returns 

1744 ------- 

1745 badAmp : `Bool` 

1746 If this is true, the entire amplifier area is covered by 

1747 defects and unusable. 

1748 

1749 """ 

1750 maskedImage = ccdExposure.getMaskedImage() 

1751 

1752 badAmp = False 

1753 

1754 # Check if entire amp region is defined as a defect 

1755 # NB: need to use amp.getBBox() for correct comparison with current 

1756 # defects definition. 

1757 if defects is not None: 

1758 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1759 

1760 # In the case of a bad amp, we will set mask to "BAD" 

1761 # (here use amp.getRawBBox() for correct association with pixels in 

1762 # current ccdExposure). 

1763 if badAmp: 

1764 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1765 afwImage.PARENT) 

1766 maskView = dataView.getMask() 

1767 maskView |= maskView.getPlaneBitMask("BAD") 

1768 del maskView 

1769 return badAmp 

1770 

1771 # Mask remaining defects after assembleCcd() to allow for defects that 

1772 # cross amplifier boundaries. Saturation and suspect pixels can be 

1773 # masked now, though. 

1774 limits = dict() 

1775 if self.config.doSaturation and not badAmp: 

1776 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1777 if self.config.doSuspect and not badAmp: 

1778 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1779 if math.isfinite(self.config.saturation): 

1780 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1781 

1782 for maskName, maskThreshold in limits.items(): 

1783 if not math.isnan(maskThreshold): 

1784 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1785 isrFunctions.makeThresholdMask( 

1786 maskedImage=dataView, 

1787 threshold=maskThreshold, 

1788 growFootprints=0, 

1789 maskName=maskName 

1790 ) 

1791 

1792 # Determine if we've fully masked this amplifier with SUSPECT and 

1793 # SAT pixels. 

1794 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1795 afwImage.PARENT) 

1796 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1797 self.config.suspectMaskName]) 

1798 if numpy.all(maskView.getArray() & maskVal > 0): 

1799 badAmp = True 

1800 maskView |= maskView.getPlaneBitMask("BAD") 

1801 

1802 return badAmp 

1803 

1804 def overscanCorrection(self, ccdExposure, amp): 

1805 """Apply overscan correction in place. 

1806 

1807 This method does initial pixel rejection of the overscan 

1808 region. The overscan can also be optionally segmented to 

1809 allow for discontinuous overscan responses to be fit 

1810 separately. The actual overscan subtraction is performed by 

1811 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

1812 after the amplifier is preprocessed. 

1813 

1814 Parameters 

1815 ---------- 

1816 ccdExposure : `lsst.afw.image.Exposure` 

1817 Exposure to have overscan correction performed. 

1818 amp : `lsst.afw.cameraGeom.Amplifer` 

1819 The amplifier to consider while correcting the overscan. 

1820 

1821 Returns 

1822 ------- 

1823 overscanResults : `lsst.pipe.base.Struct` 

1824 Result struct with components: 

1825 

1826 ``imageFit`` 

1827 Value or fit subtracted from the amplifier image data. 

1828 (scalar or `lsst.afw.image.Image`) 

1829 ``overscanFit`` 

1830 Value or fit subtracted from the overscan image data. 

1831 (scalar or `lsst.afw.image.Image`) 

1832 ``overscanImage`` 

1833 Image of the overscan region with the overscan 

1834 correction applied. This quantity is used to estimate 

1835 the amplifier read noise empirically. 

1836 (`lsst.afw.image.Image`) 

1837 ``edgeMask`` 

1838 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

1839 ``overscanMean`` 

1840 Median overscan fit value. (`float`) 

1841 ``overscanSigma`` 

1842 Clipped standard deviation of the overscan after 

1843 correction. (`float`) 

1844 

1845 Raises 

1846 ------ 

1847 RuntimeError 

1848 Raised if the ``amp`` does not contain raw pixel information. 

1849 

1850 See Also 

1851 -------- 

1852 lsst.ip.isr.overscan.OverscanTask 

1853 

1854 """ 

1855 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1856 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1857 return None 

1858 

1859 # Perform overscan correction on subregions. 

1860 overscanResults = self.overscan.run(ccdExposure, amp) 

1861 

1862 metadata = ccdExposure.getMetadata() 

1863 ampNum = amp.getName() 

1864 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanMean 

1865 metadata[f"ISR_OSCAN_SIGMA{ampNum}"] = overscanResults.overscanSigma 

1866 

1867 return overscanResults 

1868 

1869 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

1870 """Set the variance plane using the gain and read noise 

1871 

1872 The read noise is calculated from the ``overscanImage`` if the 

1873 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1874 the value from the amplifier data is used. 

1875 

1876 Parameters 

1877 ---------- 

1878 ampExposure : `lsst.afw.image.Exposure` 

1879 Exposure to process. 

1880 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

1881 Amplifier detector data. 

1882 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

1883 Image of overscan, required only for empirical read noise. 

1884 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1885 PTC dataset containing the gains and read noise. 

1886 

1887 Raises 

1888 ------ 

1889 RuntimeError 

1890 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

1891 are ``True``, but ptcDataset is not provided. 

1892 

1893 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

1894 ``overscanImage`` is ``None``. 

1895 

1896 See also 

1897 -------- 

1898 lsst.ip.isr.isrFunctions.updateVariance 

1899 """ 

1900 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

1901 if self.config.usePtcGains: 

1902 if ptcDataset is None: 

1903 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

1904 else: 

1905 gain = ptcDataset.gain[amp.getName()] 

1906 self.log.info("Using gain from Photon Transfer Curve.") 

1907 else: 

1908 gain = amp.getGain() 

1909 

1910 if math.isnan(gain): 

1911 gain = 1.0 

1912 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

1913 elif gain <= 0: 

1914 patchedGain = 1.0 

1915 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

1916 amp.getName(), gain, patchedGain) 

1917 gain = patchedGain 

1918 

1919 if self.config.doEmpiricalReadNoise and overscanImage is None: 

1920 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(), 

1921 [self.config.saturatedMaskName, 

1922 self.config.suspectMaskName, 

1923 "BAD", "NO_DATA"]) 

1924 allPixels = ampExposure.getWidth() * ampExposure.getHeight() 

1925 if allPixels == badPixels: 

1926 # If the image is bad, do not raise. 

1927 self.log.info("Skipping empirical read noise for amp %s. No good pixels.", 

1928 amp.getName()) 

1929 else: 

1930 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

1931 

1932 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

1933 stats = afwMath.StatisticsControl() 

1934 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

1935 readNoise = afwMath.makeStatistics(overscanImage.getImage(), 

1936 afwMath.STDEVCLIP, stats).getValue() 

1937 self.log.info("Calculated empirical read noise for amp %s: %f.", 

1938 amp.getName(), readNoise) 

1939 elif self.config.usePtcReadNoise: 

1940 if ptcDataset is None: 

1941 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

1942 else: 

1943 readNoise = ptcDataset.noise[amp.getName()] 

1944 self.log.info("Using read noise from Photon Transfer Curve.") 

1945 else: 

1946 readNoise = amp.getReadNoise() 

1947 

1948 metadata = ampExposure.getMetadata() 

1949 metadata[f'LSST GAIN {amp.getName()}'] = gain 

1950 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise 

1951 

1952 isrFunctions.updateVariance( 

1953 maskedImage=ampExposure.getMaskedImage(), 

1954 gain=gain, 

1955 readNoise=readNoise, 

1956 ) 

1957 

1958 def maskNegativeVariance(self, exposure): 

1959 """Identify and mask pixels with negative variance values. 

1960 

1961 Parameters 

1962 ---------- 

1963 exposure : `lsst.afw.image.Exposure` 

1964 Exposure to process. 

1965 

1966 See Also 

1967 -------- 

1968 lsst.ip.isr.isrFunctions.updateVariance 

1969 """ 

1970 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

1971 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

1972 exposure.mask.array[bad] |= maskPlane 

1973 

1974 def darkCorrection(self, exposure, darkExposure, invert=False): 

1975 """Apply dark correction in place. 

1976 

1977 Parameters 

1978 ---------- 

1979 exposure : `lsst.afw.image.Exposure` 

1980 Exposure to process. 

1981 darkExposure : `lsst.afw.image.Exposure` 

1982 Dark exposure of the same size as ``exposure``. 

1983 invert : `Bool`, optional 

1984 If True, re-add the dark to an already corrected image. 

1985 

1986 Raises 

1987 ------ 

1988 RuntimeError 

1989 Raised if either ``exposure`` or ``darkExposure`` do not 

1990 have their dark time defined. 

1991 

1992 See Also 

1993 -------- 

1994 lsst.ip.isr.isrFunctions.darkCorrection 

1995 """ 

1996 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

1997 if math.isnan(expScale): 

1998 raise RuntimeError("Exposure darktime is NAN.") 

1999 if darkExposure.getInfo().getVisitInfo() is not None \ 

2000 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2001 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2002 else: 

2003 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2004 # so getDarkTime() does not exist. 

2005 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2006 darkScale = 1.0 

2007 

2008 isrFunctions.darkCorrection( 

2009 maskedImage=exposure.getMaskedImage(), 

2010 darkMaskedImage=darkExposure.getMaskedImage(), 

2011 expScale=expScale, 

2012 darkScale=darkScale, 

2013 invert=invert, 

2014 trimToFit=self.config.doTrimToMatchCalib 

2015 ) 

2016 

2017 def doLinearize(self, detector): 

2018 """Check if linearization is needed for the detector cameraGeom. 

2019 

2020 Checks config.doLinearize and the linearity type of the first 

2021 amplifier. 

2022 

2023 Parameters 

2024 ---------- 

2025 detector : `lsst.afw.cameraGeom.Detector` 

2026 Detector to get linearity type from. 

2027 

2028 Returns 

2029 ------- 

2030 doLinearize : `Bool` 

2031 If True, linearization should be performed. 

2032 """ 

2033 return self.config.doLinearize and \ 

2034 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2035 

2036 def flatCorrection(self, exposure, flatExposure, invert=False): 

2037 """Apply flat correction in place. 

2038 

2039 Parameters 

2040 ---------- 

2041 exposure : `lsst.afw.image.Exposure` 

2042 Exposure to process. 

2043 flatExposure : `lsst.afw.image.Exposure` 

2044 Flat exposure of the same size as ``exposure``. 

2045 invert : `Bool`, optional 

2046 If True, unflatten an already flattened image. 

2047 

2048 See Also 

2049 -------- 

2050 lsst.ip.isr.isrFunctions.flatCorrection 

2051 """ 

2052 isrFunctions.flatCorrection( 

2053 maskedImage=exposure.getMaskedImage(), 

2054 flatMaskedImage=flatExposure.getMaskedImage(), 

2055 scalingType=self.config.flatScalingType, 

2056 userScale=self.config.flatUserScale, 

2057 invert=invert, 

2058 trimToFit=self.config.doTrimToMatchCalib 

2059 ) 

2060 

2061 def saturationDetection(self, exposure, amp): 

2062 """Detect and mask saturated pixels in config.saturatedMaskName. 

2063 

2064 Parameters 

2065 ---------- 

2066 exposure : `lsst.afw.image.Exposure` 

2067 Exposure to process. Only the amplifier DataSec is processed. 

2068 amp : `lsst.afw.cameraGeom.Amplifier` 

2069 Amplifier detector data. 

2070 

2071 See Also 

2072 -------- 

2073 lsst.ip.isr.isrFunctions.makeThresholdMask 

2074 """ 

2075 if not math.isnan(amp.getSaturation()): 

2076 maskedImage = exposure.getMaskedImage() 

2077 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2078 isrFunctions.makeThresholdMask( 

2079 maskedImage=dataView, 

2080 threshold=amp.getSaturation(), 

2081 growFootprints=0, 

2082 maskName=self.config.saturatedMaskName, 

2083 ) 

2084 

2085 def saturationInterpolation(self, exposure): 

2086 """Interpolate over saturated pixels, in place. 

2087 

2088 This method should be called after `saturationDetection`, to 

2089 ensure that the saturated pixels have been identified in the 

2090 SAT mask. It should also be called after `assembleCcd`, since 

2091 saturated regions may cross amplifier boundaries. 

2092 

2093 Parameters 

2094 ---------- 

2095 exposure : `lsst.afw.image.Exposure` 

2096 Exposure to process. 

2097 

2098 See Also 

2099 -------- 

2100 lsst.ip.isr.isrTask.saturationDetection 

2101 lsst.ip.isr.isrFunctions.interpolateFromMask 

2102 """ 

2103 isrFunctions.interpolateFromMask( 

2104 maskedImage=exposure.getMaskedImage(), 

2105 fwhm=self.config.fwhm, 

2106 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2107 maskNameList=list(self.config.saturatedMaskName), 

2108 ) 

2109 

2110 def suspectDetection(self, exposure, amp): 

2111 """Detect and mask suspect pixels in config.suspectMaskName. 

2112 

2113 Parameters 

2114 ---------- 

2115 exposure : `lsst.afw.image.Exposure` 

2116 Exposure to process. Only the amplifier DataSec is processed. 

2117 amp : `lsst.afw.cameraGeom.Amplifier` 

2118 Amplifier detector data. 

2119 

2120 See Also 

2121 -------- 

2122 lsst.ip.isr.isrFunctions.makeThresholdMask 

2123 

2124 Notes 

2125 ----- 

2126 Suspect pixels are pixels whose value is greater than 

2127 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2128 affected by unknown systematics; for example if non-linearity 

2129 corrections above a certain level are unstable then that would be a 

2130 useful value for suspectLevel. A value of `nan` indicates that no such 

2131 level exists and no pixels are to be masked as suspicious. 

2132 """ 

2133 suspectLevel = amp.getSuspectLevel() 

2134 if math.isnan(suspectLevel): 

2135 return 

2136 

2137 maskedImage = exposure.getMaskedImage() 

2138 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2139 isrFunctions.makeThresholdMask( 

2140 maskedImage=dataView, 

2141 threshold=suspectLevel, 

2142 growFootprints=0, 

2143 maskName=self.config.suspectMaskName, 

2144 ) 

2145 

2146 def maskDefect(self, exposure, defectBaseList): 

2147 """Mask defects using mask plane "BAD", in place. 

2148 

2149 Parameters 

2150 ---------- 

2151 exposure : `lsst.afw.image.Exposure` 

2152 Exposure to process. 

2153 defectBaseList : defect-type 

2154 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2155 or `list` of `lsst.afw.image.DefectBase`. 

2156 

2157 Notes 

2158 ----- 

2159 Call this after CCD assembly, since defects may cross amplifier 

2160 boundaries. 

2161 """ 

2162 maskedImage = exposure.getMaskedImage() 

2163 if not isinstance(defectBaseList, Defects): 

2164 # Promotes DefectBase to Defect 

2165 defectList = Defects(defectBaseList) 

2166 else: 

2167 defectList = defectBaseList 

2168 defectList.maskPixels(maskedImage, maskName="BAD") 

2169 

2170 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2171 """Mask edge pixels with applicable mask plane. 

2172 

2173 Parameters 

2174 ---------- 

2175 exposure : `lsst.afw.image.Exposure` 

2176 Exposure to process. 

2177 numEdgePixels : `int`, optional 

2178 Number of edge pixels to mask. 

2179 maskPlane : `str`, optional 

2180 Mask plane name to use. 

2181 level : `str`, optional 

2182 Level at which to mask edges. 

2183 """ 

2184 maskedImage = exposure.getMaskedImage() 

2185 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2186 

2187 if numEdgePixels > 0: 

2188 if level == 'DETECTOR': 

2189 boxes = [maskedImage.getBBox()] 

2190 elif level == 'AMP': 

2191 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2192 

2193 for box in boxes: 

2194 # This makes a bbox numEdgeSuspect pixels smaller than the 

2195 # image on each side 

2196 subImage = maskedImage[box] 

2197 box.grow(-numEdgePixels) 

2198 # Mask pixels outside box 

2199 SourceDetectionTask.setEdgeBits( 

2200 subImage, 

2201 box, 

2202 maskBitMask) 

2203 

2204 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2205 """Mask and interpolate defects using mask plane "BAD", in place. 

2206 

2207 Parameters 

2208 ---------- 

2209 exposure : `lsst.afw.image.Exposure` 

2210 Exposure to process. 

2211 defectBaseList : defects-like 

2212 List of defects to mask and interpolate. Can be 

2213 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2214 

2215 See Also 

2216 -------- 

2217 lsst.ip.isr.isrTask.maskDefect 

2218 """ 

2219 self.maskDefect(exposure, defectBaseList) 

2220 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2221 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2222 isrFunctions.interpolateFromMask( 

2223 maskedImage=exposure.getMaskedImage(), 

2224 fwhm=self.config.fwhm, 

2225 growSaturatedFootprints=0, 

2226 maskNameList=["BAD"], 

2227 ) 

2228 

2229 def maskNan(self, exposure): 

2230 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2231 

2232 Parameters 

2233 ---------- 

2234 exposure : `lsst.afw.image.Exposure` 

2235 Exposure to process. 

2236 

2237 Notes 

2238 ----- 

2239 We mask over all non-finite values (NaN, inf), including those 

2240 that are masked with other bits (because those may or may not be 

2241 interpolated over later, and we want to remove all NaN/infs). 

2242 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2243 preserve the historical name. 

2244 """ 

2245 maskedImage = exposure.getMaskedImage() 

2246 

2247 # Find and mask NaNs 

2248 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2249 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2250 numNans = maskNans(maskedImage, maskVal) 

2251 self.metadata["NUMNANS"] = numNans 

2252 if numNans > 0: 

2253 self.log.warning("There were %d unmasked NaNs.", numNans) 

2254 

2255 def maskAndInterpolateNan(self, exposure): 

2256 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2257 in place. 

2258 

2259 Parameters 

2260 ---------- 

2261 exposure : `lsst.afw.image.Exposure` 

2262 Exposure to process. 

2263 

2264 See Also 

2265 -------- 

2266 lsst.ip.isr.isrTask.maskNan 

2267 """ 

2268 self.maskNan(exposure) 

2269 isrFunctions.interpolateFromMask( 

2270 maskedImage=exposure.getMaskedImage(), 

2271 fwhm=self.config.fwhm, 

2272 growSaturatedFootprints=0, 

2273 maskNameList=["UNMASKEDNAN"], 

2274 ) 

2275 

2276 def measureBackground(self, exposure, IsrQaConfig=None): 

2277 """Measure the image background in subgrids, for quality control. 

2278 

2279 Parameters 

2280 ---------- 

2281 exposure : `lsst.afw.image.Exposure` 

2282 Exposure to process. 

2283 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2284 Configuration object containing parameters on which background 

2285 statistics and subgrids to use. 

2286 """ 

2287 if IsrQaConfig is not None: 

2288 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2289 IsrQaConfig.flatness.nIter) 

2290 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2291 statsControl.setAndMask(maskVal) 

2292 maskedImage = exposure.getMaskedImage() 

2293 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2294 skyLevel = stats.getValue(afwMath.MEDIAN) 

2295 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2296 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2297 metadata = exposure.getMetadata() 

2298 metadata["SKYLEVEL"] = skyLevel 

2299 metadata["SKYSIGMA"] = skySigma 

2300 

2301 # calcluating flatlevel over the subgrids 

2302 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2303 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2304 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2305 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2306 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2307 skyLevels = numpy.zeros((nX, nY)) 

2308 

2309 for j in range(nY): 

2310 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2311 for i in range(nX): 

2312 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2313 

2314 xLLC = xc - meshXHalf 

2315 yLLC = yc - meshYHalf 

2316 xURC = xc + meshXHalf - 1 

2317 yURC = yc + meshYHalf - 1 

2318 

2319 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2320 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2321 

2322 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2323 

2324 good = numpy.where(numpy.isfinite(skyLevels)) 

2325 skyMedian = numpy.median(skyLevels[good]) 

2326 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2327 flatness_rms = numpy.std(flatness) 

2328 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2329 

2330 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2331 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2332 nX, nY, flatness_pp, flatness_rms) 

2333 

2334 metadata["FLATNESS_PP"] = float(flatness_pp) 

2335 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2336 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2337 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2338 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2339 

2340 def roughZeroPoint(self, exposure): 

2341 """Set an approximate magnitude zero point for the exposure. 

2342 

2343 Parameters 

2344 ---------- 

2345 exposure : `lsst.afw.image.Exposure` 

2346 Exposure to process. 

2347 """ 

2348 filterLabel = exposure.getFilter() 

2349 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2350 

2351 if physicalFilter in self.config.fluxMag0T1: 

2352 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2353 else: 

2354 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2355 fluxMag0 = self.config.defaultFluxMag0T1 

2356 

2357 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2358 if not expTime > 0: # handle NaN as well as <= 0 

2359 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2360 return 

2361 

2362 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2363 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2364 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2365 

2366 @contextmanager 

2367 def flatContext(self, exp, flat, dark=None): 

2368 """Context manager that applies and removes flats and darks, 

2369 if the task is configured to apply them. 

2370 

2371 Parameters 

2372 ---------- 

2373 exp : `lsst.afw.image.Exposure` 

2374 Exposure to process. 

2375 flat : `lsst.afw.image.Exposure` 

2376 Flat exposure the same size as ``exp``. 

2377 dark : `lsst.afw.image.Exposure`, optional 

2378 Dark exposure the same size as ``exp``. 

2379 

2380 Yields 

2381 ------ 

2382 exp : `lsst.afw.image.Exposure` 

2383 The flat and dark corrected exposure. 

2384 """ 

2385 if self.config.doDark and dark is not None: 

2386 self.darkCorrection(exp, dark) 

2387 if self.config.doFlat: 

2388 self.flatCorrection(exp, flat) 

2389 try: 

2390 yield exp 

2391 finally: 

2392 if self.config.doFlat: 

2393 self.flatCorrection(exp, flat, invert=True) 

2394 if self.config.doDark and dark is not None: 

2395 self.darkCorrection(exp, dark, invert=True) 

2396 

2397 def debugView(self, exposure, stepname): 

2398 """Utility function to examine ISR exposure at different stages. 

2399 

2400 Parameters 

2401 ---------- 

2402 exposure : `lsst.afw.image.Exposure` 

2403 Exposure to view. 

2404 stepname : `str` 

2405 State of processing to view. 

2406 """ 

2407 frame = getDebugFrame(self._display, stepname) 

2408 if frame: 

2409 display = getDisplay(frame) 

2410 display.scale('asinh', 'zscale') 

2411 display.mtv(exposure) 

2412 prompt = "Press Enter to continue [c]... " 

2413 while True: 

2414 ans = input(prompt).lower() 

2415 if ans in ("", "c",): 

2416 break 

2417 

2418 

2419class FakeAmp(object): 

2420 """A Detector-like object that supports returning gain and saturation level 

2421 

2422 This is used when the input exposure does not have a detector. 

2423 

2424 Parameters 

2425 ---------- 

2426 exposure : `lsst.afw.image.Exposure` 

2427 Exposure to generate a fake amplifier for. 

2428 config : `lsst.ip.isr.isrTaskConfig` 

2429 Configuration to apply to the fake amplifier. 

2430 """ 

2431 

2432 def __init__(self, exposure, config): 

2433 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2434 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2435 self._gain = config.gain 

2436 self._readNoise = config.readNoise 

2437 self._saturation = config.saturation 

2438 

2439 def getBBox(self): 

2440 return self._bbox 

2441 

2442 def getRawBBox(self): 

2443 return self._bbox 

2444 

2445 def getRawHorizontalOverscanBBox(self): 

2446 return self._RawHorizontalOverscanBBox 

2447 

2448 def getGain(self): 

2449 return self._gain 

2450 

2451 def getReadNoise(self): 

2452 return self._readNoise 

2453 

2454 def getSaturation(self): 

2455 return self._saturation 

2456 

2457 def getSuspectLevel(self): 

2458 return float("NaN")