Coverage for python/lsst/ip/isr/isrTask.py: 16%

857 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-05 10:32 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrTask", "IsrTaskConfig"] 

23 

24import math 

25import numpy 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from contextlib import contextmanager 

35from lsstDebug import getDebugFrame 

36 

37from lsst.afw.cameraGeom import NullLinearityType 

38from lsst.afw.display import getDisplay 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from lsst.daf.butler import DimensionGraph 

59 

60 

61def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

62 """Lookup function to identify crosstalkSource entries. 

63 

64 This should return an empty list under most circumstances. Only 

65 when inter-chip crosstalk has been identified should this be 

66 populated. 

67 

68 Parameters 

69 ---------- 

70 datasetType : `str` 

71 Dataset to lookup. 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry to query. 

74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

75 Data id to transform to identify crosstalkSources. The 

76 ``detector`` entry will be stripped. 

77 collections : `lsst.daf.butler.CollectionSearch` 

78 Collections to search through. 

79 

80 Returns 

81 ------- 

82 results : `list` [`lsst.daf.butler.DatasetRef`] 

83 List of datasets that match the query that will be used as 

84 crosstalkSources. 

85 """ 

86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

88 findFirst=True)) 

89 # In some contexts, calling `.expanded()` to expand all data IDs in the 

90 # query results can be a lot faster because it vectorizes lookups. But in 

91 # this case, expandDataId shouldn't need to hit the database at all in the 

92 # steady state, because only the detector record is unknown and those are 

93 # cached in the registry. 

94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

95 

96 

97class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

98 dimensions={"instrument", "exposure", "detector"}, 

99 defaultTemplates={}): 

100 ccdExposure = cT.Input( 

101 name="raw", 

102 doc="Input exposure to process.", 

103 storageClass="Exposure", 

104 dimensions=["instrument", "exposure", "detector"], 

105 ) 

106 camera = cT.PrerequisiteInput( 

107 name="camera", 

108 storageClass="Camera", 

109 doc="Input camera to construct complete exposures.", 

110 dimensions=["instrument"], 

111 isCalibration=True, 

112 ) 

113 

114 crosstalk = cT.PrerequisiteInput( 

115 name="crosstalk", 

116 doc="Input crosstalk object", 

117 storageClass="CrosstalkCalib", 

118 dimensions=["instrument", "detector"], 

119 isCalibration=True, 

120 minimum=0, # can fall back to cameraGeom 

121 ) 

122 crosstalkSources = cT.PrerequisiteInput( 

123 name="isrOverscanCorrected", 

124 doc="Overscan corrected input images.", 

125 storageClass="Exposure", 

126 dimensions=["instrument", "exposure", "detector"], 

127 deferLoad=True, 

128 multiple=True, 

129 lookupFunction=crosstalkSourceLookup, 

130 minimum=0, # not needed for all instruments, no config to control this 

131 ) 

132 bias = cT.PrerequisiteInput( 

133 name="bias", 

134 doc="Input bias calibration.", 

135 storageClass="ExposureF", 

136 dimensions=["instrument", "detector"], 

137 isCalibration=True, 

138 ) 

139 dark = cT.PrerequisiteInput( 

140 name='dark', 

141 doc="Input dark calibration.", 

142 storageClass="ExposureF", 

143 dimensions=["instrument", "detector"], 

144 isCalibration=True, 

145 ) 

146 flat = cT.PrerequisiteInput( 

147 name="flat", 

148 doc="Input flat calibration.", 

149 storageClass="ExposureF", 

150 dimensions=["instrument", "physical_filter", "detector"], 

151 isCalibration=True, 

152 ) 

153 ptc = cT.PrerequisiteInput( 

154 name="ptc", 

155 doc="Input Photon Transfer Curve dataset", 

156 storageClass="PhotonTransferCurveDataset", 

157 dimensions=["instrument", "detector"], 

158 isCalibration=True, 

159 ) 

160 fringes = cT.PrerequisiteInput( 

161 name="fringe", 

162 doc="Input fringe calibration.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "physical_filter", "detector"], 

165 isCalibration=True, 

166 minimum=0, # only needed for some bands, even when enabled 

167 ) 

168 strayLightData = cT.PrerequisiteInput( 

169 name='yBackground', 

170 doc="Input stray light calibration.", 

171 storageClass="StrayLightData", 

172 dimensions=["instrument", "physical_filter", "detector"], 

173 deferLoad=True, 

174 isCalibration=True, 

175 minimum=0, # only needed for some bands, even when enabled 

176 ) 

177 bfKernel = cT.PrerequisiteInput( 

178 name='bfKernel', 

179 doc="Input brighter-fatter kernel.", 

180 storageClass="NumpyArray", 

181 dimensions=["instrument"], 

182 isCalibration=True, 

183 minimum=0, # can use either bfKernel or newBFKernel 

184 ) 

185 newBFKernel = cT.PrerequisiteInput( 

186 name='brighterFatterKernel', 

187 doc="Newer complete kernel + gain solutions.", 

188 storageClass="BrighterFatterKernel", 

189 dimensions=["instrument", "detector"], 

190 isCalibration=True, 

191 minimum=0, # can use either bfKernel or newBFKernel 

192 ) 

193 defects = cT.PrerequisiteInput( 

194 name='defects', 

195 doc="Input defect tables.", 

196 storageClass="Defects", 

197 dimensions=["instrument", "detector"], 

198 isCalibration=True, 

199 ) 

200 linearizer = cT.PrerequisiteInput( 

201 name='linearizer', 

202 storageClass="Linearizer", 

203 doc="Linearity correction calibration.", 

204 dimensions=["instrument", "detector"], 

205 isCalibration=True, 

206 minimum=0, # can fall back to cameraGeom 

207 ) 

208 opticsTransmission = cT.PrerequisiteInput( 

209 name="transmission_optics", 

210 storageClass="TransmissionCurve", 

211 doc="Transmission curve due to the optics.", 

212 dimensions=["instrument"], 

213 isCalibration=True, 

214 ) 

215 filterTransmission = cT.PrerequisiteInput( 

216 name="transmission_filter", 

217 storageClass="TransmissionCurve", 

218 doc="Transmission curve due to the filter.", 

219 dimensions=["instrument", "physical_filter"], 

220 isCalibration=True, 

221 ) 

222 sensorTransmission = cT.PrerequisiteInput( 

223 name="transmission_sensor", 

224 storageClass="TransmissionCurve", 

225 doc="Transmission curve due to the sensor.", 

226 dimensions=["instrument", "detector"], 

227 isCalibration=True, 

228 ) 

229 atmosphereTransmission = cT.PrerequisiteInput( 

230 name="transmission_atmosphere", 

231 storageClass="TransmissionCurve", 

232 doc="Transmission curve due to the atmosphere.", 

233 dimensions=["instrument"], 

234 isCalibration=True, 

235 ) 

236 illumMaskedImage = cT.PrerequisiteInput( 

237 name="illum", 

238 doc="Input illumination correction.", 

239 storageClass="MaskedImageF", 

240 dimensions=["instrument", "physical_filter", "detector"], 

241 isCalibration=True, 

242 ) 

243 deferredChargeCalib = cT.PrerequisiteInput( 

244 name="cpCtiCalib", 

245 doc="Deferred charge/CTI correction dataset.", 

246 storageClass="IsrCalib", 

247 dimensions=["instrument", "detector"], 

248 isCalibration=True, 

249 ) 

250 

251 outputExposure = cT.Output( 

252 name='postISRCCD', 

253 doc="Output ISR processed exposure.", 

254 storageClass="Exposure", 

255 dimensions=["instrument", "exposure", "detector"], 

256 ) 

257 preInterpExposure = cT.Output( 

258 name='preInterpISRCCD', 

259 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

260 storageClass="ExposureF", 

261 dimensions=["instrument", "exposure", "detector"], 

262 ) 

263 outputOssThumbnail = cT.Output( 

264 name="OssThumb", 

265 doc="Output Overscan-subtracted thumbnail image.", 

266 storageClass="Thumbnail", 

267 dimensions=["instrument", "exposure", "detector"], 

268 ) 

269 outputFlattenedThumbnail = cT.Output( 

270 name="FlattenedThumb", 

271 doc="Output flat-corrected thumbnail image.", 

272 storageClass="Thumbnail", 

273 dimensions=["instrument", "exposure", "detector"], 

274 ) 

275 outputStatistics = cT.Output( 

276 name="isrStatistics", 

277 doc="Output of additional statistics table.", 

278 storageClass="StructuredDataDict", 

279 dimensions=["instrument", "exposure", "detector"], 

280 ) 

281 

282 def __init__(self, *, config=None): 

283 super().__init__(config=config) 

284 

285 if config.doBias is not True: 

286 self.prerequisiteInputs.remove("bias") 

287 if config.doLinearize is not True: 

288 self.prerequisiteInputs.remove("linearizer") 

289 if config.doCrosstalk is not True: 

290 self.prerequisiteInputs.remove("crosstalkSources") 

291 self.prerequisiteInputs.remove("crosstalk") 

292 if config.doBrighterFatter is not True: 

293 self.prerequisiteInputs.remove("bfKernel") 

294 self.prerequisiteInputs.remove("newBFKernel") 

295 if config.doDefect is not True: 

296 self.prerequisiteInputs.remove("defects") 

297 if config.doDark is not True: 

298 self.prerequisiteInputs.remove("dark") 

299 if config.doFlat is not True: 

300 self.prerequisiteInputs.remove("flat") 

301 if config.doFringe is not True: 

302 self.prerequisiteInputs.remove("fringes") 

303 if config.doStrayLight is not True: 

304 self.prerequisiteInputs.remove("strayLightData") 

305 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

306 self.prerequisiteInputs.remove("ptc") 

307 if config.doAttachTransmissionCurve is not True: 

308 self.prerequisiteInputs.remove("opticsTransmission") 

309 self.prerequisiteInputs.remove("filterTransmission") 

310 self.prerequisiteInputs.remove("sensorTransmission") 

311 self.prerequisiteInputs.remove("atmosphereTransmission") 

312 else: 

313 if config.doUseOpticsTransmission is not True: 

314 self.prerequisiteInputs.remove("opticsTransmission") 

315 if config.doUseFilterTransmission is not True: 

316 self.prerequisiteInputs.remove("filterTransmission") 

317 if config.doUseSensorTransmission is not True: 

318 self.prerequisiteInputs.remove("sensorTransmission") 

319 if config.doUseAtmosphereTransmission is not True: 

320 self.prerequisiteInputs.remove("atmosphereTransmission") 

321 if config.doIlluminationCorrection is not True: 

322 self.prerequisiteInputs.remove("illumMaskedImage") 

323 if config.doDeferredCharge is not True: 

324 self.prerequisiteInputs.remove("deferredChargeCalib") 

325 

326 if config.doWrite is not True: 

327 self.outputs.remove("outputExposure") 

328 self.outputs.remove("preInterpExposure") 

329 self.outputs.remove("outputFlattenedThumbnail") 

330 self.outputs.remove("outputOssThumbnail") 

331 self.outputs.remove("outputStatistics") 

332 

333 if config.doSaveInterpPixels is not True: 

334 self.outputs.remove("preInterpExposure") 

335 if config.qa.doThumbnailOss is not True: 

336 self.outputs.remove("outputOssThumbnail") 

337 if config.qa.doThumbnailFlattened is not True: 

338 self.outputs.remove("outputFlattenedThumbnail") 

339 if config.doCalculateStatistics is not True: 

340 self.outputs.remove("outputStatistics") 

341 

342 

343class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

344 pipelineConnections=IsrTaskConnections): 

345 """Configuration parameters for IsrTask. 

346 

347 Items are grouped in the order in which they are executed by the task. 

348 """ 

349 datasetType = pexConfig.Field( 

350 dtype=str, 

351 doc="Dataset type for input data; users will typically leave this alone, " 

352 "but camera-specific ISR tasks will override it", 

353 default="raw", 

354 ) 

355 

356 fallbackFilterName = pexConfig.Field( 

357 dtype=str, 

358 doc="Fallback default filter name for calibrations.", 

359 optional=True 

360 ) 

361 useFallbackDate = pexConfig.Field( 

362 dtype=bool, 

363 doc="Pass observation date when using fallback filter.", 

364 default=False, 

365 ) 

366 expectWcs = pexConfig.Field( 

367 dtype=bool, 

368 default=True, 

369 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

370 ) 

371 fwhm = pexConfig.Field( 

372 dtype=float, 

373 doc="FWHM of PSF in arcseconds.", 

374 default=1.0, 

375 ) 

376 qa = pexConfig.ConfigField( 

377 dtype=isrQa.IsrQaConfig, 

378 doc="QA related configuration options.", 

379 ) 

380 doHeaderProvenance = pexConfig.Field( 

381 dtype=bool, 

382 default=True, 

383 doc="Write calibration identifiers into output exposure header?", 

384 ) 

385 

386 # Image conversion configuration 

387 doConvertIntToFloat = pexConfig.Field( 

388 dtype=bool, 

389 doc="Convert integer raw images to floating point values?", 

390 default=True, 

391 ) 

392 

393 # Saturated pixel handling. 

394 doSaturation = pexConfig.Field( 

395 dtype=bool, 

396 doc="Mask saturated pixels? NB: this is totally independent of the" 

397 " interpolation option - this is ONLY setting the bits in the mask." 

398 " To have them interpolated make sure doSaturationInterpolation=True", 

399 default=True, 

400 ) 

401 saturatedMaskName = pexConfig.Field( 

402 dtype=str, 

403 doc="Name of mask plane to use in saturation detection and interpolation", 

404 default="SAT", 

405 ) 

406 saturation = pexConfig.Field( 

407 dtype=float, 

408 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

409 default=float("NaN"), 

410 ) 

411 growSaturationFootprintSize = pexConfig.Field( 

412 dtype=int, 

413 doc="Number of pixels by which to grow the saturation footprints", 

414 default=1, 

415 ) 

416 

417 # Suspect pixel handling. 

418 doSuspect = pexConfig.Field( 

419 dtype=bool, 

420 doc="Mask suspect pixels?", 

421 default=False, 

422 ) 

423 suspectMaskName = pexConfig.Field( 

424 dtype=str, 

425 doc="Name of mask plane to use for suspect pixels", 

426 default="SUSPECT", 

427 ) 

428 numEdgeSuspect = pexConfig.Field( 

429 dtype=int, 

430 doc="Number of edge pixels to be flagged as untrustworthy.", 

431 default=0, 

432 ) 

433 edgeMaskLevel = pexConfig.ChoiceField( 

434 dtype=str, 

435 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

436 default="DETECTOR", 

437 allowed={ 

438 'DETECTOR': 'Mask only the edges of the full detector.', 

439 'AMP': 'Mask edges of each amplifier.', 

440 }, 

441 ) 

442 

443 # Initial masking options. 

444 doSetBadRegions = pexConfig.Field( 

445 dtype=bool, 

446 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

447 default=True, 

448 ) 

449 badStatistic = pexConfig.ChoiceField( 

450 dtype=str, 

451 doc="How to estimate the average value for BAD regions.", 

452 default='MEANCLIP', 

453 allowed={ 

454 "MEANCLIP": "Correct using the (clipped) mean of good data", 

455 "MEDIAN": "Correct using the median of the good data", 

456 }, 

457 ) 

458 

459 # Overscan subtraction configuration. 

460 doOverscan = pexConfig.Field( 

461 dtype=bool, 

462 doc="Do overscan subtraction?", 

463 default=True, 

464 ) 

465 overscan = pexConfig.ConfigurableField( 

466 target=OverscanCorrectionTask, 

467 doc="Overscan subtraction task for image segments.", 

468 ) 

469 

470 # Amplifier to CCD assembly configuration 

471 doAssembleCcd = pexConfig.Field( 

472 dtype=bool, 

473 default=True, 

474 doc="Assemble amp-level exposures into a ccd-level exposure?" 

475 ) 

476 assembleCcd = pexConfig.ConfigurableField( 

477 target=AssembleCcdTask, 

478 doc="CCD assembly task", 

479 ) 

480 

481 # General calibration configuration. 

482 doAssembleIsrExposures = pexConfig.Field( 

483 dtype=bool, 

484 default=False, 

485 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

486 ) 

487 doTrimToMatchCalib = pexConfig.Field( 

488 dtype=bool, 

489 default=False, 

490 doc="Trim raw data to match calibration bounding boxes?" 

491 ) 

492 

493 # Bias subtraction. 

494 doBias = pexConfig.Field( 

495 dtype=bool, 

496 doc="Apply bias frame correction?", 

497 default=True, 

498 ) 

499 biasDataProductName = pexConfig.Field( 

500 dtype=str, 

501 doc="Name of the bias data product", 

502 default="bias", 

503 ) 

504 doBiasBeforeOverscan = pexConfig.Field( 

505 dtype=bool, 

506 doc="Reverse order of overscan and bias correction.", 

507 default=False 

508 ) 

509 

510 # Deferred charge correction. 

511 doDeferredCharge = pexConfig.Field( 

512 dtype=bool, 

513 doc="Apply deferred charge correction?", 

514 default=False, 

515 ) 

516 deferredChargeCorrection = pexConfig.ConfigurableField( 

517 target=DeferredChargeTask, 

518 doc="Deferred charge correction task.", 

519 ) 

520 

521 # Variance construction 

522 doVariance = pexConfig.Field( 

523 dtype=bool, 

524 doc="Calculate variance?", 

525 default=True 

526 ) 

527 gain = pexConfig.Field( 

528 dtype=float, 

529 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

530 default=float("NaN"), 

531 ) 

532 readNoise = pexConfig.Field( 

533 dtype=float, 

534 doc="The read noise to use if no Detector is present in the Exposure", 

535 default=0.0, 

536 ) 

537 doEmpiricalReadNoise = pexConfig.Field( 

538 dtype=bool, 

539 default=False, 

540 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

541 ) 

542 usePtcReadNoise = pexConfig.Field( 

543 dtype=bool, 

544 default=False, 

545 doc="Use readnoise values from the Photon Transfer Curve?" 

546 ) 

547 maskNegativeVariance = pexConfig.Field( 

548 dtype=bool, 

549 default=True, 

550 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

551 "in the measurement of the overscan at an edge due to the data falling off faster " 

552 "than the overscan model can account for it." 

553 ) 

554 negativeVarianceMaskName = pexConfig.Field( 

555 dtype=str, 

556 default="BAD", 

557 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

558 ) 

559 # Linearization. 

560 doLinearize = pexConfig.Field( 

561 dtype=bool, 

562 doc="Correct for nonlinearity of the detector's response?", 

563 default=True, 

564 ) 

565 

566 # Crosstalk. 

567 doCrosstalk = pexConfig.Field( 

568 dtype=bool, 

569 doc="Apply intra-CCD crosstalk correction?", 

570 default=False, 

571 ) 

572 doCrosstalkBeforeAssemble = pexConfig.Field( 

573 dtype=bool, 

574 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

575 default=False, 

576 ) 

577 crosstalk = pexConfig.ConfigurableField( 

578 target=CrosstalkTask, 

579 doc="Intra-CCD crosstalk correction", 

580 ) 

581 

582 # Masking options. 

583 doDefect = pexConfig.Field( 

584 dtype=bool, 

585 doc="Apply correction for CCD defects, e.g. hot pixels?", 

586 default=True, 

587 ) 

588 doNanMasking = pexConfig.Field( 

589 dtype=bool, 

590 doc="Mask non-finite (NAN, inf) pixels?", 

591 default=True, 

592 ) 

593 doWidenSaturationTrails = pexConfig.Field( 

594 dtype=bool, 

595 doc="Widen bleed trails based on their width?", 

596 default=True 

597 ) 

598 

599 # Brighter-Fatter correction. 

600 doBrighterFatter = pexConfig.Field( 

601 dtype=bool, 

602 default=False, 

603 doc="Apply the brighter-fatter correction?" 

604 ) 

605 brighterFatterLevel = pexConfig.ChoiceField( 

606 dtype=str, 

607 default="DETECTOR", 

608 doc="The level at which to correct for brighter-fatter.", 

609 allowed={ 

610 "AMP": "Every amplifier treated separately.", 

611 "DETECTOR": "One kernel per detector", 

612 } 

613 ) 

614 brighterFatterMaxIter = pexConfig.Field( 

615 dtype=int, 

616 default=10, 

617 doc="Maximum number of iterations for the brighter-fatter correction" 

618 ) 

619 brighterFatterThreshold = pexConfig.Field( 

620 dtype=float, 

621 default=1000, 

622 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

623 "absolute value of the difference between the current corrected image and the one " 

624 "from the previous iteration summed over all the pixels." 

625 ) 

626 brighterFatterApplyGain = pexConfig.Field( 

627 dtype=bool, 

628 default=True, 

629 doc="Should the gain be applied when applying the brighter-fatter correction?" 

630 ) 

631 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

632 dtype=str, 

633 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

634 "correction.", 

635 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

636 ) 

637 brighterFatterMaskGrowSize = pexConfig.Field( 

638 dtype=int, 

639 default=0, 

640 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

641 "when brighter-fatter correction is applied." 

642 ) 

643 

644 # Dark subtraction. 

645 doDark = pexConfig.Field( 

646 dtype=bool, 

647 doc="Apply dark frame correction?", 

648 default=True, 

649 ) 

650 darkDataProductName = pexConfig.Field( 

651 dtype=str, 

652 doc="Name of the dark data product", 

653 default="dark", 

654 ) 

655 

656 # Camera-specific stray light removal. 

657 doStrayLight = pexConfig.Field( 

658 dtype=bool, 

659 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

660 default=False, 

661 ) 

662 strayLight = pexConfig.ConfigurableField( 

663 target=StrayLightTask, 

664 doc="y-band stray light correction" 

665 ) 

666 

667 # Flat correction. 

668 doFlat = pexConfig.Field( 

669 dtype=bool, 

670 doc="Apply flat field correction?", 

671 default=True, 

672 ) 

673 flatDataProductName = pexConfig.Field( 

674 dtype=str, 

675 doc="Name of the flat data product", 

676 default="flat", 

677 ) 

678 flatScalingType = pexConfig.ChoiceField( 

679 dtype=str, 

680 doc="The method for scaling the flat on the fly.", 

681 default='USER', 

682 allowed={ 

683 "USER": "Scale by flatUserScale", 

684 "MEAN": "Scale by the inverse of the mean", 

685 "MEDIAN": "Scale by the inverse of the median", 

686 }, 

687 ) 

688 flatUserScale = pexConfig.Field( 

689 dtype=float, 

690 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

691 default=1.0, 

692 ) 

693 doTweakFlat = pexConfig.Field( 

694 dtype=bool, 

695 doc="Tweak flats to match observed amplifier ratios?", 

696 default=False 

697 ) 

698 

699 # Amplifier normalization based on gains instead of using flats 

700 # configuration. 

701 doApplyGains = pexConfig.Field( 

702 dtype=bool, 

703 doc="Correct the amplifiers for their gains instead of applying flat correction", 

704 default=False, 

705 ) 

706 usePtcGains = pexConfig.Field( 

707 dtype=bool, 

708 doc="Use the gain values from the Photon Transfer Curve?", 

709 default=False, 

710 ) 

711 normalizeGains = pexConfig.Field( 

712 dtype=bool, 

713 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

714 default=False, 

715 ) 

716 

717 # Fringe correction. 

718 doFringe = pexConfig.Field( 

719 dtype=bool, 

720 doc="Apply fringe correction?", 

721 default=True, 

722 ) 

723 fringe = pexConfig.ConfigurableField( 

724 target=FringeTask, 

725 doc="Fringe subtraction task", 

726 ) 

727 fringeAfterFlat = pexConfig.Field( 

728 dtype=bool, 

729 doc="Do fringe subtraction after flat-fielding?", 

730 default=True, 

731 ) 

732 

733 # Amp offset correction. 

734 doAmpOffset = pexConfig.Field( 

735 doc="Calculate and apply amp offset corrections?", 

736 dtype=bool, 

737 default=False, 

738 ) 

739 ampOffset = pexConfig.ConfigurableField( 

740 doc="Amp offset correction task.", 

741 target=AmpOffsetTask, 

742 ) 

743 

744 # Initial CCD-level background statistics options. 

745 doMeasureBackground = pexConfig.Field( 

746 dtype=bool, 

747 doc="Measure the background level on the reduced image?", 

748 default=False, 

749 ) 

750 

751 # Camera-specific masking configuration. 

752 doCameraSpecificMasking = pexConfig.Field( 

753 dtype=bool, 

754 doc="Mask camera-specific bad regions?", 

755 default=False, 

756 ) 

757 masking = pexConfig.ConfigurableField( 

758 target=MaskingTask, 

759 doc="Masking task." 

760 ) 

761 

762 # Interpolation options. 

763 doInterpolate = pexConfig.Field( 

764 dtype=bool, 

765 doc="Interpolate masked pixels?", 

766 default=True, 

767 ) 

768 doSaturationInterpolation = pexConfig.Field( 

769 dtype=bool, 

770 doc="Perform interpolation over pixels masked as saturated?" 

771 " NB: This is independent of doSaturation; if that is False this plane" 

772 " will likely be blank, resulting in a no-op here.", 

773 default=True, 

774 ) 

775 doNanInterpolation = pexConfig.Field( 

776 dtype=bool, 

777 doc="Perform interpolation over pixels masked as NaN?" 

778 " NB: This is independent of doNanMasking; if that is False this plane" 

779 " will likely be blank, resulting in a no-op here.", 

780 default=True, 

781 ) 

782 doNanInterpAfterFlat = pexConfig.Field( 

783 dtype=bool, 

784 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

785 "also have to interpolate them before flat-fielding."), 

786 default=False, 

787 ) 

788 maskListToInterpolate = pexConfig.ListField( 

789 dtype=str, 

790 doc="List of mask planes that should be interpolated.", 

791 default=['SAT', 'BAD'], 

792 ) 

793 doSaveInterpPixels = pexConfig.Field( 

794 dtype=bool, 

795 doc="Save a copy of the pre-interpolated pixel values?", 

796 default=False, 

797 ) 

798 

799 # Default photometric calibration options. 

800 fluxMag0T1 = pexConfig.DictField( 

801 keytype=str, 

802 itemtype=float, 

803 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

804 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

805 )) 

806 ) 

807 defaultFluxMag0T1 = pexConfig.Field( 

808 dtype=float, 

809 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

810 default=pow(10.0, 0.4*28.0) 

811 ) 

812 

813 # Vignette correction configuration. 

814 doVignette = pexConfig.Field( 

815 dtype=bool, 

816 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

817 "according to vignetting parameters?"), 

818 default=False, 

819 ) 

820 doMaskVignettePolygon = pexConfig.Field( 

821 dtype=bool, 

822 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

823 "is False"), 

824 default=True, 

825 ) 

826 vignetteValue = pexConfig.Field( 

827 dtype=float, 

828 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

829 optional=True, 

830 default=None, 

831 ) 

832 vignette = pexConfig.ConfigurableField( 

833 target=VignetteTask, 

834 doc="Vignetting task.", 

835 ) 

836 

837 # Transmission curve configuration. 

838 doAttachTransmissionCurve = pexConfig.Field( 

839 dtype=bool, 

840 default=False, 

841 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

842 ) 

843 doUseOpticsTransmission = pexConfig.Field( 

844 dtype=bool, 

845 default=True, 

846 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

847 ) 

848 doUseFilterTransmission = pexConfig.Field( 

849 dtype=bool, 

850 default=True, 

851 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

852 ) 

853 doUseSensorTransmission = pexConfig.Field( 

854 dtype=bool, 

855 default=True, 

856 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

857 ) 

858 doUseAtmosphereTransmission = pexConfig.Field( 

859 dtype=bool, 

860 default=True, 

861 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

862 ) 

863 

864 # Illumination correction. 

865 doIlluminationCorrection = pexConfig.Field( 

866 dtype=bool, 

867 default=False, 

868 doc="Perform illumination correction?" 

869 ) 

870 illuminationCorrectionDataProductName = pexConfig.Field( 

871 dtype=str, 

872 doc="Name of the illumination correction data product.", 

873 default="illumcor", 

874 ) 

875 illumScale = pexConfig.Field( 

876 dtype=float, 

877 doc="Scale factor for the illumination correction.", 

878 default=1.0, 

879 ) 

880 illumFilters = pexConfig.ListField( 

881 dtype=str, 

882 default=[], 

883 doc="Only perform illumination correction for these filters." 

884 ) 

885 

886 # Calculate image quality statistics? 

887 doStandardStatistics = pexConfig.Field( 

888 dtype=bool, 

889 doc="Should standard image quality statistics be calculated?", 

890 default=True, 

891 ) 

892 # Calculate additional statistics? 

893 doCalculateStatistics = pexConfig.Field( 

894 dtype=bool, 

895 doc="Should additional ISR statistics be calculated?", 

896 default=False, 

897 ) 

898 isrStats = pexConfig.ConfigurableField( 

899 target=IsrStatisticsTask, 

900 doc="Task to calculate additional statistics.", 

901 ) 

902 

903 # Write the outputs to disk. If ISR is run as a subtask, this may not 

904 # be needed. 

905 doWrite = pexConfig.Field( 

906 dtype=bool, 

907 doc="Persist postISRCCD?", 

908 default=True, 

909 ) 

910 

911 def validate(self): 

912 super().validate() 

913 if self.doFlat and self.doApplyGains: 

914 raise ValueError("You may not specify both doFlat and doApplyGains") 

915 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

916 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

917 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

918 self.maskListToInterpolate.append(self.saturatedMaskName) 

919 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

920 self.maskListToInterpolate.remove(self.saturatedMaskName) 

921 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

922 self.maskListToInterpolate.append("UNMASKEDNAN") 

923 

924 

925class IsrTask(pipeBase.PipelineTask): 

926 """Apply common instrument signature correction algorithms to a raw frame. 

927 

928 The process for correcting imaging data is very similar from 

929 camera to camera. This task provides a vanilla implementation of 

930 doing these corrections, including the ability to turn certain 

931 corrections off if they are not needed. The inputs to the primary 

932 method, `run()`, are a raw exposure to be corrected and the 

933 calibration data products. The raw input is a single chip sized 

934 mosaic of all amps including overscans and other non-science 

935 pixels. 

936 

937 The __init__ method sets up the subtasks for ISR processing, using 

938 the defaults from `lsst.ip.isr`. 

939 

940 Parameters 

941 ---------- 

942 args : `list` 

943 Positional arguments passed to the Task constructor. 

944 None used at this time. 

945 kwargs : `dict`, optional 

946 Keyword arguments passed on to the Task constructor. 

947 None used at this time. 

948 """ 

949 ConfigClass = IsrTaskConfig 

950 _DefaultName = "isr" 

951 

952 def __init__(self, **kwargs): 

953 super().__init__(**kwargs) 

954 self.makeSubtask("assembleCcd") 

955 self.makeSubtask("crosstalk") 

956 self.makeSubtask("strayLight") 

957 self.makeSubtask("fringe") 

958 self.makeSubtask("masking") 

959 self.makeSubtask("overscan") 

960 self.makeSubtask("vignette") 

961 self.makeSubtask("ampOffset") 

962 self.makeSubtask("deferredChargeCorrection") 

963 self.makeSubtask("isrStats") 

964 

965 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

966 inputs = butlerQC.get(inputRefs) 

967 

968 try: 

969 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

970 except Exception as e: 

971 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

972 (inputRefs, e)) 

973 

974 detector = inputs['ccdExposure'].getDetector() 

975 

976 if self.config.doCrosstalk is True: 

977 # Crosstalk sources need to be defined by the pipeline 

978 # yaml if they exist. 

979 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

980 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

981 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

982 else: 

983 coeffVector = (self.config.crosstalk.crosstalkValues 

984 if self.config.crosstalk.useConfigCoefficients else None) 

985 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

986 inputs['crosstalk'] = crosstalkCalib 

987 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

988 if 'crosstalkSources' not in inputs: 

989 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

990 

991 if self.doLinearize(detector) is True: 

992 if 'linearizer' in inputs: 

993 if isinstance(inputs['linearizer'], dict): 

994 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

995 linearizer.fromYaml(inputs['linearizer']) 

996 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

997 elif isinstance(inputs['linearizer'], numpy.ndarray): 

998 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

999 detector=detector, 

1000 log=self.log) 

1001 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1002 else: 

1003 linearizer = inputs['linearizer'] 

1004 linearizer.log = self.log 

1005 inputs['linearizer'] = linearizer 

1006 else: 

1007 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1008 self.log.warning("Constructing linearizer from cameraGeom information.") 

1009 

1010 if self.config.doDefect is True: 

1011 if "defects" in inputs and inputs['defects'] is not None: 

1012 # defects is loaded as a BaseCatalog with columns 

1013 # x0, y0, width, height. Masking expects a list of defects 

1014 # defined by their bounding box 

1015 if not isinstance(inputs["defects"], Defects): 

1016 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1017 

1018 # Load the correct style of brighter-fatter kernel, and repack 

1019 # the information as a numpy array. 

1020 if self.config.doBrighterFatter: 

1021 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1022 if brighterFatterKernel is None: 

1023 brighterFatterKernel = inputs.get('bfKernel', None) 

1024 

1025 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1026 # This is a ISR calib kernel 

1027 detName = detector.getName() 

1028 level = brighterFatterKernel.level 

1029 

1030 # This is expected to be a dictionary of amp-wise gains. 

1031 inputs['bfGains'] = brighterFatterKernel.gain 

1032 if self.config.brighterFatterLevel == 'DETECTOR': 

1033 if level == 'DETECTOR': 

1034 if detName in brighterFatterKernel.detKernels: 

1035 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1036 else: 

1037 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1038 elif level == 'AMP': 

1039 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1040 "fatter kernels.") 

1041 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1042 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1043 elif self.config.brighterFatterLevel == 'AMP': 

1044 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1045 

1046 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1047 expId = inputs['ccdExposure'].info.id 

1048 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1049 expId=expId, 

1050 assembler=self.assembleCcd 

1051 if self.config.doAssembleIsrExposures else None) 

1052 else: 

1053 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1054 

1055 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1056 if 'strayLightData' not in inputs: 

1057 inputs['strayLightData'] = None 

1058 

1059 if self.config.doHeaderProvenance: 

1060 # Add calibration provenanace info to header. 

1061 exposureMetadata = inputs['ccdExposure'].getMetadata() 

1062 for inputName in sorted(inputs.keys()): 

1063 reference = getattr(inputRefs, inputName, None) 

1064 if reference is not None and hasattr(reference, "run"): 

1065 runKey = f"LSST CALIB RUN {inputName.upper()}" 

1066 runValue = reference.run 

1067 idKey = f"LSST CALIB UUID {inputName.upper()}" 

1068 idValue = str(reference.id) 

1069 

1070 exposureMetadata[runKey] = runValue 

1071 exposureMetadata[idKey] = idValue 

1072 

1073 outputs = self.run(**inputs) 

1074 butlerQC.put(outputs, outputRefs) 

1075 

1076 @timeMethod 

1077 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1078 crosstalk=None, crosstalkSources=None, 

1079 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1080 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1081 sensorTransmission=None, atmosphereTransmission=None, 

1082 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1083 deferredChargeCalib=None, 

1084 ): 

1085 """Perform instrument signature removal on an exposure. 

1086 

1087 Steps included in the ISR processing, in order performed, are: 

1088 

1089 - saturation and suspect pixel masking 

1090 - overscan subtraction 

1091 - CCD assembly of individual amplifiers 

1092 - bias subtraction 

1093 - variance image construction 

1094 - linearization of non-linear response 

1095 - crosstalk masking 

1096 - brighter-fatter correction 

1097 - dark subtraction 

1098 - fringe correction 

1099 - stray light subtraction 

1100 - flat correction 

1101 - masking of known defects and camera specific features 

1102 - vignette calculation 

1103 - appending transmission curve and distortion model 

1104 

1105 Parameters 

1106 ---------- 

1107 ccdExposure : `lsst.afw.image.Exposure` 

1108 The raw exposure that is to be run through ISR. The 

1109 exposure is modified by this method. 

1110 camera : `lsst.afw.cameraGeom.Camera`, optional 

1111 The camera geometry for this exposure. Required if 

1112 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1113 ``flat`` does not have an associated detector. 

1114 bias : `lsst.afw.image.Exposure`, optional 

1115 Bias calibration frame. 

1116 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1117 Functor for linearization. 

1118 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1119 Calibration for crosstalk. 

1120 crosstalkSources : `list`, optional 

1121 List of possible crosstalk sources. 

1122 dark : `lsst.afw.image.Exposure`, optional 

1123 Dark calibration frame. 

1124 flat : `lsst.afw.image.Exposure`, optional 

1125 Flat calibration frame. 

1126 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1127 Photon transfer curve dataset, with, e.g., gains 

1128 and read noise. 

1129 bfKernel : `numpy.ndarray`, optional 

1130 Brighter-fatter kernel. 

1131 bfGains : `dict` of `float`, optional 

1132 Gains used to override the detector's nominal gains for the 

1133 brighter-fatter correction. A dict keyed by amplifier name for 

1134 the detector in question. 

1135 defects : `lsst.ip.isr.Defects`, optional 

1136 List of defects. 

1137 fringes : `lsst.pipe.base.Struct`, optional 

1138 Struct containing the fringe correction data, with 

1139 elements: 

1140 

1141 ``fringes`` 

1142 fringe calibration frame (`lsst.afw.image.Exposure`) 

1143 ``seed`` 

1144 random seed derived from the ``ccdExposureId`` for random 

1145 number generator (`numpy.uint32`) 

1146 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1147 A ``TransmissionCurve`` that represents the throughput of the, 

1148 optics, to be evaluated in focal-plane coordinates. 

1149 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1150 A ``TransmissionCurve`` that represents the throughput of the 

1151 filter itself, to be evaluated in focal-plane coordinates. 

1152 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1153 A ``TransmissionCurve`` that represents the throughput of the 

1154 sensor itself, to be evaluated in post-assembly trimmed detector 

1155 coordinates. 

1156 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1157 A ``TransmissionCurve`` that represents the throughput of the 

1158 atmosphere, assumed to be spatially constant. 

1159 detectorNum : `int`, optional 

1160 The integer number for the detector to process. 

1161 strayLightData : `object`, optional 

1162 Opaque object containing calibration information for stray-light 

1163 correction. If `None`, no correction will be performed. 

1164 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1165 Illumination correction image. 

1166 

1167 Returns 

1168 ------- 

1169 result : `lsst.pipe.base.Struct` 

1170 Result struct with component: 

1171 

1172 ``exposure`` 

1173 The fully ISR corrected exposure. 

1174 (`lsst.afw.image.Exposure`) 

1175 ``outputExposure`` 

1176 An alias for ``exposure``. (`lsst.afw.image.Exposure`) 

1177 ``ossThumb`` 

1178 Thumbnail image of the exposure after overscan subtraction. 

1179 (`numpy.ndarray`) 

1180 ``flattenedThumb`` 

1181 Thumbnail image of the exposure after flat-field correction. 

1182 (`numpy.ndarray`) 

1183 ``outputStatistics`` 

1184 Values of the additional statistics calculated. 

1185 

1186 Raises 

1187 ------ 

1188 RuntimeError 

1189 Raised if a configuration option is set to `True`, but the 

1190 required calibration data has not been specified. 

1191 

1192 Notes 

1193 ----- 

1194 The current processed exposure can be viewed by setting the 

1195 appropriate `lsstDebug` entries in the ``debug.display`` 

1196 dictionary. The names of these entries correspond to some of 

1197 the `IsrTaskConfig` Boolean options, with the value denoting the 

1198 frame to use. The exposure is shown inside the matching 

1199 option check and after the processing of that step has 

1200 finished. The steps with debug points are: 

1201 

1202 * doAssembleCcd 

1203 * doBias 

1204 * doCrosstalk 

1205 * doBrighterFatter 

1206 * doDark 

1207 * doFringe 

1208 * doStrayLight 

1209 * doFlat 

1210 

1211 In addition, setting the ``postISRCCD`` entry displays the 

1212 exposure after all ISR processing has finished. 

1213 """ 

1214 

1215 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1216 bias = self.ensureExposure(bias, camera, detectorNum) 

1217 dark = self.ensureExposure(dark, camera, detectorNum) 

1218 flat = self.ensureExposure(flat, camera, detectorNum) 

1219 

1220 ccd = ccdExposure.getDetector() 

1221 filterLabel = ccdExposure.getFilter() 

1222 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1223 

1224 if not ccd: 

1225 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1226 ccd = [FakeAmp(ccdExposure, self.config)] 

1227 

1228 # Validate Input 

1229 if self.config.doBias and bias is None: 

1230 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1231 if self.doLinearize(ccd) and linearizer is None: 

1232 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1233 if self.config.doBrighterFatter and bfKernel is None: 

1234 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1235 if self.config.doDark and dark is None: 

1236 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1237 if self.config.doFlat and flat is None: 

1238 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1239 if self.config.doDefect and defects is None: 

1240 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1241 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1242 and fringes.fringes is None): 

1243 # The `fringes` object needs to be a pipeBase.Struct, as 

1244 # we use it as a `dict` for the parameters of 

1245 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1246 # not be `None` if `doFringe=True`. Otherwise, raise. 

1247 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1248 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1249 and illumMaskedImage is None): 

1250 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1251 if (self.config.doDeferredCharge and deferredChargeCalib is None): 

1252 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1253 

1254 if self.config.doHeaderProvenance: 

1255 # Inputs have been validated, so we can add their date 

1256 # information to the output header. 

1257 exposureMetadata = ccdExposure.getMetadata() 

1258 if self.config.doBias: 

1259 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias) 

1260 if self.config.doBrighterFatter: 

1261 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel) 

1262 if self.config.doCrosstalk: 

1263 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk) 

1264 if self.config.doDark: 

1265 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark) 

1266 if self.config.doDefect: 

1267 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects) 

1268 if self.config.doDeferredCharge: 

1269 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib) 

1270 if self.config.doFlat: 

1271 exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat) 

1272 if (self.config.doFringe and physicalFilter in self.fringe.config.filters): 

1273 exposureMetadata["LSST CALIB DATE FRINGE"] = self.extractCalibDate(fringes.fringes) 

1274 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters): 

1275 exposureMetadata["LSST CALIB DATE ILLUMINATION"] = self.extractCalibDate(illumMaskedImage) 

1276 if self.doLinearize(ccd): 

1277 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer) 

1278 if self.config.usePtcGains or self.config.usePtcReadNoise: 

1279 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc) 

1280 if self.config.doStrayLight: 

1281 exposureMetadata["LSST CALIB DATE STRAYLIGHT"] = self.extractCalibDate(strayLightData) 

1282 if self.config.doAttachTransmissionCurve: 

1283 exposureMetadata["LSST CALIB DATE OPTICS_TR"] = self.extractCalibDate(opticsTransmission) 

1284 exposureMetadata["LSST CALIB DATE FILTER_TR"] = self.extractCalibDate(filterTransmission) 

1285 exposureMetadata["LSST CALIB DATE SENSOR_TR"] = self.extractCalibDate(sensorTransmission) 

1286 exposureMetadata["LSST CALIB DATE ATMOSP_TR"] = self.extractCalibDate(atmosphereTransmission) 

1287 

1288 # Begin ISR processing. 

1289 if self.config.doConvertIntToFloat: 

1290 self.log.info("Converting exposure to floating point values.") 

1291 ccdExposure = self.convertIntToFloat(ccdExposure) 

1292 

1293 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1294 self.log.info("Applying bias correction.") 

1295 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1296 trimToFit=self.config.doTrimToMatchCalib) 

1297 self.debugView(ccdExposure, "doBias") 

1298 

1299 # Amplifier level processing. 

1300 overscans = [] 

1301 

1302 if self.config.doOverscan and self.config.overscan.doParallelOverscan: 

1303 # This will attempt to mask bleed pixels across all amplifiers. 

1304 self.overscan.maskParallelOverscan(ccdExposure, ccd) 

1305 

1306 for amp in ccd: 

1307 # if ccdExposure is one amp, 

1308 # check for coverage to prevent performing ops multiple times 

1309 if ccdExposure.getBBox().contains(amp.getBBox()): 

1310 # Check for fully masked bad amplifiers, 

1311 # and generate masks for SUSPECT and SATURATED values. 

1312 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1313 

1314 if self.config.doOverscan and not badAmp: 

1315 # Overscan correction on amp-by-amp basis. 

1316 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1317 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1318 if overscanResults is not None and \ 

1319 self.config.qa is not None and self.config.qa.saveStats is True: 

1320 

1321 self.metadata[f"FIT MEDIAN {amp.getName()}"] = overscanResults.overscanMean 

1322 self.metadata[f"FIT STDEV {amp.getName()}"] = overscanResults.overscanSigma 

1323 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1324 amp.getName(), overscanResults.overscanMean, 

1325 overscanResults.overscanSigma) 

1326 

1327 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = overscanResults.residualMean 

1328 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = overscanResults.residualSigma 

1329 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1330 amp.getName(), overscanResults.residualMean, 

1331 overscanResults.residualSigma) 

1332 

1333 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1334 else: 

1335 if badAmp: 

1336 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1337 overscanResults = None 

1338 

1339 overscans.append(overscanResults if overscanResults is not None else None) 

1340 else: 

1341 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1342 

1343 if self.config.doDeferredCharge: 

1344 self.log.info("Applying deferred charge/CTI correction.") 

1345 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib) 

1346 self.debugView(ccdExposure, "doDeferredCharge") 

1347 

1348 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1349 self.log.info("Applying crosstalk correction.") 

1350 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1351 crosstalkSources=crosstalkSources, camera=camera) 

1352 self.debugView(ccdExposure, "doCrosstalk") 

1353 

1354 if self.config.doAssembleCcd: 

1355 self.log.info("Assembling CCD from amplifiers.") 

1356 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1357 

1358 if self.config.expectWcs and not ccdExposure.getWcs(): 

1359 self.log.warning("No WCS found in input exposure.") 

1360 self.debugView(ccdExposure, "doAssembleCcd") 

1361 

1362 ossThumb = None 

1363 if self.config.qa.doThumbnailOss: 

1364 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1365 

1366 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1367 self.log.info("Applying bias correction.") 

1368 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1369 trimToFit=self.config.doTrimToMatchCalib) 

1370 self.debugView(ccdExposure, "doBias") 

1371 

1372 if self.config.doVariance: 

1373 for amp, overscanResults in zip(ccd, overscans): 

1374 if ccdExposure.getBBox().contains(amp.getBBox()): 

1375 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1376 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1377 if overscanResults is not None: 

1378 self.updateVariance(ampExposure, amp, 

1379 overscanImage=overscanResults.overscanImage, 

1380 ptcDataset=ptc) 

1381 else: 

1382 self.updateVariance(ampExposure, amp, 

1383 overscanImage=None, 

1384 ptcDataset=ptc) 

1385 if self.config.qa is not None and self.config.qa.saveStats is True: 

1386 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1387 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1388 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1389 qaStats.getValue(afwMath.MEDIAN) 

1390 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1391 qaStats.getValue(afwMath.STDEVCLIP) 

1392 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1393 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1394 qaStats.getValue(afwMath.STDEVCLIP)) 

1395 if self.config.maskNegativeVariance: 

1396 self.maskNegativeVariance(ccdExposure) 

1397 

1398 if self.doLinearize(ccd): 

1399 self.log.info("Applying linearizer.") 

1400 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1401 detector=ccd, log=self.log) 

1402 

1403 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1404 self.log.info("Applying crosstalk correction.") 

1405 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1406 crosstalkSources=crosstalkSources, isTrimmed=True) 

1407 self.debugView(ccdExposure, "doCrosstalk") 

1408 

1409 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1410 # widen trails, and do anything else the camera needs. Saturated and 

1411 # suspect pixels have already been masked. 

1412 if self.config.doDefect: 

1413 self.log.info("Masking defects.") 

1414 self.maskDefect(ccdExposure, defects) 

1415 

1416 if self.config.numEdgeSuspect > 0: 

1417 self.log.info("Masking edges as SUSPECT.") 

1418 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1419 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1420 

1421 if self.config.doNanMasking: 

1422 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1423 self.maskNan(ccdExposure) 

1424 

1425 if self.config.doWidenSaturationTrails: 

1426 self.log.info("Widening saturation trails.") 

1427 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1428 

1429 if self.config.doCameraSpecificMasking: 

1430 self.log.info("Masking regions for camera specific reasons.") 

1431 self.masking.run(ccdExposure) 

1432 

1433 if self.config.doBrighterFatter: 

1434 # We need to apply flats and darks before we can interpolate, and 

1435 # we need to interpolate before we do B-F, but we do B-F without 

1436 # the flats and darks applied so we can work in units of electrons 

1437 # or holes. This context manager applies and then removes the darks 

1438 # and flats. 

1439 # 

1440 # We also do not want to interpolate values here, so operate on 

1441 # temporary images so we can apply only the BF-correction and roll 

1442 # back the interpolation. 

1443 interpExp = ccdExposure.clone() 

1444 with self.flatContext(interpExp, flat, dark): 

1445 isrFunctions.interpolateFromMask( 

1446 maskedImage=interpExp.getMaskedImage(), 

1447 fwhm=self.config.fwhm, 

1448 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1449 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1450 ) 

1451 bfExp = interpExp.clone() 

1452 

1453 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1454 type(bfKernel), type(bfGains)) 

1455 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1456 self.config.brighterFatterMaxIter, 

1457 self.config.brighterFatterThreshold, 

1458 self.config.brighterFatterApplyGain, 

1459 bfGains) 

1460 if bfResults[1] == self.config.brighterFatterMaxIter: 

1461 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1462 bfResults[0]) 

1463 else: 

1464 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1465 bfResults[1]) 

1466 image = ccdExposure.getMaskedImage().getImage() 

1467 bfCorr = bfExp.getMaskedImage().getImage() 

1468 bfCorr -= interpExp.getMaskedImage().getImage() 

1469 image += bfCorr 

1470 

1471 # Applying the brighter-fatter correction applies a 

1472 # convolution to the science image. At the edges this 

1473 # convolution may not have sufficient valid pixels to 

1474 # produce a valid correction. Mark pixels within the size 

1475 # of the brighter-fatter kernel as EDGE to warn of this 

1476 # fact. 

1477 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1478 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1479 maskPlane="EDGE") 

1480 

1481 if self.config.brighterFatterMaskGrowSize > 0: 

1482 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1483 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1484 isrFunctions.growMasks(ccdExposure.getMask(), 

1485 radius=self.config.brighterFatterMaskGrowSize, 

1486 maskNameList=maskPlane, 

1487 maskValue=maskPlane) 

1488 

1489 self.debugView(ccdExposure, "doBrighterFatter") 

1490 

1491 if self.config.doDark: 

1492 self.log.info("Applying dark correction.") 

1493 self.darkCorrection(ccdExposure, dark) 

1494 self.debugView(ccdExposure, "doDark") 

1495 

1496 if self.config.doFringe and not self.config.fringeAfterFlat: 

1497 self.log.info("Applying fringe correction before flat.") 

1498 self.fringe.run(ccdExposure, **fringes.getDict()) 

1499 self.debugView(ccdExposure, "doFringe") 

1500 

1501 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1502 self.log.info("Checking strayLight correction.") 

1503 self.strayLight.run(ccdExposure, strayLightData) 

1504 self.debugView(ccdExposure, "doStrayLight") 

1505 

1506 if self.config.doFlat: 

1507 self.log.info("Applying flat correction.") 

1508 self.flatCorrection(ccdExposure, flat) 

1509 self.debugView(ccdExposure, "doFlat") 

1510 

1511 if self.config.doApplyGains: 

1512 self.log.info("Applying gain correction instead of flat.") 

1513 if self.config.usePtcGains: 

1514 self.log.info("Using gains from the Photon Transfer Curve.") 

1515 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1516 ptcGains=ptc.gain) 

1517 else: 

1518 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1519 

1520 if self.config.doFringe and self.config.fringeAfterFlat: 

1521 self.log.info("Applying fringe correction after flat.") 

1522 self.fringe.run(ccdExposure, **fringes.getDict()) 

1523 

1524 if self.config.doVignette: 

1525 if self.config.doMaskVignettePolygon: 

1526 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1527 else: 

1528 self.log.info("Constructing and attaching vignette polygon.") 

1529 self.vignettePolygon = self.vignette.run( 

1530 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1531 vignetteValue=self.config.vignetteValue, log=self.log) 

1532 

1533 if self.config.doAttachTransmissionCurve: 

1534 self.log.info("Adding transmission curves.") 

1535 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1536 filterTransmission=filterTransmission, 

1537 sensorTransmission=sensorTransmission, 

1538 atmosphereTransmission=atmosphereTransmission) 

1539 

1540 flattenedThumb = None 

1541 if self.config.qa.doThumbnailFlattened: 

1542 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1543 

1544 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1545 self.log.info("Performing illumination correction.") 

1546 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1547 illumMaskedImage, illumScale=self.config.illumScale, 

1548 trimToFit=self.config.doTrimToMatchCalib) 

1549 

1550 preInterpExp = None 

1551 if self.config.doSaveInterpPixels: 

1552 preInterpExp = ccdExposure.clone() 

1553 

1554 # Reset and interpolate bad pixels. 

1555 # 

1556 # Large contiguous bad regions (which should have the BAD mask 

1557 # bit set) should have their values set to the image median. 

1558 # This group should include defects and bad amplifiers. As the 

1559 # area covered by these defects are large, there's little 

1560 # reason to expect that interpolation would provide a more 

1561 # useful value. 

1562 # 

1563 # Smaller defects can be safely interpolated after the larger 

1564 # regions have had their pixel values reset. This ensures 

1565 # that the remaining defects adjacent to bad amplifiers (as an 

1566 # example) do not attempt to interpolate extreme values. 

1567 if self.config.doSetBadRegions: 

1568 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1569 if badPixelCount > 0: 

1570 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1571 

1572 if self.config.doInterpolate: 

1573 self.log.info("Interpolating masked pixels.") 

1574 isrFunctions.interpolateFromMask( 

1575 maskedImage=ccdExposure.getMaskedImage(), 

1576 fwhm=self.config.fwhm, 

1577 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1578 maskNameList=list(self.config.maskListToInterpolate) 

1579 ) 

1580 

1581 self.roughZeroPoint(ccdExposure) 

1582 

1583 # correct for amp offsets within the CCD 

1584 if self.config.doAmpOffset: 

1585 self.log.info("Correcting amp offsets.") 

1586 self.ampOffset.run(ccdExposure) 

1587 

1588 if self.config.doMeasureBackground: 

1589 self.log.info("Measuring background level.") 

1590 self.measureBackground(ccdExposure, self.config.qa) 

1591 

1592 if self.config.qa is not None and self.config.qa.saveStats is True: 

1593 for amp in ccd: 

1594 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1595 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1596 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1597 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1598 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1599 qaStats.getValue(afwMath.STDEVCLIP) 

1600 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1601 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1602 qaStats.getValue(afwMath.STDEVCLIP)) 

1603 

1604 # Calculate standard image quality statistics 

1605 if self.config.doStandardStatistics: 

1606 metadata = ccdExposure.getMetadata() 

1607 for amp in ccd: 

1608 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1609 ampName = amp.getName() 

1610 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels( 

1611 ampExposure.getMaskedImage(), 

1612 [self.config.saturatedMaskName] 

1613 ) 

1614 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels( 

1615 ampExposure.getMaskedImage(), 

1616 ["BAD"] 

1617 ) 

1618 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1619 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP) 

1620 

1621 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN) 

1622 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN) 

1623 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP) 

1624 

1625 k1 = f"LSST ISR FINAL MEDIAN {ampName}" 

1626 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}" 

1627 if self.config.doOverscan and k1 in metadata and k2 in metadata: 

1628 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2] 

1629 else: 

1630 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan 

1631 

1632 # calculate additional statistics. 

1633 outputStatistics = None 

1634 if self.config.doCalculateStatistics: 

1635 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1636 ptc=ptc).results 

1637 

1638 self.debugView(ccdExposure, "postISRCCD") 

1639 

1640 return pipeBase.Struct( 

1641 exposure=ccdExposure, 

1642 ossThumb=ossThumb, 

1643 flattenedThumb=flattenedThumb, 

1644 

1645 preInterpExposure=preInterpExp, 

1646 outputExposure=ccdExposure, 

1647 outputOssThumbnail=ossThumb, 

1648 outputFlattenedThumbnail=flattenedThumb, 

1649 outputStatistics=outputStatistics, 

1650 ) 

1651 

1652 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1653 """Ensure that the data returned by Butler is a fully constructed exp. 

1654 

1655 ISR requires exposure-level image data for historical reasons, so if we 

1656 did not recieve that from Butler, construct it from what we have, 

1657 modifying the input in place. 

1658 

1659 Parameters 

1660 ---------- 

1661 inputExp : `lsst.afw.image` image-type. 

1662 The input data structure obtained from Butler. 

1663 Can be `lsst.afw.image.Exposure`, 

1664 `lsst.afw.image.DecoratedImageU`, 

1665 or `lsst.afw.image.ImageF` 

1666 camera : `lsst.afw.cameraGeom.camera`, optional 

1667 The camera associated with the image. Used to find the appropriate 

1668 detector if detector is not already set. 

1669 detectorNum : `int`, optional 

1670 The detector in the camera to attach, if the detector is not 

1671 already set. 

1672 

1673 Returns 

1674 ------- 

1675 inputExp : `lsst.afw.image.Exposure` 

1676 The re-constructed exposure, with appropriate detector parameters. 

1677 

1678 Raises 

1679 ------ 

1680 TypeError 

1681 Raised if the input data cannot be used to construct an exposure. 

1682 """ 

1683 if isinstance(inputExp, afwImage.DecoratedImageU): 

1684 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1685 elif isinstance(inputExp, afwImage.ImageF): 

1686 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1687 elif isinstance(inputExp, afwImage.MaskedImageF): 

1688 inputExp = afwImage.makeExposure(inputExp) 

1689 elif isinstance(inputExp, afwImage.Exposure): 

1690 pass 

1691 elif inputExp is None: 

1692 # Assume this will be caught by the setup if it is a problem. 

1693 return inputExp 

1694 else: 

1695 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1696 (type(inputExp), )) 

1697 

1698 if inputExp.getDetector() is None: 

1699 if camera is None or detectorNum is None: 

1700 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1701 'without a detector set.') 

1702 inputExp.setDetector(camera[detectorNum]) 

1703 

1704 return inputExp 

1705 

1706 @staticmethod 

1707 def extractCalibDate(calib): 

1708 """Extract common calibration metadata values that will be written to 

1709 output header. 

1710 

1711 Parameters 

1712 ---------- 

1713 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib` 

1714 Calibration to pull date information from. 

1715 

1716 Returns 

1717 ------- 

1718 dateString : `str` 

1719 Calibration creation date string to add to header. 

1720 """ 

1721 if hasattr(calib, "getMetadata"): 

1722 if 'CALIB_CREATION_DATE' in calib.getMetadata(): 

1723 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"), 

1724 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown"))) 

1725 else: 

1726 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"), 

1727 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown"))) 

1728 else: 

1729 return "Unknown Unknown" 

1730 

1731 def convertIntToFloat(self, exposure): 

1732 """Convert exposure image from uint16 to float. 

1733 

1734 If the exposure does not need to be converted, the input is 

1735 immediately returned. For exposures that are converted to use 

1736 floating point pixels, the variance is set to unity and the 

1737 mask to zero. 

1738 

1739 Parameters 

1740 ---------- 

1741 exposure : `lsst.afw.image.Exposure` 

1742 The raw exposure to be converted. 

1743 

1744 Returns 

1745 ------- 

1746 newexposure : `lsst.afw.image.Exposure` 

1747 The input ``exposure``, converted to floating point pixels. 

1748 

1749 Raises 

1750 ------ 

1751 RuntimeError 

1752 Raised if the exposure type cannot be converted to float. 

1753 

1754 """ 

1755 if isinstance(exposure, afwImage.ExposureF): 

1756 # Nothing to be done 

1757 self.log.debug("Exposure already of type float.") 

1758 return exposure 

1759 if not hasattr(exposure, "convertF"): 

1760 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1761 

1762 newexposure = exposure.convertF() 

1763 newexposure.variance[:] = 1 

1764 newexposure.mask[:] = 0x0 

1765 

1766 return newexposure 

1767 

1768 def maskAmplifier(self, ccdExposure, amp, defects): 

1769 """Identify bad amplifiers, saturated and suspect pixels. 

1770 

1771 Parameters 

1772 ---------- 

1773 ccdExposure : `lsst.afw.image.Exposure` 

1774 Input exposure to be masked. 

1775 amp : `lsst.afw.cameraGeom.Amplifier` 

1776 Catalog of parameters defining the amplifier on this 

1777 exposure to mask. 

1778 defects : `lsst.ip.isr.Defects` 

1779 List of defects. Used to determine if the entire 

1780 amplifier is bad. 

1781 

1782 Returns 

1783 ------- 

1784 badAmp : `Bool` 

1785 If this is true, the entire amplifier area is covered by 

1786 defects and unusable. 

1787 

1788 """ 

1789 maskedImage = ccdExposure.getMaskedImage() 

1790 

1791 badAmp = False 

1792 

1793 # Check if entire amp region is defined as a defect 

1794 # NB: need to use amp.getBBox() for correct comparison with current 

1795 # defects definition. 

1796 if defects is not None: 

1797 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1798 

1799 # In the case of a bad amp, we will set mask to "BAD" 

1800 # (here use amp.getRawBBox() for correct association with pixels in 

1801 # current ccdExposure). 

1802 if badAmp: 

1803 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1804 afwImage.PARENT) 

1805 maskView = dataView.getMask() 

1806 maskView |= maskView.getPlaneBitMask("BAD") 

1807 del maskView 

1808 return badAmp 

1809 

1810 # Mask remaining defects after assembleCcd() to allow for defects that 

1811 # cross amplifier boundaries. Saturation and suspect pixels can be 

1812 # masked now, though. 

1813 limits = dict() 

1814 if self.config.doSaturation and not badAmp: 

1815 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1816 if self.config.doSuspect and not badAmp: 

1817 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1818 if math.isfinite(self.config.saturation): 

1819 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1820 

1821 for maskName, maskThreshold in limits.items(): 

1822 if not math.isnan(maskThreshold): 

1823 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1824 isrFunctions.makeThresholdMask( 

1825 maskedImage=dataView, 

1826 threshold=maskThreshold, 

1827 growFootprints=0, 

1828 maskName=maskName 

1829 ) 

1830 

1831 # Determine if we've fully masked this amplifier with SUSPECT and 

1832 # SAT pixels. 

1833 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1834 afwImage.PARENT) 

1835 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1836 self.config.suspectMaskName]) 

1837 if numpy.all(maskView.getArray() & maskVal > 0): 

1838 badAmp = True 

1839 maskView |= maskView.getPlaneBitMask("BAD") 

1840 

1841 return badAmp 

1842 

1843 def overscanCorrection(self, ccdExposure, amp): 

1844 """Apply overscan correction in place. 

1845 

1846 This method does initial pixel rejection of the overscan 

1847 region. The overscan can also be optionally segmented to 

1848 allow for discontinuous overscan responses to be fit 

1849 separately. The actual overscan subtraction is performed by 

1850 the `lsst.ip.isr.overscan.OverscanTask`, which is called here 

1851 after the amplifier is preprocessed. 

1852 

1853 Parameters 

1854 ---------- 

1855 ccdExposure : `lsst.afw.image.Exposure` 

1856 Exposure to have overscan correction performed. 

1857 amp : `lsst.afw.cameraGeom.Amplifer` 

1858 The amplifier to consider while correcting the overscan. 

1859 

1860 Returns 

1861 ------- 

1862 overscanResults : `lsst.pipe.base.Struct` 

1863 Result struct with components: 

1864 

1865 ``imageFit`` 

1866 Value or fit subtracted from the amplifier image data. 

1867 (scalar or `lsst.afw.image.Image`) 

1868 ``overscanFit`` 

1869 Value or fit subtracted from the overscan image data. 

1870 (scalar or `lsst.afw.image.Image`) 

1871 ``overscanImage`` 

1872 Image of the overscan region with the overscan 

1873 correction applied. This quantity is used to estimate 

1874 the amplifier read noise empirically. 

1875 (`lsst.afw.image.Image`) 

1876 ``edgeMask`` 

1877 Mask of the suspect pixels. (`lsst.afw.image.Mask`) 

1878 ``overscanMean`` 

1879 Median overscan fit value. (`float`) 

1880 ``overscanSigma`` 

1881 Clipped standard deviation of the overscan after 

1882 correction. (`float`) 

1883 

1884 Raises 

1885 ------ 

1886 RuntimeError 

1887 Raised if the ``amp`` does not contain raw pixel information. 

1888 

1889 See Also 

1890 -------- 

1891 lsst.ip.isr.overscan.OverscanTask 

1892 """ 

1893 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1894 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1895 return None 

1896 

1897 # Perform overscan correction on subregions. 

1898 overscanResults = self.overscan.run(ccdExposure, amp) 

1899 

1900 metadata = ccdExposure.getMetadata() 

1901 ampName = amp.getName() 

1902 

1903 keyBase = "LSST ISR OVERSCAN" 

1904 # Updated quantities 

1905 if isinstance(overscanResults.overscanMean, float): 

1906 # Serial overscan correction only: 

1907 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean 

1908 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian 

1909 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma 

1910 

1911 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean 

1912 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian 

1913 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma 

1914 elif isinstance(overscanResults.overscanMean, tuple): 

1915 # Both serial and parallel overscan have run: 

1916 metadata[f"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0] 

1917 metadata[f"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0] 

1918 metadata[f"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0] 

1919 

1920 metadata[f"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1] 

1921 metadata[f"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1] 

1922 metadata[f"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1] 

1923 

1924 metadata[f"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0] 

1925 metadata[f"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0] 

1926 metadata[f"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0] 

1927 

1928 metadata[f"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1] 

1929 metadata[f"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1] 

1930 metadata[f"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1] 

1931 else: 

1932 self.log.warning("Unexpected type for overscan values; none added to header.") 

1933 

1934 return overscanResults 

1935 

1936 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

1937 """Set the variance plane using the gain and read noise 

1938 

1939 The read noise is calculated from the ``overscanImage`` if the 

1940 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1941 the value from the amplifier data is used. 

1942 

1943 Parameters 

1944 ---------- 

1945 ampExposure : `lsst.afw.image.Exposure` 

1946 Exposure to process. 

1947 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp` 

1948 Amplifier detector data. 

1949 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

1950 Image of overscan, required only for empirical read noise. 

1951 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1952 PTC dataset containing the gains and read noise. 

1953 

1954 Raises 

1955 ------ 

1956 RuntimeError 

1957 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

1958 are ``True``, but ptcDataset is not provided. 

1959 

1960 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

1961 ``overscanImage`` is ``None``. 

1962 

1963 See also 

1964 -------- 

1965 lsst.ip.isr.isrFunctions.updateVariance 

1966 """ 

1967 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

1968 if self.config.usePtcGains: 

1969 if ptcDataset is None: 

1970 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

1971 else: 

1972 gain = ptcDataset.gain[amp.getName()] 

1973 self.log.info("Using gain from Photon Transfer Curve.") 

1974 else: 

1975 gain = amp.getGain() 

1976 

1977 if math.isnan(gain): 

1978 gain = 1.0 

1979 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

1980 elif gain <= 0: 

1981 patchedGain = 1.0 

1982 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

1983 amp.getName(), gain, patchedGain) 

1984 gain = patchedGain 

1985 

1986 if self.config.doEmpiricalReadNoise and overscanImage is None: 

1987 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(), 

1988 [self.config.saturatedMaskName, 

1989 self.config.suspectMaskName, 

1990 "BAD", "NO_DATA"]) 

1991 allPixels = ampExposure.getWidth() * ampExposure.getHeight() 

1992 if allPixels == badPixels: 

1993 # If the image is bad, do not raise. 

1994 self.log.info("Skipping empirical read noise for amp %s. No good pixels.", 

1995 amp.getName()) 

1996 else: 

1997 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

1998 

1999 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2000 stats = afwMath.StatisticsControl() 

2001 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2002 readNoise = afwMath.makeStatistics(overscanImage.getImage(), 

2003 afwMath.STDEVCLIP, stats).getValue() 

2004 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2005 amp.getName(), readNoise) 

2006 elif self.config.usePtcReadNoise: 

2007 if ptcDataset is None: 

2008 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2009 else: 

2010 readNoise = ptcDataset.noise[amp.getName()] 

2011 self.log.info("Using read noise from Photon Transfer Curve.") 

2012 else: 

2013 readNoise = amp.getReadNoise() 

2014 

2015 metadata = ampExposure.getMetadata() 

2016 metadata[f'LSST GAIN {amp.getName()}'] = gain 

2017 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise 

2018 

2019 isrFunctions.updateVariance( 

2020 maskedImage=ampExposure.getMaskedImage(), 

2021 gain=gain, 

2022 readNoise=readNoise, 

2023 ) 

2024 

2025 def maskNegativeVariance(self, exposure): 

2026 """Identify and mask pixels with negative variance values. 

2027 

2028 Parameters 

2029 ---------- 

2030 exposure : `lsst.afw.image.Exposure` 

2031 Exposure to process. 

2032 

2033 See Also 

2034 -------- 

2035 lsst.ip.isr.isrFunctions.updateVariance 

2036 """ 

2037 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2038 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2039 exposure.mask.array[bad] |= maskPlane 

2040 

2041 def darkCorrection(self, exposure, darkExposure, invert=False): 

2042 """Apply dark correction in place. 

2043 

2044 Parameters 

2045 ---------- 

2046 exposure : `lsst.afw.image.Exposure` 

2047 Exposure to process. 

2048 darkExposure : `lsst.afw.image.Exposure` 

2049 Dark exposure of the same size as ``exposure``. 

2050 invert : `Bool`, optional 

2051 If True, re-add the dark to an already corrected image. 

2052 

2053 Raises 

2054 ------ 

2055 RuntimeError 

2056 Raised if either ``exposure`` or ``darkExposure`` do not 

2057 have their dark time defined. 

2058 

2059 See Also 

2060 -------- 

2061 lsst.ip.isr.isrFunctions.darkCorrection 

2062 """ 

2063 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2064 if math.isnan(expScale): 

2065 raise RuntimeError("Exposure darktime is NAN.") 

2066 if darkExposure.getInfo().getVisitInfo() is not None \ 

2067 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2068 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2069 else: 

2070 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2071 # so getDarkTime() does not exist. 

2072 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2073 darkScale = 1.0 

2074 

2075 isrFunctions.darkCorrection( 

2076 maskedImage=exposure.getMaskedImage(), 

2077 darkMaskedImage=darkExposure.getMaskedImage(), 

2078 expScale=expScale, 

2079 darkScale=darkScale, 

2080 invert=invert, 

2081 trimToFit=self.config.doTrimToMatchCalib 

2082 ) 

2083 

2084 def doLinearize(self, detector): 

2085 """Check if linearization is needed for the detector cameraGeom. 

2086 

2087 Checks config.doLinearize and the linearity type of the first 

2088 amplifier. 

2089 

2090 Parameters 

2091 ---------- 

2092 detector : `lsst.afw.cameraGeom.Detector` 

2093 Detector to get linearity type from. 

2094 

2095 Returns 

2096 ------- 

2097 doLinearize : `Bool` 

2098 If True, linearization should be performed. 

2099 """ 

2100 return self.config.doLinearize and \ 

2101 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2102 

2103 def flatCorrection(self, exposure, flatExposure, invert=False): 

2104 """Apply flat correction in place. 

2105 

2106 Parameters 

2107 ---------- 

2108 exposure : `lsst.afw.image.Exposure` 

2109 Exposure to process. 

2110 flatExposure : `lsst.afw.image.Exposure` 

2111 Flat exposure of the same size as ``exposure``. 

2112 invert : `Bool`, optional 

2113 If True, unflatten an already flattened image. 

2114 

2115 See Also 

2116 -------- 

2117 lsst.ip.isr.isrFunctions.flatCorrection 

2118 """ 

2119 isrFunctions.flatCorrection( 

2120 maskedImage=exposure.getMaskedImage(), 

2121 flatMaskedImage=flatExposure.getMaskedImage(), 

2122 scalingType=self.config.flatScalingType, 

2123 userScale=self.config.flatUserScale, 

2124 invert=invert, 

2125 trimToFit=self.config.doTrimToMatchCalib 

2126 ) 

2127 

2128 def saturationDetection(self, exposure, amp): 

2129 """Detect and mask saturated pixels in config.saturatedMaskName. 

2130 

2131 Parameters 

2132 ---------- 

2133 exposure : `lsst.afw.image.Exposure` 

2134 Exposure to process. Only the amplifier DataSec is processed. 

2135 amp : `lsst.afw.cameraGeom.Amplifier` 

2136 Amplifier detector data. 

2137 

2138 See Also 

2139 -------- 

2140 lsst.ip.isr.isrFunctions.makeThresholdMask 

2141 """ 

2142 if not math.isnan(amp.getSaturation()): 

2143 maskedImage = exposure.getMaskedImage() 

2144 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2145 isrFunctions.makeThresholdMask( 

2146 maskedImage=dataView, 

2147 threshold=amp.getSaturation(), 

2148 growFootprints=0, 

2149 maskName=self.config.saturatedMaskName, 

2150 ) 

2151 

2152 def saturationInterpolation(self, exposure): 

2153 """Interpolate over saturated pixels, in place. 

2154 

2155 This method should be called after `saturationDetection`, to 

2156 ensure that the saturated pixels have been identified in the 

2157 SAT mask. It should also be called after `assembleCcd`, since 

2158 saturated regions may cross amplifier boundaries. 

2159 

2160 Parameters 

2161 ---------- 

2162 exposure : `lsst.afw.image.Exposure` 

2163 Exposure to process. 

2164 

2165 See Also 

2166 -------- 

2167 lsst.ip.isr.isrTask.saturationDetection 

2168 lsst.ip.isr.isrFunctions.interpolateFromMask 

2169 """ 

2170 isrFunctions.interpolateFromMask( 

2171 maskedImage=exposure.getMaskedImage(), 

2172 fwhm=self.config.fwhm, 

2173 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2174 maskNameList=list(self.config.saturatedMaskName), 

2175 ) 

2176 

2177 def suspectDetection(self, exposure, amp): 

2178 """Detect and mask suspect pixels in config.suspectMaskName. 

2179 

2180 Parameters 

2181 ---------- 

2182 exposure : `lsst.afw.image.Exposure` 

2183 Exposure to process. Only the amplifier DataSec is processed. 

2184 amp : `lsst.afw.cameraGeom.Amplifier` 

2185 Amplifier detector data. 

2186 

2187 See Also 

2188 -------- 

2189 lsst.ip.isr.isrFunctions.makeThresholdMask 

2190 

2191 Notes 

2192 ----- 

2193 Suspect pixels are pixels whose value is greater than 

2194 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2195 affected by unknown systematics; for example if non-linearity 

2196 corrections above a certain level are unstable then that would be a 

2197 useful value for suspectLevel. A value of `nan` indicates that no such 

2198 level exists and no pixels are to be masked as suspicious. 

2199 """ 

2200 suspectLevel = amp.getSuspectLevel() 

2201 if math.isnan(suspectLevel): 

2202 return 

2203 

2204 maskedImage = exposure.getMaskedImage() 

2205 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2206 isrFunctions.makeThresholdMask( 

2207 maskedImage=dataView, 

2208 threshold=suspectLevel, 

2209 growFootprints=0, 

2210 maskName=self.config.suspectMaskName, 

2211 ) 

2212 

2213 def maskDefect(self, exposure, defectBaseList): 

2214 """Mask defects using mask plane "BAD", in place. 

2215 

2216 Parameters 

2217 ---------- 

2218 exposure : `lsst.afw.image.Exposure` 

2219 Exposure to process. 

2220 defectBaseList : defect-type 

2221 List of defects to mask. Can be of type `lsst.ip.isr.Defects` 

2222 or `list` of `lsst.afw.image.DefectBase`. 

2223 

2224 Notes 

2225 ----- 

2226 Call this after CCD assembly, since defects may cross amplifier 

2227 boundaries. 

2228 """ 

2229 maskedImage = exposure.getMaskedImage() 

2230 if not isinstance(defectBaseList, Defects): 

2231 # Promotes DefectBase to Defect 

2232 defectList = Defects(defectBaseList) 

2233 else: 

2234 defectList = defectBaseList 

2235 defectList.maskPixels(maskedImage, maskName="BAD") 

2236 

2237 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2238 """Mask edge pixels with applicable mask plane. 

2239 

2240 Parameters 

2241 ---------- 

2242 exposure : `lsst.afw.image.Exposure` 

2243 Exposure to process. 

2244 numEdgePixels : `int`, optional 

2245 Number of edge pixels to mask. 

2246 maskPlane : `str`, optional 

2247 Mask plane name to use. 

2248 level : `str`, optional 

2249 Level at which to mask edges. 

2250 """ 

2251 maskedImage = exposure.getMaskedImage() 

2252 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2253 

2254 if numEdgePixels > 0: 

2255 if level == 'DETECTOR': 

2256 boxes = [maskedImage.getBBox()] 

2257 elif level == 'AMP': 

2258 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2259 

2260 for box in boxes: 

2261 # This makes a bbox numEdgeSuspect pixels smaller than the 

2262 # image on each side 

2263 subImage = maskedImage[box] 

2264 box.grow(-numEdgePixels) 

2265 # Mask pixels outside box 

2266 SourceDetectionTask.setEdgeBits( 

2267 subImage, 

2268 box, 

2269 maskBitMask) 

2270 

2271 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2272 """Mask and interpolate defects using mask plane "BAD", in place. 

2273 

2274 Parameters 

2275 ---------- 

2276 exposure : `lsst.afw.image.Exposure` 

2277 Exposure to process. 

2278 defectBaseList : defects-like 

2279 List of defects to mask and interpolate. Can be 

2280 `lsst.ip.isr.Defects` or `list` of `lsst.afw.image.DefectBase`. 

2281 

2282 See Also 

2283 -------- 

2284 lsst.ip.isr.isrTask.maskDefect 

2285 """ 

2286 self.maskDefect(exposure, defectBaseList) 

2287 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2288 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2289 isrFunctions.interpolateFromMask( 

2290 maskedImage=exposure.getMaskedImage(), 

2291 fwhm=self.config.fwhm, 

2292 growSaturatedFootprints=0, 

2293 maskNameList=["BAD"], 

2294 ) 

2295 

2296 def maskNan(self, exposure): 

2297 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2298 

2299 Parameters 

2300 ---------- 

2301 exposure : `lsst.afw.image.Exposure` 

2302 Exposure to process. 

2303 

2304 Notes 

2305 ----- 

2306 We mask over all non-finite values (NaN, inf), including those 

2307 that are masked with other bits (because those may or may not be 

2308 interpolated over later, and we want to remove all NaN/infs). 

2309 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2310 preserve the historical name. 

2311 """ 

2312 maskedImage = exposure.getMaskedImage() 

2313 

2314 # Find and mask NaNs 

2315 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2316 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2317 numNans = maskNans(maskedImage, maskVal) 

2318 self.metadata["NUMNANS"] = numNans 

2319 if numNans > 0: 

2320 self.log.warning("There were %d unmasked NaNs.", numNans) 

2321 

2322 def maskAndInterpolateNan(self, exposure): 

2323 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2324 in place. 

2325 

2326 Parameters 

2327 ---------- 

2328 exposure : `lsst.afw.image.Exposure` 

2329 Exposure to process. 

2330 

2331 See Also 

2332 -------- 

2333 lsst.ip.isr.isrTask.maskNan 

2334 """ 

2335 self.maskNan(exposure) 

2336 isrFunctions.interpolateFromMask( 

2337 maskedImage=exposure.getMaskedImage(), 

2338 fwhm=self.config.fwhm, 

2339 growSaturatedFootprints=0, 

2340 maskNameList=["UNMASKEDNAN"], 

2341 ) 

2342 

2343 def measureBackground(self, exposure, IsrQaConfig=None): 

2344 """Measure the image background in subgrids, for quality control. 

2345 

2346 Parameters 

2347 ---------- 

2348 exposure : `lsst.afw.image.Exposure` 

2349 Exposure to process. 

2350 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2351 Configuration object containing parameters on which background 

2352 statistics and subgrids to use. 

2353 """ 

2354 if IsrQaConfig is not None: 

2355 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2356 IsrQaConfig.flatness.nIter) 

2357 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2358 statsControl.setAndMask(maskVal) 

2359 maskedImage = exposure.getMaskedImage() 

2360 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2361 skyLevel = stats.getValue(afwMath.MEDIAN) 

2362 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2363 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2364 metadata = exposure.getMetadata() 

2365 metadata["SKYLEVEL"] = skyLevel 

2366 metadata["SKYSIGMA"] = skySigma 

2367 

2368 # calcluating flatlevel over the subgrids 

2369 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2370 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2371 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2372 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2373 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2374 skyLevels = numpy.zeros((nX, nY)) 

2375 

2376 for j in range(nY): 

2377 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2378 for i in range(nX): 

2379 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2380 

2381 xLLC = xc - meshXHalf 

2382 yLLC = yc - meshYHalf 

2383 xURC = xc + meshXHalf - 1 

2384 yURC = yc + meshYHalf - 1 

2385 

2386 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2387 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2388 

2389 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2390 

2391 good = numpy.where(numpy.isfinite(skyLevels)) 

2392 skyMedian = numpy.median(skyLevels[good]) 

2393 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2394 flatness_rms = numpy.std(flatness) 

2395 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2396 

2397 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2398 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2399 nX, nY, flatness_pp, flatness_rms) 

2400 

2401 metadata["FLATNESS_PP"] = float(flatness_pp) 

2402 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2403 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2404 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2405 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2406 

2407 def roughZeroPoint(self, exposure): 

2408 """Set an approximate magnitude zero point for the exposure. 

2409 

2410 Parameters 

2411 ---------- 

2412 exposure : `lsst.afw.image.Exposure` 

2413 Exposure to process. 

2414 """ 

2415 filterLabel = exposure.getFilter() 

2416 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2417 

2418 if physicalFilter in self.config.fluxMag0T1: 

2419 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2420 else: 

2421 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2422 fluxMag0 = self.config.defaultFluxMag0T1 

2423 

2424 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2425 if not expTime > 0: # handle NaN as well as <= 0 

2426 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2427 return 

2428 

2429 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2430 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2431 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2432 

2433 @contextmanager 

2434 def flatContext(self, exp, flat, dark=None): 

2435 """Context manager that applies and removes flats and darks, 

2436 if the task is configured to apply them. 

2437 

2438 Parameters 

2439 ---------- 

2440 exp : `lsst.afw.image.Exposure` 

2441 Exposure to process. 

2442 flat : `lsst.afw.image.Exposure` 

2443 Flat exposure the same size as ``exp``. 

2444 dark : `lsst.afw.image.Exposure`, optional 

2445 Dark exposure the same size as ``exp``. 

2446 

2447 Yields 

2448 ------ 

2449 exp : `lsst.afw.image.Exposure` 

2450 The flat and dark corrected exposure. 

2451 """ 

2452 if self.config.doDark and dark is not None: 

2453 self.darkCorrection(exp, dark) 

2454 if self.config.doFlat: 

2455 self.flatCorrection(exp, flat) 

2456 try: 

2457 yield exp 

2458 finally: 

2459 if self.config.doFlat: 

2460 self.flatCorrection(exp, flat, invert=True) 

2461 if self.config.doDark and dark is not None: 

2462 self.darkCorrection(exp, dark, invert=True) 

2463 

2464 def debugView(self, exposure, stepname): 

2465 """Utility function to examine ISR exposure at different stages. 

2466 

2467 Parameters 

2468 ---------- 

2469 exposure : `lsst.afw.image.Exposure` 

2470 Exposure to view. 

2471 stepname : `str` 

2472 State of processing to view. 

2473 """ 

2474 frame = getDebugFrame(self._display, stepname) 

2475 if frame: 

2476 display = getDisplay(frame) 

2477 display.scale('asinh', 'zscale') 

2478 display.mtv(exposure) 

2479 prompt = "Press Enter to continue [c]... " 

2480 while True: 

2481 ans = input(prompt).lower() 

2482 if ans in ("", "c",): 

2483 break 

2484 

2485 

2486class FakeAmp(object): 

2487 """A Detector-like object that supports returning gain and saturation level 

2488 

2489 This is used when the input exposure does not have a detector. 

2490 

2491 Parameters 

2492 ---------- 

2493 exposure : `lsst.afw.image.Exposure` 

2494 Exposure to generate a fake amplifier for. 

2495 config : `lsst.ip.isr.isrTaskConfig` 

2496 Configuration to apply to the fake amplifier. 

2497 """ 

2498 

2499 def __init__(self, exposure, config): 

2500 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2501 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2502 self._gain = config.gain 

2503 self._readNoise = config.readNoise 

2504 self._saturation = config.saturation 

2505 

2506 def getBBox(self): 

2507 return self._bbox 

2508 

2509 def getRawBBox(self): 

2510 return self._bbox 

2511 

2512 def getRawHorizontalOverscanBBox(self): 

2513 return self._RawHorizontalOverscanBBox 

2514 

2515 def getGain(self): 

2516 return self._gain 

2517 

2518 def getReadNoise(self): 

2519 return self._readNoise 

2520 

2521 def getSaturation(self): 

2522 return self._saturation 

2523 

2524 def getSuspectLevel(self): 

2525 return float("NaN")