Coverage for python/lsst/ip/isr/isrTask.py: 18%

833 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-04 02:56 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import NullLinearityType, ReadoutCorner 

36from lsst.afw.display import getDisplay 

37from lsst.meas.algorithms.detection import SourceDetectionTask 

38from lsst.utils.timer import timeMethod 

39 

40from . import isrFunctions 

41from . import isrQa 

42from . import linearize 

43from .defects import Defects 

44 

45from .assembleCcdTask import AssembleCcdTask 

46from .crosstalk import CrosstalkTask, CrosstalkCalib 

47from .fringe import FringeTask 

48from .isr import maskNans 

49from .masking import MaskingTask 

50from .overscan import OverscanCorrectionTask 

51from .straylight import StrayLightTask 

52from .vignette import VignetteTask 

53from .ampOffset import AmpOffsetTask 

54from .deferredCharge import DeferredChargeTask 

55from .isrStatistics import IsrStatisticsTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 Parameters 

70 ---------- 

71 datasetType : `str` 

72 Dataset to lookup. 

73 registry : `lsst.daf.butler.Registry` 

74 Butler registry to query. 

75 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

76 Data id to transform to identify crosstalkSources. The 

77 ``detector`` entry will be stripped. 

78 collections : `lsst.daf.butler.CollectionSearch` 

79 Collections to search through. 

80 

81 Returns 

82 ------- 

83 results : `list` [`lsst.daf.butler.DatasetRef`] 

84 List of datasets that match the query that will be used as 

85 crosstalkSources. 

86 """ 

87 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

88 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

89 findFirst=True)) 

90 # In some contexts, calling `.expanded()` to expand all data IDs in the 

91 # query results can be a lot faster because it vectorizes lookups. But in 

92 # this case, expandDataId shouldn't need to hit the database at all in the 

93 # steady state, because only the detector record is unknown and those are 

94 # cached in the registry. 

95 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

96 

97 

98class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

99 dimensions={"instrument", "exposure", "detector"}, 

100 defaultTemplates={}): 

101 ccdExposure = cT.Input( 

102 name="raw", 

103 doc="Input exposure to process.", 

104 storageClass="Exposure", 

105 dimensions=["instrument", "exposure", "detector"], 

106 ) 

107 camera = cT.PrerequisiteInput( 

108 name="camera", 

109 storageClass="Camera", 

110 doc="Input camera to construct complete exposures.", 

111 dimensions=["instrument"], 

112 isCalibration=True, 

113 ) 

114 

115 crosstalk = cT.PrerequisiteInput( 

116 name="crosstalk", 

117 doc="Input crosstalk object", 

118 storageClass="CrosstalkCalib", 

119 dimensions=["instrument", "detector"], 

120 isCalibration=True, 

121 minimum=0, # can fall back to cameraGeom 

122 ) 

123 crosstalkSources = cT.PrerequisiteInput( 

124 name="isrOverscanCorrected", 

125 doc="Overscan corrected input images.", 

126 storageClass="Exposure", 

127 dimensions=["instrument", "exposure", "detector"], 

128 deferLoad=True, 

129 multiple=True, 

130 lookupFunction=crosstalkSourceLookup, 

131 minimum=0, # not needed for all instruments, no config to control this 

132 ) 

133 bias = cT.PrerequisiteInput( 

134 name="bias", 

135 doc="Input bias calibration.", 

136 storageClass="ExposureF", 

137 dimensions=["instrument", "detector"], 

138 isCalibration=True, 

139 ) 

140 dark = cT.PrerequisiteInput( 

141 name='dark', 

142 doc="Input dark calibration.", 

143 storageClass="ExposureF", 

144 dimensions=["instrument", "detector"], 

145 isCalibration=True, 

146 ) 

147 flat = cT.PrerequisiteInput( 

148 name="flat", 

149 doc="Input flat calibration.", 

150 storageClass="ExposureF", 

151 dimensions=["instrument", "physical_filter", "detector"], 

152 isCalibration=True, 

153 ) 

154 ptc = cT.PrerequisiteInput( 

155 name="ptc", 

156 doc="Input Photon Transfer Curve dataset", 

157 storageClass="PhotonTransferCurveDataset", 

158 dimensions=["instrument", "detector"], 

159 isCalibration=True, 

160 ) 

161 fringes = cT.PrerequisiteInput( 

162 name="fringe", 

163 doc="Input fringe calibration.", 

164 storageClass="ExposureF", 

165 dimensions=["instrument", "physical_filter", "detector"], 

166 isCalibration=True, 

167 minimum=0, # only needed for some bands, even when enabled 

168 ) 

169 strayLightData = cT.PrerequisiteInput( 

170 name='yBackground', 

171 doc="Input stray light calibration.", 

172 storageClass="StrayLightData", 

173 dimensions=["instrument", "physical_filter", "detector"], 

174 deferLoad=True, 

175 isCalibration=True, 

176 minimum=0, # only needed for some bands, even when enabled 

177 ) 

178 bfKernel = cT.PrerequisiteInput( 

179 name='bfKernel', 

180 doc="Input brighter-fatter kernel.", 

181 storageClass="NumpyArray", 

182 dimensions=["instrument"], 

183 isCalibration=True, 

184 minimum=0, # can use either bfKernel or newBFKernel 

185 ) 

186 newBFKernel = cT.PrerequisiteInput( 

187 name='brighterFatterKernel', 

188 doc="Newer complete kernel + gain solutions.", 

189 storageClass="BrighterFatterKernel", 

190 dimensions=["instrument", "detector"], 

191 isCalibration=True, 

192 minimum=0, # can use either bfKernel or newBFKernel 

193 ) 

194 defects = cT.PrerequisiteInput( 

195 name='defects', 

196 doc="Input defect tables.", 

197 storageClass="Defects", 

198 dimensions=["instrument", "detector"], 

199 isCalibration=True, 

200 ) 

201 linearizer = cT.PrerequisiteInput( 

202 name='linearizer', 

203 storageClass="Linearizer", 

204 doc="Linearity correction calibration.", 

205 dimensions=["instrument", "detector"], 

206 isCalibration=True, 

207 minimum=0, # can fall back to cameraGeom 

208 ) 

209 opticsTransmission = cT.PrerequisiteInput( 

210 name="transmission_optics", 

211 storageClass="TransmissionCurve", 

212 doc="Transmission curve due to the optics.", 

213 dimensions=["instrument"], 

214 isCalibration=True, 

215 ) 

216 filterTransmission = cT.PrerequisiteInput( 

217 name="transmission_filter", 

218 storageClass="TransmissionCurve", 

219 doc="Transmission curve due to the filter.", 

220 dimensions=["instrument", "physical_filter"], 

221 isCalibration=True, 

222 ) 

223 sensorTransmission = cT.PrerequisiteInput( 

224 name="transmission_sensor", 

225 storageClass="TransmissionCurve", 

226 doc="Transmission curve due to the sensor.", 

227 dimensions=["instrument", "detector"], 

228 isCalibration=True, 

229 ) 

230 atmosphereTransmission = cT.PrerequisiteInput( 

231 name="transmission_atmosphere", 

232 storageClass="TransmissionCurve", 

233 doc="Transmission curve due to the atmosphere.", 

234 dimensions=["instrument"], 

235 isCalibration=True, 

236 ) 

237 illumMaskedImage = cT.PrerequisiteInput( 

238 name="illum", 

239 doc="Input illumination correction.", 

240 storageClass="MaskedImageF", 

241 dimensions=["instrument", "physical_filter", "detector"], 

242 isCalibration=True, 

243 ) 

244 deferredChargeCalib = cT.PrerequisiteInput( 

245 name="deferredCharge", 

246 doc="Deferred charge/CTI correction dataset.", 

247 storageClass="IsrCalib", 

248 dimensions=["instrument", "detector"], 

249 isCalibration=True, 

250 ) 

251 

252 outputExposure = cT.Output( 

253 name='postISRCCD', 

254 doc="Output ISR processed exposure.", 

255 storageClass="Exposure", 

256 dimensions=["instrument", "exposure", "detector"], 

257 ) 

258 preInterpExposure = cT.Output( 

259 name='preInterpISRCCD', 

260 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

261 storageClass="ExposureF", 

262 dimensions=["instrument", "exposure", "detector"], 

263 ) 

264 outputOssThumbnail = cT.Output( 

265 name="OssThumb", 

266 doc="Output Overscan-subtracted thumbnail image.", 

267 storageClass="Thumbnail", 

268 dimensions=["instrument", "exposure", "detector"], 

269 ) 

270 outputFlattenedThumbnail = cT.Output( 

271 name="FlattenedThumb", 

272 doc="Output flat-corrected thumbnail image.", 

273 storageClass="Thumbnail", 

274 dimensions=["instrument", "exposure", "detector"], 

275 ) 

276 outputStatistics = cT.Output( 

277 name="isrStatistics", 

278 doc="Output of additional statistics table.", 

279 storageClass="StructuredDataDict", 

280 dimensions=["instrument", "exposure", "detector"], 

281 ) 

282 

283 def __init__(self, *, config=None): 

284 super().__init__(config=config) 

285 

286 if config.doBias is not True: 

287 self.prerequisiteInputs.remove("bias") 

288 if config.doLinearize is not True: 

289 self.prerequisiteInputs.remove("linearizer") 

290 if config.doCrosstalk is not True: 

291 self.prerequisiteInputs.remove("crosstalkSources") 

292 self.prerequisiteInputs.remove("crosstalk") 

293 if config.doBrighterFatter is not True: 

294 self.prerequisiteInputs.remove("bfKernel") 

295 self.prerequisiteInputs.remove("newBFKernel") 

296 if config.doDefect is not True: 

297 self.prerequisiteInputs.remove("defects") 

298 if config.doDark is not True: 

299 self.prerequisiteInputs.remove("dark") 

300 if config.doFlat is not True: 

301 self.prerequisiteInputs.remove("flat") 

302 if config.doFringe is not True: 

303 self.prerequisiteInputs.remove("fringes") 

304 if config.doStrayLight is not True: 

305 self.prerequisiteInputs.remove("strayLightData") 

306 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

307 self.prerequisiteInputs.remove("ptc") 

308 if config.doAttachTransmissionCurve is not True: 

309 self.prerequisiteInputs.remove("opticsTransmission") 

310 self.prerequisiteInputs.remove("filterTransmission") 

311 self.prerequisiteInputs.remove("sensorTransmission") 

312 self.prerequisiteInputs.remove("atmosphereTransmission") 

313 else: 

314 if config.doUseOpticsTransmission is not True: 

315 self.prerequisiteInputs.remove("opticsTransmission") 

316 if config.doUseFilterTransmission is not True: 

317 self.prerequisiteInputs.remove("filterTransmission") 

318 if config.doUseSensorTransmission is not True: 

319 self.prerequisiteInputs.remove("sensorTransmission") 

320 if config.doUseAtmosphereTransmission is not True: 

321 self.prerequisiteInputs.remove("atmosphereTransmission") 

322 if config.doIlluminationCorrection is not True: 

323 self.prerequisiteInputs.remove("illumMaskedImage") 

324 if config.doDeferredCharge is not True: 

325 self.prerequisiteInputs.remove("deferredChargeCalib") 

326 

327 if config.doWrite is not True: 

328 self.outputs.remove("outputExposure") 

329 self.outputs.remove("preInterpExposure") 

330 self.outputs.remove("outputFlattenedThumbnail") 

331 self.outputs.remove("outputOssThumbnail") 

332 self.outputs.remove("outputStatistics") 

333 

334 if config.doSaveInterpPixels is not True: 

335 self.outputs.remove("preInterpExposure") 

336 if config.qa.doThumbnailOss is not True: 

337 self.outputs.remove("outputOssThumbnail") 

338 if config.qa.doThumbnailFlattened is not True: 

339 self.outputs.remove("outputFlattenedThumbnail") 

340 if config.doCalculateStatistics is not True: 

341 self.outputs.remove("outputStatistics") 

342 

343 

344class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

345 pipelineConnections=IsrTaskConnections): 

346 """Configuration parameters for IsrTask. 

347 

348 Items are grouped in the order in which they are executed by the task. 

349 """ 

350 datasetType = pexConfig.Field( 

351 dtype=str, 

352 doc="Dataset type for input data; users will typically leave this alone, " 

353 "but camera-specific ISR tasks will override it", 

354 default="raw", 

355 ) 

356 

357 fallbackFilterName = pexConfig.Field( 

358 dtype=str, 

359 doc="Fallback default filter name for calibrations.", 

360 optional=True 

361 ) 

362 useFallbackDate = pexConfig.Field( 

363 dtype=bool, 

364 doc="Pass observation date when using fallback filter.", 

365 default=False, 

366 ) 

367 expectWcs = pexConfig.Field( 

368 dtype=bool, 

369 default=True, 

370 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

371 ) 

372 fwhm = pexConfig.Field( 

373 dtype=float, 

374 doc="FWHM of PSF in arcseconds.", 

375 default=1.0, 

376 ) 

377 qa = pexConfig.ConfigField( 

378 dtype=isrQa.IsrQaConfig, 

379 doc="QA related configuration options.", 

380 ) 

381 

382 # Image conversion configuration 

383 doConvertIntToFloat = pexConfig.Field( 

384 dtype=bool, 

385 doc="Convert integer raw images to floating point values?", 

386 default=True, 

387 ) 

388 

389 # Saturated pixel handling. 

390 doSaturation = pexConfig.Field( 

391 dtype=bool, 

392 doc="Mask saturated pixels? NB: this is totally independent of the" 

393 " interpolation option - this is ONLY setting the bits in the mask." 

394 " To have them interpolated make sure doSaturationInterpolation=True", 

395 default=True, 

396 ) 

397 saturatedMaskName = pexConfig.Field( 

398 dtype=str, 

399 doc="Name of mask plane to use in saturation detection and interpolation", 

400 default="SAT", 

401 ) 

402 saturation = pexConfig.Field( 

403 dtype=float, 

404 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

405 default=float("NaN"), 

406 ) 

407 growSaturationFootprintSize = pexConfig.Field( 

408 dtype=int, 

409 doc="Number of pixels by which to grow the saturation footprints", 

410 default=1, 

411 ) 

412 

413 # Suspect pixel handling. 

414 doSuspect = pexConfig.Field( 

415 dtype=bool, 

416 doc="Mask suspect pixels?", 

417 default=False, 

418 ) 

419 suspectMaskName = pexConfig.Field( 

420 dtype=str, 

421 doc="Name of mask plane to use for suspect pixels", 

422 default="SUSPECT", 

423 ) 

424 numEdgeSuspect = pexConfig.Field( 

425 dtype=int, 

426 doc="Number of edge pixels to be flagged as untrustworthy.", 

427 default=0, 

428 ) 

429 edgeMaskLevel = pexConfig.ChoiceField( 

430 dtype=str, 

431 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

432 default="DETECTOR", 

433 allowed={ 

434 'DETECTOR': 'Mask only the edges of the full detector.', 

435 'AMP': 'Mask edges of each amplifier.', 

436 }, 

437 ) 

438 

439 # Initial masking options. 

440 doSetBadRegions = pexConfig.Field( 

441 dtype=bool, 

442 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

443 default=True, 

444 ) 

445 badStatistic = pexConfig.ChoiceField( 

446 dtype=str, 

447 doc="How to estimate the average value for BAD regions.", 

448 default='MEANCLIP', 

449 allowed={ 

450 "MEANCLIP": "Correct using the (clipped) mean of good data", 

451 "MEDIAN": "Correct using the median of the good data", 

452 }, 

453 ) 

454 

455 # Overscan subtraction configuration. 

456 doOverscan = pexConfig.Field( 

457 dtype=bool, 

458 doc="Do overscan subtraction?", 

459 default=True, 

460 ) 

461 overscan = pexConfig.ConfigurableField( 

462 target=OverscanCorrectionTask, 

463 doc="Overscan subtraction task for image segments.", 

464 ) 

465 overscanFitType = pexConfig.ChoiceField( 

466 dtype=str, 

467 doc="The method for fitting the overscan bias level.", 

468 default='MEDIAN', 

469 allowed={ 

470 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

471 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

472 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

473 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

474 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

475 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

476 "MEAN": "Correct using the mean of the overscan region", 

477 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

478 "MEDIAN": "Correct using the median of the overscan region", 

479 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

480 }, 

481 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

482 " This option will no longer be used, and will be removed after v20.") 

483 ) 

484 overscanOrder = pexConfig.Field( 

485 dtype=int, 

486 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

487 "or number of spline knots if overscan fit type is a spline."), 

488 default=1, 

489 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

490 " This option will no longer be used, and will be removed after v20.") 

491 ) 

492 overscanNumSigmaClip = pexConfig.Field( 

493 dtype=float, 

494 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

495 default=3.0, 

496 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

497 " This option will no longer be used, and will be removed after v20.") 

498 ) 

499 overscanIsInt = pexConfig.Field( 

500 dtype=bool, 

501 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

502 " and overscan.FitType=MEDIAN_PER_ROW.", 

503 default=True, 

504 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

505 " This option will no longer be used, and will be removed after v20.") 

506 ) 

507 # These options do not get deprecated, as they define how we slice up the 

508 # image data. 

509 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

510 dtype=int, 

511 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

512 default=0, 

513 ) 

514 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

515 dtype=int, 

516 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

517 default=0, 

518 ) 

519 overscanMaxDev = pexConfig.Field( 519 ↛ exitline 519 didn't jump to the function exit

520 dtype=float, 

521 doc="Maximum deviation from the median for overscan", 

522 default=1000.0, check=lambda x: x > 0 

523 ) 

524 overscanBiasJump = pexConfig.Field( 

525 dtype=bool, 

526 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

527 default=False, 

528 ) 

529 overscanBiasJumpKeyword = pexConfig.Field( 

530 dtype=str, 

531 doc="Header keyword containing information about devices.", 

532 default="NO_SUCH_KEY", 

533 ) 

534 overscanBiasJumpDevices = pexConfig.ListField( 

535 dtype=str, 

536 doc="List of devices that need piecewise overscan correction.", 

537 default=(), 

538 ) 

539 overscanBiasJumpLocation = pexConfig.Field( 

540 dtype=int, 

541 doc="Location of bias jump along y-axis.", 

542 default=0, 

543 ) 

544 

545 # Amplifier to CCD assembly configuration 

546 doAssembleCcd = pexConfig.Field( 

547 dtype=bool, 

548 default=True, 

549 doc="Assemble amp-level exposures into a ccd-level exposure?" 

550 ) 

551 assembleCcd = pexConfig.ConfigurableField( 

552 target=AssembleCcdTask, 

553 doc="CCD assembly task", 

554 ) 

555 

556 # General calibration configuration. 

557 doAssembleIsrExposures = pexConfig.Field( 

558 dtype=bool, 

559 default=False, 

560 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

561 ) 

562 doTrimToMatchCalib = pexConfig.Field( 

563 dtype=bool, 

564 default=False, 

565 doc="Trim raw data to match calibration bounding boxes?" 

566 ) 

567 

568 # Bias subtraction. 

569 doBias = pexConfig.Field( 

570 dtype=bool, 

571 doc="Apply bias frame correction?", 

572 default=True, 

573 ) 

574 biasDataProductName = pexConfig.Field( 

575 dtype=str, 

576 doc="Name of the bias data product", 

577 default="bias", 

578 ) 

579 doBiasBeforeOverscan = pexConfig.Field( 

580 dtype=bool, 

581 doc="Reverse order of overscan and bias correction.", 

582 default=False 

583 ) 

584 

585 # Deferred charge correction. 

586 doDeferredCharge = pexConfig.Field( 

587 dtype=bool, 

588 doc="Apply deferred charge correction?", 

589 default=False, 

590 ) 

591 deferredChargeCorrection = pexConfig.ConfigurableField( 

592 target=DeferredChargeTask, 

593 doc="Deferred charge correction task.", 

594 ) 

595 

596 # Variance construction 

597 doVariance = pexConfig.Field( 

598 dtype=bool, 

599 doc="Calculate variance?", 

600 default=True 

601 ) 

602 gain = pexConfig.Field( 

603 dtype=float, 

604 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

605 default=float("NaN"), 

606 ) 

607 readNoise = pexConfig.Field( 

608 dtype=float, 

609 doc="The read noise to use if no Detector is present in the Exposure", 

610 default=0.0, 

611 ) 

612 doEmpiricalReadNoise = pexConfig.Field( 

613 dtype=bool, 

614 default=False, 

615 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

616 ) 

617 usePtcReadNoise = pexConfig.Field( 

618 dtype=bool, 

619 default=False, 

620 doc="Use readnoise values from the Photon Transfer Curve?" 

621 ) 

622 maskNegativeVariance = pexConfig.Field( 

623 dtype=bool, 

624 default=True, 

625 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

626 "in the measurement of the overscan at an edge due to the data falling off faster " 

627 "than the overscan model can account for it." 

628 ) 

629 negativeVarianceMaskName = pexConfig.Field( 

630 dtype=str, 

631 default="BAD", 

632 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

633 ) 

634 # Linearization. 

635 doLinearize = pexConfig.Field( 

636 dtype=bool, 

637 doc="Correct for nonlinearity of the detector's response?", 

638 default=True, 

639 ) 

640 

641 # Crosstalk. 

642 doCrosstalk = pexConfig.Field( 

643 dtype=bool, 

644 doc="Apply intra-CCD crosstalk correction?", 

645 default=False, 

646 ) 

647 doCrosstalkBeforeAssemble = pexConfig.Field( 

648 dtype=bool, 

649 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

650 default=False, 

651 ) 

652 crosstalk = pexConfig.ConfigurableField( 

653 target=CrosstalkTask, 

654 doc="Intra-CCD crosstalk correction", 

655 ) 

656 

657 # Masking options. 

658 doDefect = pexConfig.Field( 

659 dtype=bool, 

660 doc="Apply correction for CCD defects, e.g. hot pixels?", 

661 default=True, 

662 ) 

663 doNanMasking = pexConfig.Field( 

664 dtype=bool, 

665 doc="Mask non-finite (NAN, inf) pixels?", 

666 default=True, 

667 ) 

668 doWidenSaturationTrails = pexConfig.Field( 

669 dtype=bool, 

670 doc="Widen bleed trails based on their width?", 

671 default=True 

672 ) 

673 

674 # Brighter-Fatter correction. 

675 doBrighterFatter = pexConfig.Field( 

676 dtype=bool, 

677 default=False, 

678 doc="Apply the brighter-fatter correction?" 

679 ) 

680 brighterFatterLevel = pexConfig.ChoiceField( 

681 dtype=str, 

682 default="DETECTOR", 

683 doc="The level at which to correct for brighter-fatter.", 

684 allowed={ 

685 "AMP": "Every amplifier treated separately.", 

686 "DETECTOR": "One kernel per detector", 

687 } 

688 ) 

689 brighterFatterMaxIter = pexConfig.Field( 

690 dtype=int, 

691 default=10, 

692 doc="Maximum number of iterations for the brighter-fatter correction" 

693 ) 

694 brighterFatterThreshold = pexConfig.Field( 

695 dtype=float, 

696 default=1000, 

697 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

698 "absolute value of the difference between the current corrected image and the one " 

699 "from the previous iteration summed over all the pixels." 

700 ) 

701 brighterFatterApplyGain = pexConfig.Field( 

702 dtype=bool, 

703 default=True, 

704 doc="Should the gain be applied when applying the brighter-fatter correction?" 

705 ) 

706 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

707 dtype=str, 

708 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

709 "correction.", 

710 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

711 ) 

712 brighterFatterMaskGrowSize = pexConfig.Field( 

713 dtype=int, 

714 default=0, 

715 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

716 "when brighter-fatter correction is applied." 

717 ) 

718 

719 # Dark subtraction. 

720 doDark = pexConfig.Field( 

721 dtype=bool, 

722 doc="Apply dark frame correction?", 

723 default=True, 

724 ) 

725 darkDataProductName = pexConfig.Field( 

726 dtype=str, 

727 doc="Name of the dark data product", 

728 default="dark", 

729 ) 

730 

731 # Camera-specific stray light removal. 

732 doStrayLight = pexConfig.Field( 

733 dtype=bool, 

734 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

735 default=False, 

736 ) 

737 strayLight = pexConfig.ConfigurableField( 

738 target=StrayLightTask, 

739 doc="y-band stray light correction" 

740 ) 

741 

742 # Flat correction. 

743 doFlat = pexConfig.Field( 

744 dtype=bool, 

745 doc="Apply flat field correction?", 

746 default=True, 

747 ) 

748 flatDataProductName = pexConfig.Field( 

749 dtype=str, 

750 doc="Name of the flat data product", 

751 default="flat", 

752 ) 

753 flatScalingType = pexConfig.ChoiceField( 

754 dtype=str, 

755 doc="The method for scaling the flat on the fly.", 

756 default='USER', 

757 allowed={ 

758 "USER": "Scale by flatUserScale", 

759 "MEAN": "Scale by the inverse of the mean", 

760 "MEDIAN": "Scale by the inverse of the median", 

761 }, 

762 ) 

763 flatUserScale = pexConfig.Field( 

764 dtype=float, 

765 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

766 default=1.0, 

767 ) 

768 doTweakFlat = pexConfig.Field( 

769 dtype=bool, 

770 doc="Tweak flats to match observed amplifier ratios?", 

771 default=False 

772 ) 

773 

774 # Amplifier normalization based on gains instead of using flats 

775 # configuration. 

776 doApplyGains = pexConfig.Field( 

777 dtype=bool, 

778 doc="Correct the amplifiers for their gains instead of applying flat correction", 

779 default=False, 

780 ) 

781 usePtcGains = pexConfig.Field( 

782 dtype=bool, 

783 doc="Use the gain values from the Photon Transfer Curve?", 

784 default=False, 

785 ) 

786 normalizeGains = pexConfig.Field( 

787 dtype=bool, 

788 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

789 default=False, 

790 ) 

791 

792 # Fringe correction. 

793 doFringe = pexConfig.Field( 

794 dtype=bool, 

795 doc="Apply fringe correction?", 

796 default=True, 

797 ) 

798 fringe = pexConfig.ConfigurableField( 

799 target=FringeTask, 

800 doc="Fringe subtraction task", 

801 ) 

802 fringeAfterFlat = pexConfig.Field( 

803 dtype=bool, 

804 doc="Do fringe subtraction after flat-fielding?", 

805 default=True, 

806 ) 

807 

808 # Amp offset correction. 

809 doAmpOffset = pexConfig.Field( 

810 doc="Calculate and apply amp offset corrections?", 

811 dtype=bool, 

812 default=False, 

813 ) 

814 ampOffset = pexConfig.ConfigurableField( 

815 doc="Amp offset correction task.", 

816 target=AmpOffsetTask, 

817 ) 

818 

819 # Initial CCD-level background statistics options. 

820 doMeasureBackground = pexConfig.Field( 

821 dtype=bool, 

822 doc="Measure the background level on the reduced image?", 

823 default=False, 

824 ) 

825 

826 # Camera-specific masking configuration. 

827 doCameraSpecificMasking = pexConfig.Field( 

828 dtype=bool, 

829 doc="Mask camera-specific bad regions?", 

830 default=False, 

831 ) 

832 masking = pexConfig.ConfigurableField( 

833 target=MaskingTask, 

834 doc="Masking task." 

835 ) 

836 

837 # Interpolation options. 

838 doInterpolate = pexConfig.Field( 

839 dtype=bool, 

840 doc="Interpolate masked pixels?", 

841 default=True, 

842 ) 

843 doSaturationInterpolation = pexConfig.Field( 

844 dtype=bool, 

845 doc="Perform interpolation over pixels masked as saturated?" 

846 " NB: This is independent of doSaturation; if that is False this plane" 

847 " will likely be blank, resulting in a no-op here.", 

848 default=True, 

849 ) 

850 doNanInterpolation = pexConfig.Field( 

851 dtype=bool, 

852 doc="Perform interpolation over pixels masked as NaN?" 

853 " NB: This is independent of doNanMasking; if that is False this plane" 

854 " will likely be blank, resulting in a no-op here.", 

855 default=True, 

856 ) 

857 doNanInterpAfterFlat = pexConfig.Field( 

858 dtype=bool, 

859 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

860 "also have to interpolate them before flat-fielding."), 

861 default=False, 

862 ) 

863 maskListToInterpolate = pexConfig.ListField( 

864 dtype=str, 

865 doc="List of mask planes that should be interpolated.", 

866 default=['SAT', 'BAD'], 

867 ) 

868 doSaveInterpPixels = pexConfig.Field( 

869 dtype=bool, 

870 doc="Save a copy of the pre-interpolated pixel values?", 

871 default=False, 

872 ) 

873 

874 # Default photometric calibration options. 

875 fluxMag0T1 = pexConfig.DictField( 

876 keytype=str, 

877 itemtype=float, 

878 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

879 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

880 )) 

881 ) 

882 defaultFluxMag0T1 = pexConfig.Field( 

883 dtype=float, 

884 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

885 default=pow(10.0, 0.4*28.0) 

886 ) 

887 

888 # Vignette correction configuration. 

889 doVignette = pexConfig.Field( 

890 dtype=bool, 

891 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

892 "according to vignetting parameters?"), 

893 default=False, 

894 ) 

895 doMaskVignettePolygon = pexConfig.Field( 

896 dtype=bool, 

897 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

898 "is False"), 

899 default=True, 

900 ) 

901 vignetteValue = pexConfig.Field( 

902 dtype=float, 

903 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

904 optional=True, 

905 default=None, 

906 ) 

907 vignette = pexConfig.ConfigurableField( 

908 target=VignetteTask, 

909 doc="Vignetting task.", 

910 ) 

911 

912 # Transmission curve configuration. 

913 doAttachTransmissionCurve = pexConfig.Field( 

914 dtype=bool, 

915 default=False, 

916 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

917 ) 

918 doUseOpticsTransmission = pexConfig.Field( 

919 dtype=bool, 

920 default=True, 

921 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

922 ) 

923 doUseFilterTransmission = pexConfig.Field( 

924 dtype=bool, 

925 default=True, 

926 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

927 ) 

928 doUseSensorTransmission = pexConfig.Field( 

929 dtype=bool, 

930 default=True, 

931 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

932 ) 

933 doUseAtmosphereTransmission = pexConfig.Field( 

934 dtype=bool, 

935 default=True, 

936 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

937 ) 

938 

939 # Illumination correction. 

940 doIlluminationCorrection = pexConfig.Field( 

941 dtype=bool, 

942 default=False, 

943 doc="Perform illumination correction?" 

944 ) 

945 illuminationCorrectionDataProductName = pexConfig.Field( 

946 dtype=str, 

947 doc="Name of the illumination correction data product.", 

948 default="illumcor", 

949 ) 

950 illumScale = pexConfig.Field( 

951 dtype=float, 

952 doc="Scale factor for the illumination correction.", 

953 default=1.0, 

954 ) 

955 illumFilters = pexConfig.ListField( 

956 dtype=str, 

957 default=[], 

958 doc="Only perform illumination correction for these filters." 

959 ) 

960 

961 # Calculate additional statistics? 

962 doCalculateStatistics = pexConfig.Field( 

963 dtype=bool, 

964 doc="Should additional ISR statistics be calculated?", 

965 default=False, 

966 ) 

967 isrStats = pexConfig.ConfigurableField( 

968 target=IsrStatisticsTask, 

969 doc="Task to calculate additional statistics.", 

970 ) 

971 

972 # Write the outputs to disk. If ISR is run as a subtask, this may not 

973 # be needed. 

974 doWrite = pexConfig.Field( 

975 dtype=bool, 

976 doc="Persist postISRCCD?", 

977 default=True, 

978 ) 

979 

980 def validate(self): 

981 super().validate() 

982 if self.doFlat and self.doApplyGains: 

983 raise ValueError("You may not specify both doFlat and doApplyGains") 

984 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

985 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

986 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

987 self.maskListToInterpolate.append(self.saturatedMaskName) 

988 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

989 self.maskListToInterpolate.remove(self.saturatedMaskName) 

990 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

991 self.maskListToInterpolate.append("UNMASKEDNAN") 

992 

993 

994class IsrTask(pipeBase.PipelineTask): 

995 """Apply common instrument signature correction algorithms to a raw frame. 

996 

997 The process for correcting imaging data is very similar from 

998 camera to camera. This task provides a vanilla implementation of 

999 doing these corrections, including the ability to turn certain 

1000 corrections off if they are not needed. The inputs to the primary 

1001 method, `run()`, are a raw exposure to be corrected and the 

1002 calibration data products. The raw input is a single chip sized 

1003 mosaic of all amps including overscans and other non-science 

1004 pixels. 

1005 

1006 The __init__ method sets up the subtasks for ISR processing, using 

1007 the defaults from `lsst.ip.isr`. 

1008 

1009 Parameters 

1010 ---------- 

1011 args : `list` 

1012 Positional arguments passed to the Task constructor. 

1013 None used at this time. 

1014 kwargs : `dict`, optional 

1015 Keyword arguments passed on to the Task constructor. 

1016 None used at this time. 

1017 """ 

1018 ConfigClass = IsrTaskConfig 

1019 _DefaultName = "isr" 

1020 

1021 def __init__(self, **kwargs): 

1022 super().__init__(**kwargs) 

1023 self.makeSubtask("assembleCcd") 

1024 self.makeSubtask("crosstalk") 

1025 self.makeSubtask("strayLight") 

1026 self.makeSubtask("fringe") 

1027 self.makeSubtask("masking") 

1028 self.makeSubtask("overscan") 

1029 self.makeSubtask("vignette") 

1030 self.makeSubtask("ampOffset") 

1031 self.makeSubtask("deferredChargeCorrection") 

1032 self.makeSubtask("isrStats") 

1033 

1034 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1035 inputs = butlerQC.get(inputRefs) 

1036 

1037 try: 

1038 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

1039 except Exception as e: 

1040 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

1041 (inputRefs, e)) 

1042 

1043 detector = inputs['ccdExposure'].getDetector() 

1044 

1045 if self.config.doCrosstalk is True: 

1046 # Crosstalk sources need to be defined by the pipeline 

1047 # yaml if they exist. 

1048 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

1049 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

1050 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

1051 else: 

1052 coeffVector = (self.config.crosstalk.crosstalkValues 

1053 if self.config.crosstalk.useConfigCoefficients else None) 

1054 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1055 inputs['crosstalk'] = crosstalkCalib 

1056 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1057 if 'crosstalkSources' not in inputs: 

1058 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1059 

1060 if self.doLinearize(detector) is True: 

1061 if 'linearizer' in inputs: 

1062 if isinstance(inputs['linearizer'], dict): 

1063 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1064 linearizer.fromYaml(inputs['linearizer']) 

1065 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1066 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1067 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1068 detector=detector, 

1069 log=self.log) 

1070 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1071 else: 

1072 linearizer = inputs['linearizer'] 

1073 linearizer.log = self.log 

1074 inputs['linearizer'] = linearizer 

1075 else: 

1076 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1077 self.log.warning("Constructing linearizer from cameraGeom information.") 

1078 

1079 if self.config.doDefect is True: 

1080 if "defects" in inputs and inputs['defects'] is not None: 

1081 # defects is loaded as a BaseCatalog with columns 

1082 # x0, y0, width, height. Masking expects a list of defects 

1083 # defined by their bounding box 

1084 if not isinstance(inputs["defects"], Defects): 

1085 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1086 

1087 # Load the correct style of brighter-fatter kernel, and repack 

1088 # the information as a numpy array. 

1089 if self.config.doBrighterFatter: 

1090 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1091 if brighterFatterKernel is None: 

1092 brighterFatterKernel = inputs.get('bfKernel', None) 

1093 

1094 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1095 # This is a ISR calib kernel 

1096 detName = detector.getName() 

1097 level = brighterFatterKernel.level 

1098 

1099 # This is expected to be a dictionary of amp-wise gains. 

1100 inputs['bfGains'] = brighterFatterKernel.gain 

1101 if self.config.brighterFatterLevel == 'DETECTOR': 

1102 if level == 'DETECTOR': 

1103 if detName in brighterFatterKernel.detKernels: 

1104 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1105 else: 

1106 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1107 elif level == 'AMP': 

1108 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1109 "fatter kernels.") 

1110 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1111 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1112 elif self.config.brighterFatterLevel == 'AMP': 

1113 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1114 

1115 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1116 expId = inputs['ccdExposure'].info.id 

1117 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1118 expId=expId, 

1119 assembler=self.assembleCcd 

1120 if self.config.doAssembleIsrExposures else None) 

1121 else: 

1122 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1123 

1124 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1125 if 'strayLightData' not in inputs: 

1126 inputs['strayLightData'] = None 

1127 

1128 outputs = self.run(**inputs) 

1129 butlerQC.put(outputs, outputRefs) 

1130 

1131 @timeMethod 

1132 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1133 crosstalk=None, crosstalkSources=None, 

1134 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1135 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1136 sensorTransmission=None, atmosphereTransmission=None, 

1137 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1138 deferredCharge=None, 

1139 ): 

1140 """Perform instrument signature removal on an exposure. 

1141 

1142 Steps included in the ISR processing, in order performed, are: 

1143 - saturation and suspect pixel masking 

1144 - overscan subtraction 

1145 - CCD assembly of individual amplifiers 

1146 - bias subtraction 

1147 - variance image construction 

1148 - linearization of non-linear response 

1149 - crosstalk masking 

1150 - brighter-fatter correction 

1151 - dark subtraction 

1152 - fringe correction 

1153 - stray light subtraction 

1154 - flat correction 

1155 - masking of known defects and camera specific features 

1156 - vignette calculation 

1157 - appending transmission curve and distortion model 

1158 

1159 Parameters 

1160 ---------- 

1161 ccdExposure : `lsst.afw.image.Exposure` 

1162 The raw exposure that is to be run through ISR. The 

1163 exposure is modified by this method. 

1164 camera : `lsst.afw.cameraGeom.Camera`, optional 

1165 The camera geometry for this exposure. Required if 

1166 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1167 ``flat`` does not have an associated detector. 

1168 bias : `lsst.afw.image.Exposure`, optional 

1169 Bias calibration frame. 

1170 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1171 Functor for linearization. 

1172 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1173 Calibration for crosstalk. 

1174 crosstalkSources : `list`, optional 

1175 List of possible crosstalk sources. 

1176 dark : `lsst.afw.image.Exposure`, optional 

1177 Dark calibration frame. 

1178 flat : `lsst.afw.image.Exposure`, optional 

1179 Flat calibration frame. 

1180 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1181 Photon transfer curve dataset, with, e.g., gains 

1182 and read noise. 

1183 bfKernel : `numpy.ndarray`, optional 

1184 Brighter-fatter kernel. 

1185 bfGains : `dict` of `float`, optional 

1186 Gains used to override the detector's nominal gains for the 

1187 brighter-fatter correction. A dict keyed by amplifier name for 

1188 the detector in question. 

1189 defects : `lsst.ip.isr.Defects`, optional 

1190 List of defects. 

1191 fringes : `lsst.pipe.base.Struct`, optional 

1192 Struct containing the fringe correction data, with 

1193 elements: 

1194 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1195 - ``seed``: random seed derived from the ccdExposureId for random 

1196 number generator (`uint32`) 

1197 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1198 A ``TransmissionCurve`` that represents the throughput of the, 

1199 optics, to be evaluated in focal-plane coordinates. 

1200 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1201 A ``TransmissionCurve`` that represents the throughput of the 

1202 filter itself, to be evaluated in focal-plane coordinates. 

1203 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1204 A ``TransmissionCurve`` that represents the throughput of the 

1205 sensor itself, to be evaluated in post-assembly trimmed detector 

1206 coordinates. 

1207 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1208 A ``TransmissionCurve`` that represents the throughput of the 

1209 atmosphere, assumed to be spatially constant. 

1210 detectorNum : `int`, optional 

1211 The integer number for the detector to process. 

1212 strayLightData : `object`, optional 

1213 Opaque object containing calibration information for stray-light 

1214 correction. If `None`, no correction will be performed. 

1215 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1216 Illumination correction image. 

1217 

1218 Returns 

1219 ------- 

1220 result : `lsst.pipe.base.Struct` 

1221 Result struct with component: 

1222 - ``exposure`` : `afw.image.Exposure` 

1223 The fully ISR corrected exposure. 

1224 - ``outputExposure`` : `afw.image.Exposure` 

1225 An alias for `exposure` 

1226 - ``ossThumb`` : `numpy.ndarray` 

1227 Thumbnail image of the exposure after overscan subtraction. 

1228 - ``flattenedThumb`` : `numpy.ndarray` 

1229 Thumbnail image of the exposure after flat-field correction. 

1230 - ``outputStatistics`` : `` 

1231 Values of the additional statistics calculated. 

1232 

1233 Raises 

1234 ------ 

1235 RuntimeError 

1236 Raised if a configuration option is set to True, but the 

1237 required calibration data has not been specified. 

1238 

1239 Notes 

1240 ----- 

1241 The current processed exposure can be viewed by setting the 

1242 appropriate lsstDebug entries in the `debug.display` 

1243 dictionary. The names of these entries correspond to some of 

1244 the IsrTaskConfig Boolean options, with the value denoting the 

1245 frame to use. The exposure is shown inside the matching 

1246 option check and after the processing of that step has 

1247 finished. The steps with debug points are: 

1248 

1249 doAssembleCcd 

1250 doBias 

1251 doCrosstalk 

1252 doBrighterFatter 

1253 doDark 

1254 doFringe 

1255 doStrayLight 

1256 doFlat 

1257 

1258 In addition, setting the "postISRCCD" entry displays the 

1259 exposure after all ISR processing has finished. 

1260 

1261 """ 

1262 

1263 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1264 bias = self.ensureExposure(bias, camera, detectorNum) 

1265 dark = self.ensureExposure(dark, camera, detectorNum) 

1266 flat = self.ensureExposure(flat, camera, detectorNum) 

1267 

1268 ccd = ccdExposure.getDetector() 

1269 filterLabel = ccdExposure.getFilter() 

1270 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1271 

1272 if not ccd: 

1273 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1274 ccd = [FakeAmp(ccdExposure, self.config)] 

1275 

1276 # Validate Input 

1277 if self.config.doBias and bias is None: 

1278 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1279 if self.doLinearize(ccd) and linearizer is None: 

1280 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1281 if self.config.doBrighterFatter and bfKernel is None: 

1282 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1283 if self.config.doDark and dark is None: 

1284 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1285 if self.config.doFlat and flat is None: 

1286 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1287 if self.config.doDefect and defects is None: 

1288 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1289 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1290 and fringes.fringes is None): 

1291 # The `fringes` object needs to be a pipeBase.Struct, as 

1292 # we use it as a `dict` for the parameters of 

1293 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1294 # not be `None` if `doFringe=True`. Otherwise, raise. 

1295 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1296 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1297 and illumMaskedImage is None): 

1298 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1299 if (self.config.doDeferredCharge and deferredCharge is None): 

1300 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1301 

1302 # Begin ISR processing. 

1303 if self.config.doConvertIntToFloat: 

1304 self.log.info("Converting exposure to floating point values.") 

1305 ccdExposure = self.convertIntToFloat(ccdExposure) 

1306 

1307 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1308 self.log.info("Applying bias correction.") 

1309 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1310 trimToFit=self.config.doTrimToMatchCalib) 

1311 self.debugView(ccdExposure, "doBias") 

1312 

1313 # Amplifier level processing. 

1314 overscans = [] 

1315 for amp in ccd: 

1316 # if ccdExposure is one amp, 

1317 # check for coverage to prevent performing ops multiple times 

1318 if ccdExposure.getBBox().contains(amp.getBBox()): 

1319 # Check for fully masked bad amplifiers, 

1320 # and generate masks for SUSPECT and SATURATED values. 

1321 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1322 

1323 if self.config.doOverscan and not badAmp: 

1324 # Overscan correction on amp-by-amp basis. 

1325 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1326 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1327 if overscanResults is not None and \ 

1328 self.config.qa is not None and self.config.qa.saveStats is True: 

1329 if isinstance(overscanResults.overscanFit, float): 

1330 qaMedian = overscanResults.overscanFit 

1331 qaStdev = float("NaN") 

1332 else: 

1333 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1334 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1335 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1336 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1337 

1338 self.metadata[f"FIT MEDIAN {amp.getName()}"] = qaMedian 

1339 self.metadata[f"FIT STDEV {amp.getName()}"] = qaStdev 

1340 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1341 amp.getName(), qaMedian, qaStdev) 

1342 

1343 # Residuals after overscan correction 

1344 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1345 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1346 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1347 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1348 

1349 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter 

1350 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter 

1351 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1352 amp.getName(), qaMedianAfter, qaStdevAfter) 

1353 

1354 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1355 else: 

1356 if badAmp: 

1357 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1358 overscanResults = None 

1359 

1360 overscans.append(overscanResults if overscanResults is not None else None) 

1361 else: 

1362 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1363 

1364 if self.config.doDeferredCharge: 

1365 self.log.info("Applying deferred charge/CTI correction.") 

1366 self.deferredChargeCorrection.run(ccdExposure, deferredCharge) 

1367 self.debugView(ccdExposure, "doDeferredCharge") 

1368 

1369 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1370 self.log.info("Applying crosstalk correction.") 

1371 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1372 crosstalkSources=crosstalkSources, camera=camera) 

1373 self.debugView(ccdExposure, "doCrosstalk") 

1374 

1375 if self.config.doAssembleCcd: 

1376 self.log.info("Assembling CCD from amplifiers.") 

1377 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1378 

1379 if self.config.expectWcs and not ccdExposure.getWcs(): 

1380 self.log.warning("No WCS found in input exposure.") 

1381 self.debugView(ccdExposure, "doAssembleCcd") 

1382 

1383 ossThumb = None 

1384 if self.config.qa.doThumbnailOss: 

1385 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1386 

1387 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1388 self.log.info("Applying bias correction.") 

1389 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1390 trimToFit=self.config.doTrimToMatchCalib) 

1391 self.debugView(ccdExposure, "doBias") 

1392 

1393 if self.config.doVariance: 

1394 for amp, overscanResults in zip(ccd, overscans): 

1395 if ccdExposure.getBBox().contains(amp.getBBox()): 

1396 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1397 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1398 if overscanResults is not None: 

1399 self.updateVariance(ampExposure, amp, 

1400 overscanImage=overscanResults.overscanImage, 

1401 ptcDataset=ptc) 

1402 else: 

1403 self.updateVariance(ampExposure, amp, 

1404 overscanImage=None, 

1405 ptcDataset=ptc) 

1406 if self.config.qa is not None and self.config.qa.saveStats is True: 

1407 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1408 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1409 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1410 qaStats.getValue(afwMath.MEDIAN) 

1411 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1412 qaStats.getValue(afwMath.STDEVCLIP) 

1413 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1414 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1415 qaStats.getValue(afwMath.STDEVCLIP)) 

1416 if self.config.maskNegativeVariance: 

1417 self.maskNegativeVariance(ccdExposure) 

1418 

1419 if self.doLinearize(ccd): 

1420 self.log.info("Applying linearizer.") 

1421 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1422 detector=ccd, log=self.log) 

1423 

1424 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1425 self.log.info("Applying crosstalk correction.") 

1426 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1427 crosstalkSources=crosstalkSources, isTrimmed=True) 

1428 self.debugView(ccdExposure, "doCrosstalk") 

1429 

1430 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1431 # widen trails, and do anything else the camera needs. Saturated and 

1432 # suspect pixels have already been masked. 

1433 if self.config.doDefect: 

1434 self.log.info("Masking defects.") 

1435 self.maskDefect(ccdExposure, defects) 

1436 

1437 if self.config.numEdgeSuspect > 0: 

1438 self.log.info("Masking edges as SUSPECT.") 

1439 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1440 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1441 

1442 if self.config.doNanMasking: 

1443 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1444 self.maskNan(ccdExposure) 

1445 

1446 if self.config.doWidenSaturationTrails: 

1447 self.log.info("Widening saturation trails.") 

1448 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1449 

1450 if self.config.doCameraSpecificMasking: 

1451 self.log.info("Masking regions for camera specific reasons.") 

1452 self.masking.run(ccdExposure) 

1453 

1454 if self.config.doBrighterFatter: 

1455 # We need to apply flats and darks before we can interpolate, and 

1456 # we need to interpolate before we do B-F, but we do B-F without 

1457 # the flats and darks applied so we can work in units of electrons 

1458 # or holes. This context manager applies and then removes the darks 

1459 # and flats. 

1460 # 

1461 # We also do not want to interpolate values here, so operate on 

1462 # temporary images so we can apply only the BF-correction and roll 

1463 # back the interpolation. 

1464 interpExp = ccdExposure.clone() 

1465 with self.flatContext(interpExp, flat, dark): 

1466 isrFunctions.interpolateFromMask( 

1467 maskedImage=interpExp.getMaskedImage(), 

1468 fwhm=self.config.fwhm, 

1469 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1470 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1471 ) 

1472 bfExp = interpExp.clone() 

1473 

1474 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1475 type(bfKernel), type(bfGains)) 

1476 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1477 self.config.brighterFatterMaxIter, 

1478 self.config.brighterFatterThreshold, 

1479 self.config.brighterFatterApplyGain, 

1480 bfGains) 

1481 if bfResults[1] == self.config.brighterFatterMaxIter: 

1482 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1483 bfResults[0]) 

1484 else: 

1485 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1486 bfResults[1]) 

1487 image = ccdExposure.getMaskedImage().getImage() 

1488 bfCorr = bfExp.getMaskedImage().getImage() 

1489 bfCorr -= interpExp.getMaskedImage().getImage() 

1490 image += bfCorr 

1491 

1492 # Applying the brighter-fatter correction applies a 

1493 # convolution to the science image. At the edges this 

1494 # convolution may not have sufficient valid pixels to 

1495 # produce a valid correction. Mark pixels within the size 

1496 # of the brighter-fatter kernel as EDGE to warn of this 

1497 # fact. 

1498 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1499 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1500 maskPlane="EDGE") 

1501 

1502 if self.config.brighterFatterMaskGrowSize > 0: 

1503 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1504 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1505 isrFunctions.growMasks(ccdExposure.getMask(), 

1506 radius=self.config.brighterFatterMaskGrowSize, 

1507 maskNameList=maskPlane, 

1508 maskValue=maskPlane) 

1509 

1510 self.debugView(ccdExposure, "doBrighterFatter") 

1511 

1512 if self.config.doDark: 

1513 self.log.info("Applying dark correction.") 

1514 self.darkCorrection(ccdExposure, dark) 

1515 self.debugView(ccdExposure, "doDark") 

1516 

1517 if self.config.doFringe and not self.config.fringeAfterFlat: 

1518 self.log.info("Applying fringe correction before flat.") 

1519 self.fringe.run(ccdExposure, **fringes.getDict()) 

1520 self.debugView(ccdExposure, "doFringe") 

1521 

1522 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1523 self.log.info("Checking strayLight correction.") 

1524 self.strayLight.run(ccdExposure, strayLightData) 

1525 self.debugView(ccdExposure, "doStrayLight") 

1526 

1527 if self.config.doFlat: 

1528 self.log.info("Applying flat correction.") 

1529 self.flatCorrection(ccdExposure, flat) 

1530 self.debugView(ccdExposure, "doFlat") 

1531 

1532 if self.config.doApplyGains: 

1533 self.log.info("Applying gain correction instead of flat.") 

1534 if self.config.usePtcGains: 

1535 self.log.info("Using gains from the Photon Transfer Curve.") 

1536 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1537 ptcGains=ptc.gain) 

1538 else: 

1539 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1540 

1541 if self.config.doFringe and self.config.fringeAfterFlat: 

1542 self.log.info("Applying fringe correction after flat.") 

1543 self.fringe.run(ccdExposure, **fringes.getDict()) 

1544 

1545 if self.config.doVignette: 

1546 if self.config.doMaskVignettePolygon: 

1547 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1548 else: 

1549 self.log.info("Constructing and attaching vignette polygon.") 

1550 self.vignettePolygon = self.vignette.run( 

1551 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1552 vignetteValue=self.config.vignetteValue, log=self.log) 

1553 

1554 if self.config.doAttachTransmissionCurve: 

1555 self.log.info("Adding transmission curves.") 

1556 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1557 filterTransmission=filterTransmission, 

1558 sensorTransmission=sensorTransmission, 

1559 atmosphereTransmission=atmosphereTransmission) 

1560 

1561 flattenedThumb = None 

1562 if self.config.qa.doThumbnailFlattened: 

1563 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1564 

1565 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1566 self.log.info("Performing illumination correction.") 

1567 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1568 illumMaskedImage, illumScale=self.config.illumScale, 

1569 trimToFit=self.config.doTrimToMatchCalib) 

1570 

1571 preInterpExp = None 

1572 if self.config.doSaveInterpPixels: 

1573 preInterpExp = ccdExposure.clone() 

1574 

1575 # Reset and interpolate bad pixels. 

1576 # 

1577 # Large contiguous bad regions (which should have the BAD mask 

1578 # bit set) should have their values set to the image median. 

1579 # This group should include defects and bad amplifiers. As the 

1580 # area covered by these defects are large, there's little 

1581 # reason to expect that interpolation would provide a more 

1582 # useful value. 

1583 # 

1584 # Smaller defects can be safely interpolated after the larger 

1585 # regions have had their pixel values reset. This ensures 

1586 # that the remaining defects adjacent to bad amplifiers (as an 

1587 # example) do not attempt to interpolate extreme values. 

1588 if self.config.doSetBadRegions: 

1589 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1590 if badPixelCount > 0: 

1591 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1592 

1593 if self.config.doInterpolate: 

1594 self.log.info("Interpolating masked pixels.") 

1595 isrFunctions.interpolateFromMask( 

1596 maskedImage=ccdExposure.getMaskedImage(), 

1597 fwhm=self.config.fwhm, 

1598 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1599 maskNameList=list(self.config.maskListToInterpolate) 

1600 ) 

1601 

1602 self.roughZeroPoint(ccdExposure) 

1603 

1604 # correct for amp offsets within the CCD 

1605 if self.config.doAmpOffset: 

1606 self.log.info("Correcting amp offsets.") 

1607 self.ampOffset.run(ccdExposure) 

1608 

1609 if self.config.doMeasureBackground: 

1610 self.log.info("Measuring background level.") 

1611 self.measureBackground(ccdExposure, self.config.qa) 

1612 

1613 if self.config.qa is not None and self.config.qa.saveStats is True: 

1614 for amp in ccd: 

1615 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1616 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1617 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1618 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1619 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1620 qaStats.getValue(afwMath.STDEVCLIP) 

1621 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1622 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1623 qaStats.getValue(afwMath.STDEVCLIP)) 

1624 

1625 # calculate additional statistics. 

1626 outputStatistics = None 

1627 if self.config.doCalculateStatistics: 

1628 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1629 ptc=ptc).results 

1630 

1631 self.debugView(ccdExposure, "postISRCCD") 

1632 

1633 return pipeBase.Struct( 

1634 exposure=ccdExposure, 

1635 ossThumb=ossThumb, 

1636 flattenedThumb=flattenedThumb, 

1637 

1638 preInterpExposure=preInterpExp, 

1639 outputExposure=ccdExposure, 

1640 outputOssThumbnail=ossThumb, 

1641 outputFlattenedThumbnail=flattenedThumb, 

1642 outputStatistics=outputStatistics, 

1643 ) 

1644 

1645 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1646 """Ensure that the data returned by Butler is a fully constructed exp. 

1647 

1648 ISR requires exposure-level image data for historical reasons, so if we 

1649 did not recieve that from Butler, construct it from what we have, 

1650 modifying the input in place. 

1651 

1652 Parameters 

1653 ---------- 

1654 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, 

1655 or `lsst.afw.image.ImageF` 

1656 The input data structure obtained from Butler. 

1657 camera : `lsst.afw.cameraGeom.camera`, optional 

1658 The camera associated with the image. Used to find the appropriate 

1659 detector if detector is not already set. 

1660 detectorNum : `int`, optional 

1661 The detector in the camera to attach, if the detector is not 

1662 already set. 

1663 

1664 Returns 

1665 ------- 

1666 inputExp : `lsst.afw.image.Exposure` 

1667 The re-constructed exposure, with appropriate detector parameters. 

1668 

1669 Raises 

1670 ------ 

1671 TypeError 

1672 Raised if the input data cannot be used to construct an exposure. 

1673 """ 

1674 if isinstance(inputExp, afwImage.DecoratedImageU): 

1675 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1676 elif isinstance(inputExp, afwImage.ImageF): 

1677 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1678 elif isinstance(inputExp, afwImage.MaskedImageF): 

1679 inputExp = afwImage.makeExposure(inputExp) 

1680 elif isinstance(inputExp, afwImage.Exposure): 

1681 pass 

1682 elif inputExp is None: 

1683 # Assume this will be caught by the setup if it is a problem. 

1684 return inputExp 

1685 else: 

1686 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1687 (type(inputExp), )) 

1688 

1689 if inputExp.getDetector() is None: 

1690 if camera is None or detectorNum is None: 

1691 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1692 'without a detector set.') 

1693 inputExp.setDetector(camera[detectorNum]) 

1694 

1695 return inputExp 

1696 

1697 def convertIntToFloat(self, exposure): 

1698 """Convert exposure image from uint16 to float. 

1699 

1700 If the exposure does not need to be converted, the input is 

1701 immediately returned. For exposures that are converted to use 

1702 floating point pixels, the variance is set to unity and the 

1703 mask to zero. 

1704 

1705 Parameters 

1706 ---------- 

1707 exposure : `lsst.afw.image.Exposure` 

1708 The raw exposure to be converted. 

1709 

1710 Returns 

1711 ------- 

1712 newexposure : `lsst.afw.image.Exposure` 

1713 The input ``exposure``, converted to floating point pixels. 

1714 

1715 Raises 

1716 ------ 

1717 RuntimeError 

1718 Raised if the exposure type cannot be converted to float. 

1719 

1720 """ 

1721 if isinstance(exposure, afwImage.ExposureF): 

1722 # Nothing to be done 

1723 self.log.debug("Exposure already of type float.") 

1724 return exposure 

1725 if not hasattr(exposure, "convertF"): 

1726 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1727 

1728 newexposure = exposure.convertF() 

1729 newexposure.variance[:] = 1 

1730 newexposure.mask[:] = 0x0 

1731 

1732 return newexposure 

1733 

1734 def maskAmplifier(self, ccdExposure, amp, defects): 

1735 """Identify bad amplifiers, saturated and suspect pixels. 

1736 

1737 Parameters 

1738 ---------- 

1739 ccdExposure : `lsst.afw.image.Exposure` 

1740 Input exposure to be masked. 

1741 amp : `lsst.afw.table.AmpInfoCatalog` 

1742 Catalog of parameters defining the amplifier on this 

1743 exposure to mask. 

1744 defects : `lsst.ip.isr.Defects` 

1745 List of defects. Used to determine if the entire 

1746 amplifier is bad. 

1747 

1748 Returns 

1749 ------- 

1750 badAmp : `Bool` 

1751 If this is true, the entire amplifier area is covered by 

1752 defects and unusable. 

1753 

1754 """ 

1755 maskedImage = ccdExposure.getMaskedImage() 

1756 

1757 badAmp = False 

1758 

1759 # Check if entire amp region is defined as a defect 

1760 # NB: need to use amp.getBBox() for correct comparison with current 

1761 # defects definition. 

1762 if defects is not None: 

1763 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1764 

1765 # In the case of a bad amp, we will set mask to "BAD" 

1766 # (here use amp.getRawBBox() for correct association with pixels in 

1767 # current ccdExposure). 

1768 if badAmp: 

1769 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1770 afwImage.PARENT) 

1771 maskView = dataView.getMask() 

1772 maskView |= maskView.getPlaneBitMask("BAD") 

1773 del maskView 

1774 return badAmp 

1775 

1776 # Mask remaining defects after assembleCcd() to allow for defects that 

1777 # cross amplifier boundaries. Saturation and suspect pixels can be 

1778 # masked now, though. 

1779 limits = dict() 

1780 if self.config.doSaturation and not badAmp: 

1781 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1782 if self.config.doSuspect and not badAmp: 

1783 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1784 if math.isfinite(self.config.saturation): 

1785 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1786 

1787 for maskName, maskThreshold in limits.items(): 

1788 if not math.isnan(maskThreshold): 

1789 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1790 isrFunctions.makeThresholdMask( 

1791 maskedImage=dataView, 

1792 threshold=maskThreshold, 

1793 growFootprints=0, 

1794 maskName=maskName 

1795 ) 

1796 

1797 # Determine if we've fully masked this amplifier with SUSPECT and 

1798 # SAT pixels. 

1799 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1800 afwImage.PARENT) 

1801 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1802 self.config.suspectMaskName]) 

1803 if numpy.all(maskView.getArray() & maskVal > 0): 

1804 badAmp = True 

1805 maskView |= maskView.getPlaneBitMask("BAD") 

1806 

1807 return badAmp 

1808 

1809 def overscanCorrection(self, ccdExposure, amp): 

1810 """Apply overscan correction in place. 

1811 

1812 This method does initial pixel rejection of the overscan 

1813 region. The overscan can also be optionally segmented to 

1814 allow for discontinuous overscan responses to be fit 

1815 separately. The actual overscan subtraction is performed by 

1816 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1817 which is called here after the amplifier is preprocessed. 

1818 

1819 Parameters 

1820 ---------- 

1821 ccdExposure : `lsst.afw.image.Exposure` 

1822 Exposure to have overscan correction performed. 

1823 amp : `lsst.afw.cameraGeom.Amplifer` 

1824 The amplifier to consider while correcting the overscan. 

1825 

1826 Returns 

1827 ------- 

1828 overscanResults : `lsst.pipe.base.Struct` 

1829 Result struct with components: 

1830 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1831 Value or fit subtracted from the amplifier image data. 

1832 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1833 Value or fit subtracted from the overscan image data. 

1834 - ``overscanImage`` : `lsst.afw.image.Image` 

1835 Image of the overscan region with the overscan 

1836 correction applied. This quantity is used to estimate 

1837 the amplifier read noise empirically. 

1838 

1839 Raises 

1840 ------ 

1841 RuntimeError 

1842 Raised if the ``amp`` does not contain raw pixel information. 

1843 

1844 See Also 

1845 -------- 

1846 lsst.ip.isr.isrFunctions.overscanCorrection 

1847 """ 

1848 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1849 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1850 return None 

1851 

1852 statControl = afwMath.StatisticsControl() 

1853 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1854 

1855 # Determine the bounding boxes 

1856 dataBBox = amp.getRawDataBBox() 

1857 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1858 dx0 = 0 

1859 dx1 = 0 

1860 

1861 prescanBBox = amp.getRawPrescanBBox() 

1862 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1863 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1864 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1865 else: 

1866 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1867 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1868 

1869 # Determine if we need to work on subregions of the amplifier 

1870 # and overscan. 

1871 imageBBoxes = [] 

1872 overscanBBoxes = [] 

1873 

1874 if ((self.config.overscanBiasJump 

1875 and self.config.overscanBiasJumpLocation) 

1876 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

1877 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1878 self.config.overscanBiasJumpDevices)): 

1879 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1880 yLower = self.config.overscanBiasJumpLocation 

1881 yUpper = dataBBox.getHeight() - yLower 

1882 else: 

1883 yUpper = self.config.overscanBiasJumpLocation 

1884 yLower = dataBBox.getHeight() - yUpper 

1885 

1886 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1887 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1888 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1889 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1890 yLower))) 

1891 

1892 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1893 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1894 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1895 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1896 yUpper))) 

1897 else: 

1898 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1899 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1900 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1901 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1902 oscanBBox.getHeight()))) 

1903 

1904 # Perform overscan correction on subregions, ensuring saturated 

1905 # pixels are masked. 

1906 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1907 ampImage = ccdExposure.maskedImage[imageBBox] 

1908 overscanImage = ccdExposure.maskedImage[overscanBBox] 

1909 

1910 overscanArray = overscanImage.image.array 

1911 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

1912 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

1913 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

1914 

1915 statControl = afwMath.StatisticsControl() 

1916 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1917 

1918 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

1919 

1920 # If we trimmed columns, we need to restore them. 

1921 if dx0 != 0 or dx1 != 0: 

1922 fullOverscan = ccdExposure.maskedImage[oscanBBox] 

1923 overscanVector = overscanResults.overscanFit.array[:, 0] 

1924 overscanModel = afwImage.ImageF(fullOverscan.getDimensions()) 

1925 overscanModel.array[:, :] = 0.0 

1926 overscanModel.array[:, 0:dx0] = overscanVector[:, numpy.newaxis] 

1927 overscanModel.array[:, dx1:] = overscanVector[:, numpy.newaxis] 

1928 fullOverscanImage = fullOverscan.getImage() 

1929 fullOverscanImage -= overscanModel 

1930 overscanResults = pipeBase.Struct(imageFit=overscanResults.imageFit, 

1931 overscanFit=overscanModel, 

1932 overscanImage=fullOverscan, 

1933 edgeMask=overscanResults.edgeMask) 

1934 

1935 # Measure average overscan levels and record them in the metadata. 

1936 levelStat = afwMath.MEDIAN 

1937 sigmaStat = afwMath.STDEVCLIP 

1938 

1939 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

1940 self.config.qa.flatness.nIter) 

1941 metadata = ccdExposure.getMetadata() 

1942 ampNum = amp.getName() 

1943 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

1944 if isinstance(overscanResults.overscanFit, float): 

1945 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit 

1946 metadata[f"ISR_OSCAN_SIGMA{ampNum}"] = 0.0 

1947 else: 

1948 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

1949 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat) 

1950 metadata[f"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat) 

1951 

1952 return overscanResults 

1953 

1954 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

1955 """Set the variance plane using the gain and read noise 

1956 

1957 The read noise is calculated from the ``overscanImage`` if the 

1958 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1959 the value from the amplifier data is used. 

1960 

1961 Parameters 

1962 ---------- 

1963 ampExposure : `lsst.afw.image.Exposure` 

1964 Exposure to process. 

1965 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

1966 Amplifier detector data. 

1967 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

1968 Image of overscan, required only for empirical read noise. 

1969 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1970 PTC dataset containing the gains and read noise. 

1971 

1972 

1973 Raises 

1974 ------ 

1975 RuntimeError 

1976 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

1977 are ``True``, but ptcDataset is not provided. 

1978 

1979 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

1980 ``overscanImage`` is ``None``. 

1981 

1982 See also 

1983 -------- 

1984 lsst.ip.isr.isrFunctions.updateVariance 

1985 """ 

1986 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

1987 if self.config.usePtcGains: 

1988 if ptcDataset is None: 

1989 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

1990 else: 

1991 gain = ptcDataset.gain[amp.getName()] 

1992 self.log.info("Using gain from Photon Transfer Curve.") 

1993 else: 

1994 gain = amp.getGain() 

1995 

1996 if math.isnan(gain): 

1997 gain = 1.0 

1998 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

1999 elif gain <= 0: 

2000 patchedGain = 1.0 

2001 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2002 amp.getName(), gain, patchedGain) 

2003 gain = patchedGain 

2004 

2005 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2006 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2007 

2008 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2009 stats = afwMath.StatisticsControl() 

2010 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2011 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2012 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2013 amp.getName(), readNoise) 

2014 elif self.config.usePtcReadNoise: 

2015 if ptcDataset is None: 

2016 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2017 else: 

2018 readNoise = ptcDataset.noise[amp.getName()] 

2019 self.log.info("Using read noise from Photon Transfer Curve.") 

2020 else: 

2021 readNoise = amp.getReadNoise() 

2022 

2023 isrFunctions.updateVariance( 

2024 maskedImage=ampExposure.getMaskedImage(), 

2025 gain=gain, 

2026 readNoise=readNoise, 

2027 ) 

2028 

2029 def maskNegativeVariance(self, exposure): 

2030 """Identify and mask pixels with negative variance values. 

2031 

2032 Parameters 

2033 ---------- 

2034 exposure : `lsst.afw.image.Exposure` 

2035 Exposure to process. 

2036 

2037 See Also 

2038 -------- 

2039 lsst.ip.isr.isrFunctions.updateVariance 

2040 """ 

2041 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2042 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2043 exposure.mask.array[bad] |= maskPlane 

2044 

2045 def darkCorrection(self, exposure, darkExposure, invert=False): 

2046 """Apply dark correction in place. 

2047 

2048 Parameters 

2049 ---------- 

2050 exposure : `lsst.afw.image.Exposure` 

2051 Exposure to process. 

2052 darkExposure : `lsst.afw.image.Exposure` 

2053 Dark exposure of the same size as ``exposure``. 

2054 invert : `Bool`, optional 

2055 If True, re-add the dark to an already corrected image. 

2056 

2057 Raises 

2058 ------ 

2059 RuntimeError 

2060 Raised if either ``exposure`` or ``darkExposure`` do not 

2061 have their dark time defined. 

2062 

2063 See Also 

2064 -------- 

2065 lsst.ip.isr.isrFunctions.darkCorrection 

2066 """ 

2067 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2068 if math.isnan(expScale): 

2069 raise RuntimeError("Exposure darktime is NAN.") 

2070 if darkExposure.getInfo().getVisitInfo() is not None \ 

2071 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2072 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2073 else: 

2074 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2075 # so getDarkTime() does not exist. 

2076 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2077 darkScale = 1.0 

2078 

2079 isrFunctions.darkCorrection( 

2080 maskedImage=exposure.getMaskedImage(), 

2081 darkMaskedImage=darkExposure.getMaskedImage(), 

2082 expScale=expScale, 

2083 darkScale=darkScale, 

2084 invert=invert, 

2085 trimToFit=self.config.doTrimToMatchCalib 

2086 ) 

2087 

2088 def doLinearize(self, detector): 

2089 """Check if linearization is needed for the detector cameraGeom. 

2090 

2091 Checks config.doLinearize and the linearity type of the first 

2092 amplifier. 

2093 

2094 Parameters 

2095 ---------- 

2096 detector : `lsst.afw.cameraGeom.Detector` 

2097 Detector to get linearity type from. 

2098 

2099 Returns 

2100 ------- 

2101 doLinearize : `Bool` 

2102 If True, linearization should be performed. 

2103 """ 

2104 return self.config.doLinearize and \ 

2105 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2106 

2107 def flatCorrection(self, exposure, flatExposure, invert=False): 

2108 """Apply flat correction in place. 

2109 

2110 Parameters 

2111 ---------- 

2112 exposure : `lsst.afw.image.Exposure` 

2113 Exposure to process. 

2114 flatExposure : `lsst.afw.image.Exposure` 

2115 Flat exposure of the same size as ``exposure``. 

2116 invert : `Bool`, optional 

2117 If True, unflatten an already flattened image. 

2118 

2119 See Also 

2120 -------- 

2121 lsst.ip.isr.isrFunctions.flatCorrection 

2122 """ 

2123 isrFunctions.flatCorrection( 

2124 maskedImage=exposure.getMaskedImage(), 

2125 flatMaskedImage=flatExposure.getMaskedImage(), 

2126 scalingType=self.config.flatScalingType, 

2127 userScale=self.config.flatUserScale, 

2128 invert=invert, 

2129 trimToFit=self.config.doTrimToMatchCalib 

2130 ) 

2131 

2132 def saturationDetection(self, exposure, amp): 

2133 """Detect and mask saturated pixels in config.saturatedMaskName. 

2134 

2135 Parameters 

2136 ---------- 

2137 exposure : `lsst.afw.image.Exposure` 

2138 Exposure to process. Only the amplifier DataSec is processed. 

2139 amp : `lsst.afw.table.AmpInfoCatalog` 

2140 Amplifier detector data. 

2141 

2142 See Also 

2143 -------- 

2144 lsst.ip.isr.isrFunctions.makeThresholdMask 

2145 """ 

2146 if not math.isnan(amp.getSaturation()): 

2147 maskedImage = exposure.getMaskedImage() 

2148 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2149 isrFunctions.makeThresholdMask( 

2150 maskedImage=dataView, 

2151 threshold=amp.getSaturation(), 

2152 growFootprints=0, 

2153 maskName=self.config.saturatedMaskName, 

2154 ) 

2155 

2156 def saturationInterpolation(self, exposure): 

2157 """Interpolate over saturated pixels, in place. 

2158 

2159 This method should be called after `saturationDetection`, to 

2160 ensure that the saturated pixels have been identified in the 

2161 SAT mask. It should also be called after `assembleCcd`, since 

2162 saturated regions may cross amplifier boundaries. 

2163 

2164 Parameters 

2165 ---------- 

2166 exposure : `lsst.afw.image.Exposure` 

2167 Exposure to process. 

2168 

2169 See Also 

2170 -------- 

2171 lsst.ip.isr.isrTask.saturationDetection 

2172 lsst.ip.isr.isrFunctions.interpolateFromMask 

2173 """ 

2174 isrFunctions.interpolateFromMask( 

2175 maskedImage=exposure.getMaskedImage(), 

2176 fwhm=self.config.fwhm, 

2177 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2178 maskNameList=list(self.config.saturatedMaskName), 

2179 ) 

2180 

2181 def suspectDetection(self, exposure, amp): 

2182 """Detect and mask suspect pixels in config.suspectMaskName. 

2183 

2184 Parameters 

2185 ---------- 

2186 exposure : `lsst.afw.image.Exposure` 

2187 Exposure to process. Only the amplifier DataSec is processed. 

2188 amp : `lsst.afw.table.AmpInfoCatalog` 

2189 Amplifier detector data. 

2190 

2191 See Also 

2192 -------- 

2193 lsst.ip.isr.isrFunctions.makeThresholdMask 

2194 

2195 Notes 

2196 ----- 

2197 Suspect pixels are pixels whose value is greater than 

2198 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2199 affected by unknown systematics; for example if non-linearity 

2200 corrections above a certain level are unstable then that would be a 

2201 useful value for suspectLevel. A value of `nan` indicates that no such 

2202 level exists and no pixels are to be masked as suspicious. 

2203 """ 

2204 suspectLevel = amp.getSuspectLevel() 

2205 if math.isnan(suspectLevel): 

2206 return 

2207 

2208 maskedImage = exposure.getMaskedImage() 

2209 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2210 isrFunctions.makeThresholdMask( 

2211 maskedImage=dataView, 

2212 threshold=suspectLevel, 

2213 growFootprints=0, 

2214 maskName=self.config.suspectMaskName, 

2215 ) 

2216 

2217 def maskDefect(self, exposure, defectBaseList): 

2218 """Mask defects using mask plane "BAD", in place. 

2219 

2220 Parameters 

2221 ---------- 

2222 exposure : `lsst.afw.image.Exposure` 

2223 Exposure to process. 

2224 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2225 `lsst.afw.image.DefectBase`. 

2226 List of defects to mask. 

2227 

2228 Notes 

2229 ----- 

2230 Call this after CCD assembly, since defects may cross amplifier 

2231 boundaries. 

2232 """ 

2233 maskedImage = exposure.getMaskedImage() 

2234 if not isinstance(defectBaseList, Defects): 

2235 # Promotes DefectBase to Defect 

2236 defectList = Defects(defectBaseList) 

2237 else: 

2238 defectList = defectBaseList 

2239 defectList.maskPixels(maskedImage, maskName="BAD") 

2240 

2241 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2242 """Mask edge pixels with applicable mask plane. 

2243 

2244 Parameters 

2245 ---------- 

2246 exposure : `lsst.afw.image.Exposure` 

2247 Exposure to process. 

2248 numEdgePixels : `int`, optional 

2249 Number of edge pixels to mask. 

2250 maskPlane : `str`, optional 

2251 Mask plane name to use. 

2252 level : `str`, optional 

2253 Level at which to mask edges. 

2254 """ 

2255 maskedImage = exposure.getMaskedImage() 

2256 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2257 

2258 if numEdgePixels > 0: 

2259 if level == 'DETECTOR': 

2260 boxes = [maskedImage.getBBox()] 

2261 elif level == 'AMP': 

2262 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2263 

2264 for box in boxes: 

2265 # This makes a bbox numEdgeSuspect pixels smaller than the 

2266 # image on each side 

2267 subImage = maskedImage[box] 

2268 box.grow(-numEdgePixels) 

2269 # Mask pixels outside box 

2270 SourceDetectionTask.setEdgeBits( 

2271 subImage, 

2272 box, 

2273 maskBitMask) 

2274 

2275 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2276 """Mask and interpolate defects using mask plane "BAD", in place. 

2277 

2278 Parameters 

2279 ---------- 

2280 exposure : `lsst.afw.image.Exposure` 

2281 Exposure to process. 

2282 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2283 `lsst.afw.image.DefectBase`. 

2284 List of defects to mask and interpolate. 

2285 

2286 See Also 

2287 -------- 

2288 lsst.ip.isr.isrTask.maskDefect 

2289 """ 

2290 self.maskDefect(exposure, defectBaseList) 

2291 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2292 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2293 isrFunctions.interpolateFromMask( 

2294 maskedImage=exposure.getMaskedImage(), 

2295 fwhm=self.config.fwhm, 

2296 growSaturatedFootprints=0, 

2297 maskNameList=["BAD"], 

2298 ) 

2299 

2300 def maskNan(self, exposure): 

2301 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2302 

2303 Parameters 

2304 ---------- 

2305 exposure : `lsst.afw.image.Exposure` 

2306 Exposure to process. 

2307 

2308 Notes 

2309 ----- 

2310 We mask over all non-finite values (NaN, inf), including those 

2311 that are masked with other bits (because those may or may not be 

2312 interpolated over later, and we want to remove all NaN/infs). 

2313 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2314 preserve the historical name. 

2315 """ 

2316 maskedImage = exposure.getMaskedImage() 

2317 

2318 # Find and mask NaNs 

2319 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2320 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2321 numNans = maskNans(maskedImage, maskVal) 

2322 self.metadata["NUMNANS"] = numNans 

2323 if numNans > 0: 

2324 self.log.warning("There were %d unmasked NaNs.", numNans) 

2325 

2326 def maskAndInterpolateNan(self, exposure): 

2327 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2328 in place. 

2329 

2330 Parameters 

2331 ---------- 

2332 exposure : `lsst.afw.image.Exposure` 

2333 Exposure to process. 

2334 

2335 See Also 

2336 -------- 

2337 lsst.ip.isr.isrTask.maskNan 

2338 """ 

2339 self.maskNan(exposure) 

2340 isrFunctions.interpolateFromMask( 

2341 maskedImage=exposure.getMaskedImage(), 

2342 fwhm=self.config.fwhm, 

2343 growSaturatedFootprints=0, 

2344 maskNameList=["UNMASKEDNAN"], 

2345 ) 

2346 

2347 def measureBackground(self, exposure, IsrQaConfig=None): 

2348 """Measure the image background in subgrids, for quality control. 

2349 

2350 Parameters 

2351 ---------- 

2352 exposure : `lsst.afw.image.Exposure` 

2353 Exposure to process. 

2354 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2355 Configuration object containing parameters on which background 

2356 statistics and subgrids to use. 

2357 """ 

2358 if IsrQaConfig is not None: 

2359 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2360 IsrQaConfig.flatness.nIter) 

2361 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2362 statsControl.setAndMask(maskVal) 

2363 maskedImage = exposure.getMaskedImage() 

2364 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2365 skyLevel = stats.getValue(afwMath.MEDIAN) 

2366 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2367 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2368 metadata = exposure.getMetadata() 

2369 metadata["SKYLEVEL"] = skyLevel 

2370 metadata["SKYSIGMA"] = skySigma 

2371 

2372 # calcluating flatlevel over the subgrids 

2373 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2374 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2375 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2376 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2377 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2378 skyLevels = numpy.zeros((nX, nY)) 

2379 

2380 for j in range(nY): 

2381 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2382 for i in range(nX): 

2383 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2384 

2385 xLLC = xc - meshXHalf 

2386 yLLC = yc - meshYHalf 

2387 xURC = xc + meshXHalf - 1 

2388 yURC = yc + meshYHalf - 1 

2389 

2390 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2391 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2392 

2393 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2394 

2395 good = numpy.where(numpy.isfinite(skyLevels)) 

2396 skyMedian = numpy.median(skyLevels[good]) 

2397 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2398 flatness_rms = numpy.std(flatness) 

2399 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2400 

2401 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2402 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2403 nX, nY, flatness_pp, flatness_rms) 

2404 

2405 metadata["FLATNESS_PP"] = float(flatness_pp) 

2406 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2407 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2408 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2409 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2410 

2411 def roughZeroPoint(self, exposure): 

2412 """Set an approximate magnitude zero point for the exposure. 

2413 

2414 Parameters 

2415 ---------- 

2416 exposure : `lsst.afw.image.Exposure` 

2417 Exposure to process. 

2418 """ 

2419 filterLabel = exposure.getFilter() 

2420 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2421 

2422 if physicalFilter in self.config.fluxMag0T1: 

2423 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2424 else: 

2425 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2426 fluxMag0 = self.config.defaultFluxMag0T1 

2427 

2428 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2429 if not expTime > 0: # handle NaN as well as <= 0 

2430 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2431 return 

2432 

2433 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2434 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2435 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2436 

2437 @contextmanager 

2438 def flatContext(self, exp, flat, dark=None): 

2439 """Context manager that applies and removes flats and darks, 

2440 if the task is configured to apply them. 

2441 

2442 Parameters 

2443 ---------- 

2444 exp : `lsst.afw.image.Exposure` 

2445 Exposure to process. 

2446 flat : `lsst.afw.image.Exposure` 

2447 Flat exposure the same size as ``exp``. 

2448 dark : `lsst.afw.image.Exposure`, optional 

2449 Dark exposure the same size as ``exp``. 

2450 

2451 Yields 

2452 ------ 

2453 exp : `lsst.afw.image.Exposure` 

2454 The flat and dark corrected exposure. 

2455 """ 

2456 if self.config.doDark and dark is not None: 

2457 self.darkCorrection(exp, dark) 

2458 if self.config.doFlat: 

2459 self.flatCorrection(exp, flat) 

2460 try: 

2461 yield exp 

2462 finally: 

2463 if self.config.doFlat: 

2464 self.flatCorrection(exp, flat, invert=True) 

2465 if self.config.doDark and dark is not None: 

2466 self.darkCorrection(exp, dark, invert=True) 

2467 

2468 def debugView(self, exposure, stepname): 

2469 """Utility function to examine ISR exposure at different stages. 

2470 

2471 Parameters 

2472 ---------- 

2473 exposure : `lsst.afw.image.Exposure` 

2474 Exposure to view. 

2475 stepname : `str` 

2476 State of processing to view. 

2477 """ 

2478 frame = getDebugFrame(self._display, stepname) 

2479 if frame: 

2480 display = getDisplay(frame) 

2481 display.scale('asinh', 'zscale') 

2482 display.mtv(exposure) 

2483 prompt = "Press Enter to continue [c]... " 

2484 while True: 

2485 ans = input(prompt).lower() 

2486 if ans in ("", "c",): 

2487 break 

2488 

2489 

2490class FakeAmp(object): 

2491 """A Detector-like object that supports returning gain and saturation level 

2492 

2493 This is used when the input exposure does not have a detector. 

2494 

2495 Parameters 

2496 ---------- 

2497 exposure : `lsst.afw.image.Exposure` 

2498 Exposure to generate a fake amplifier for. 

2499 config : `lsst.ip.isr.isrTaskConfig` 

2500 Configuration to apply to the fake amplifier. 

2501 """ 

2502 

2503 def __init__(self, exposure, config): 

2504 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2505 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2506 self._gain = config.gain 

2507 self._readNoise = config.readNoise 

2508 self._saturation = config.saturation 

2509 

2510 def getBBox(self): 

2511 return self._bbox 

2512 

2513 def getRawBBox(self): 

2514 return self._bbox 

2515 

2516 def getRawHorizontalOverscanBBox(self): 

2517 return self._RawHorizontalOverscanBBox 

2518 

2519 def getGain(self): 

2520 return self._gain 

2521 

2522 def getReadNoise(self): 

2523 return self._readNoise 

2524 

2525 def getSaturation(self): 

2526 return self._saturation 

2527 

2528 def getSuspectLevel(self): 

2529 return float("NaN")