Coverage for python/lsst/ip/isr/isrTask.py: 17%

944 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-27 02:18 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import NullLinearityType, ReadoutCorner 

36from lsst.afw.display import getDisplay 

37from lsst.daf.persistence import ButlerDataRef 

38from lsst.daf.persistence.butler import NoResults 

39from lsst.meas.algorithms.detection import SourceDetectionTask 

40from lsst.utils.timer import timeMethod 

41 

42from . import isrFunctions 

43from . import isrQa 

44from . import linearize 

45from .defects import Defects 

46 

47from .assembleCcdTask import AssembleCcdTask 

48from .crosstalk import CrosstalkTask, CrosstalkCalib 

49from .fringe import FringeTask 

50from .isr import maskNans 

51from .masking import MaskingTask 

52from .overscan import OverscanCorrectionTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55from .ampOffset import AmpOffsetTask 

56from .deferredCharge import DeferredChargeTask 

57from .isrStatistics import IsrStatisticsTask 

58from lsst.daf.butler import DimensionGraph 

59 

60 

61__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

62 

63 

64def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

65 """Lookup function to identify crosstalkSource entries. 

66 

67 This should return an empty list under most circumstances. Only 

68 when inter-chip crosstalk has been identified should this be 

69 populated. 

70 

71 Parameters 

72 ---------- 

73 datasetType : `str` 

74 Dataset to lookup. 

75 registry : `lsst.daf.butler.Registry` 

76 Butler registry to query. 

77 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

78 Data id to transform to identify crosstalkSources. The 

79 ``detector`` entry will be stripped. 

80 collections : `lsst.daf.butler.CollectionSearch` 

81 Collections to search through. 

82 

83 Returns 

84 ------- 

85 results : `list` [`lsst.daf.butler.DatasetRef`] 

86 List of datasets that match the query that will be used as 

87 crosstalkSources. 

88 """ 

89 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

90 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId, 

91 findFirst=True)) 

92 # In some contexts, calling `.expanded()` to expand all data IDs in the 

93 # query results can be a lot faster because it vectorizes lookups. But in 

94 # this case, expandDataId shouldn't need to hit the database at all in the 

95 # steady state, because only the detector record is unknown and those are 

96 # cached in the registry. 

97 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results] 

98 

99 

100class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

101 dimensions={"instrument", "exposure", "detector"}, 

102 defaultTemplates={}): 

103 ccdExposure = cT.Input( 

104 name="raw", 

105 doc="Input exposure to process.", 

106 storageClass="Exposure", 

107 dimensions=["instrument", "exposure", "detector"], 

108 ) 

109 camera = cT.PrerequisiteInput( 

110 name="camera", 

111 storageClass="Camera", 

112 doc="Input camera to construct complete exposures.", 

113 dimensions=["instrument"], 

114 isCalibration=True, 

115 ) 

116 

117 crosstalk = cT.PrerequisiteInput( 

118 name="crosstalk", 

119 doc="Input crosstalk object", 

120 storageClass="CrosstalkCalib", 

121 dimensions=["instrument", "detector"], 

122 isCalibration=True, 

123 minimum=0, # can fall back to cameraGeom 

124 ) 

125 crosstalkSources = cT.PrerequisiteInput( 

126 name="isrOverscanCorrected", 

127 doc="Overscan corrected input images.", 

128 storageClass="Exposure", 

129 dimensions=["instrument", "exposure", "detector"], 

130 deferLoad=True, 

131 multiple=True, 

132 lookupFunction=crosstalkSourceLookup, 

133 minimum=0, # not needed for all instruments, no config to control this 

134 ) 

135 bias = cT.PrerequisiteInput( 

136 name="bias", 

137 doc="Input bias calibration.", 

138 storageClass="ExposureF", 

139 dimensions=["instrument", "detector"], 

140 isCalibration=True, 

141 ) 

142 dark = cT.PrerequisiteInput( 

143 name='dark', 

144 doc="Input dark calibration.", 

145 storageClass="ExposureF", 

146 dimensions=["instrument", "detector"], 

147 isCalibration=True, 

148 ) 

149 flat = cT.PrerequisiteInput( 

150 name="flat", 

151 doc="Input flat calibration.", 

152 storageClass="ExposureF", 

153 dimensions=["instrument", "physical_filter", "detector"], 

154 isCalibration=True, 

155 ) 

156 ptc = cT.PrerequisiteInput( 

157 name="ptc", 

158 doc="Input Photon Transfer Curve dataset", 

159 storageClass="PhotonTransferCurveDataset", 

160 dimensions=["instrument", "detector"], 

161 isCalibration=True, 

162 ) 

163 fringes = cT.PrerequisiteInput( 

164 name="fringe", 

165 doc="Input fringe calibration.", 

166 storageClass="ExposureF", 

167 dimensions=["instrument", "physical_filter", "detector"], 

168 isCalibration=True, 

169 minimum=0, # only needed for some bands, even when enabled 

170 ) 

171 strayLightData = cT.PrerequisiteInput( 

172 name='yBackground', 

173 doc="Input stray light calibration.", 

174 storageClass="StrayLightData", 

175 dimensions=["instrument", "physical_filter", "detector"], 

176 deferLoad=True, 

177 isCalibration=True, 

178 minimum=0, # only needed for some bands, even when enabled 

179 ) 

180 bfKernel = cT.PrerequisiteInput( 

181 name='bfKernel', 

182 doc="Input brighter-fatter kernel.", 

183 storageClass="NumpyArray", 

184 dimensions=["instrument"], 

185 isCalibration=True, 

186 minimum=0, # can use either bfKernel or newBFKernel 

187 ) 

188 newBFKernel = cT.PrerequisiteInput( 

189 name='brighterFatterKernel', 

190 doc="Newer complete kernel + gain solutions.", 

191 storageClass="BrighterFatterKernel", 

192 dimensions=["instrument", "detector"], 

193 isCalibration=True, 

194 minimum=0, # can use either bfKernel or newBFKernel 

195 ) 

196 defects = cT.PrerequisiteInput( 

197 name='defects', 

198 doc="Input defect tables.", 

199 storageClass="Defects", 

200 dimensions=["instrument", "detector"], 

201 isCalibration=True, 

202 ) 

203 linearizer = cT.PrerequisiteInput( 

204 name='linearizer', 

205 storageClass="Linearizer", 

206 doc="Linearity correction calibration.", 

207 dimensions=["instrument", "detector"], 

208 isCalibration=True, 

209 minimum=0, # can fall back to cameraGeom 

210 ) 

211 opticsTransmission = cT.PrerequisiteInput( 

212 name="transmission_optics", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the optics.", 

215 dimensions=["instrument"], 

216 isCalibration=True, 

217 ) 

218 filterTransmission = cT.PrerequisiteInput( 

219 name="transmission_filter", 

220 storageClass="TransmissionCurve", 

221 doc="Transmission curve due to the filter.", 

222 dimensions=["instrument", "physical_filter"], 

223 isCalibration=True, 

224 ) 

225 sensorTransmission = cT.PrerequisiteInput( 

226 name="transmission_sensor", 

227 storageClass="TransmissionCurve", 

228 doc="Transmission curve due to the sensor.", 

229 dimensions=["instrument", "detector"], 

230 isCalibration=True, 

231 ) 

232 atmosphereTransmission = cT.PrerequisiteInput( 

233 name="transmission_atmosphere", 

234 storageClass="TransmissionCurve", 

235 doc="Transmission curve due to the atmosphere.", 

236 dimensions=["instrument"], 

237 isCalibration=True, 

238 ) 

239 illumMaskedImage = cT.PrerequisiteInput( 

240 name="illum", 

241 doc="Input illumination correction.", 

242 storageClass="MaskedImageF", 

243 dimensions=["instrument", "physical_filter", "detector"], 

244 isCalibration=True, 

245 ) 

246 deferredChargeCalib = cT.PrerequisiteInput( 

247 name="deferredCharge", 

248 doc="Deferred charge/CTI correction dataset.", 

249 storageClass="IsrCalib", 

250 dimensions=["instrument", "detector"], 

251 isCalibration=True, 

252 ) 

253 

254 outputExposure = cT.Output( 

255 name='postISRCCD', 

256 doc="Output ISR processed exposure.", 

257 storageClass="Exposure", 

258 dimensions=["instrument", "exposure", "detector"], 

259 ) 

260 preInterpExposure = cT.Output( 

261 name='preInterpISRCCD', 

262 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

263 storageClass="ExposureF", 

264 dimensions=["instrument", "exposure", "detector"], 

265 ) 

266 outputOssThumbnail = cT.Output( 

267 name="OssThumb", 

268 doc="Output Overscan-subtracted thumbnail image.", 

269 storageClass="Thumbnail", 

270 dimensions=["instrument", "exposure", "detector"], 

271 ) 

272 outputFlattenedThumbnail = cT.Output( 

273 name="FlattenedThumb", 

274 doc="Output flat-corrected thumbnail image.", 

275 storageClass="Thumbnail", 

276 dimensions=["instrument", "exposure", "detector"], 

277 ) 

278 outputStatistics = cT.Output( 

279 name="isrStatistics", 

280 doc="Output of additional statistics table.", 

281 storageClass="StructuredDataDict", 

282 dimensions=["instrument", "exposure", "detector"], 

283 ) 

284 

285 def __init__(self, *, config=None): 

286 super().__init__(config=config) 

287 

288 if config.doBias is not True: 

289 self.prerequisiteInputs.remove("bias") 

290 if config.doLinearize is not True: 

291 self.prerequisiteInputs.remove("linearizer") 

292 if config.doCrosstalk is not True: 

293 self.prerequisiteInputs.remove("crosstalkSources") 

294 self.prerequisiteInputs.remove("crosstalk") 

295 if config.doBrighterFatter is not True: 

296 self.prerequisiteInputs.remove("bfKernel") 

297 self.prerequisiteInputs.remove("newBFKernel") 

298 if config.doDefect is not True: 

299 self.prerequisiteInputs.remove("defects") 

300 if config.doDark is not True: 

301 self.prerequisiteInputs.remove("dark") 

302 if config.doFlat is not True: 

303 self.prerequisiteInputs.remove("flat") 

304 if config.doFringe is not True: 

305 self.prerequisiteInputs.remove("fringes") 

306 if config.doStrayLight is not True: 

307 self.prerequisiteInputs.remove("strayLightData") 

308 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

309 self.prerequisiteInputs.remove("ptc") 

310 if config.doAttachTransmissionCurve is not True: 

311 self.prerequisiteInputs.remove("opticsTransmission") 

312 self.prerequisiteInputs.remove("filterTransmission") 

313 self.prerequisiteInputs.remove("sensorTransmission") 

314 self.prerequisiteInputs.remove("atmosphereTransmission") 

315 else: 

316 if config.doUseOpticsTransmission is not True: 

317 self.prerequisiteInputs.remove("opticsTransmission") 

318 if config.doUseFilterTransmission is not True: 

319 self.prerequisiteInputs.remove("filterTransmission") 

320 if config.doUseSensorTransmission is not True: 

321 self.prerequisiteInputs.remove("sensorTransmission") 

322 if config.doUseAtmosphereTransmission is not True: 

323 self.prerequisiteInputs.remove("atmosphereTransmission") 

324 if config.doIlluminationCorrection is not True: 

325 self.prerequisiteInputs.remove("illumMaskedImage") 

326 if config.doDeferredCharge is not True: 

327 self.prerequisiteInputs.remove("deferredChargeCalib") 

328 

329 if config.doWrite is not True: 

330 self.outputs.remove("outputExposure") 

331 self.outputs.remove("preInterpExposure") 

332 self.outputs.remove("outputFlattenedThumbnail") 

333 self.outputs.remove("outputOssThumbnail") 

334 self.outputs.remove("outputStatistics") 

335 

336 if config.doSaveInterpPixels is not True: 

337 self.outputs.remove("preInterpExposure") 

338 if config.qa.doThumbnailOss is not True: 

339 self.outputs.remove("outputOssThumbnail") 

340 if config.qa.doThumbnailFlattened is not True: 

341 self.outputs.remove("outputFlattenedThumbnail") 

342 if config.doCalculateStatistics is not True: 

343 self.outputs.remove("outputStatistics") 

344 

345 

346class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

347 pipelineConnections=IsrTaskConnections): 

348 """Configuration parameters for IsrTask. 

349 

350 Items are grouped in the order in which they are executed by the task. 

351 """ 

352 datasetType = pexConfig.Field( 

353 dtype=str, 

354 doc="Dataset type for input data; users will typically leave this alone, " 

355 "but camera-specific ISR tasks will override it", 

356 default="raw", 

357 ) 

358 

359 fallbackFilterName = pexConfig.Field( 

360 dtype=str, 

361 doc="Fallback default filter name for calibrations.", 

362 optional=True 

363 ) 

364 useFallbackDate = pexConfig.Field( 

365 dtype=bool, 

366 doc="Pass observation date when using fallback filter.", 

367 default=False, 

368 ) 

369 expectWcs = pexConfig.Field( 

370 dtype=bool, 

371 default=True, 

372 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

373 ) 

374 fwhm = pexConfig.Field( 

375 dtype=float, 

376 doc="FWHM of PSF in arcseconds.", 

377 default=1.0, 

378 ) 

379 qa = pexConfig.ConfigField( 

380 dtype=isrQa.IsrQaConfig, 

381 doc="QA related configuration options.", 

382 ) 

383 

384 # Image conversion configuration 

385 doConvertIntToFloat = pexConfig.Field( 

386 dtype=bool, 

387 doc="Convert integer raw images to floating point values?", 

388 default=True, 

389 ) 

390 

391 # Saturated pixel handling. 

392 doSaturation = pexConfig.Field( 

393 dtype=bool, 

394 doc="Mask saturated pixels? NB: this is totally independent of the" 

395 " interpolation option - this is ONLY setting the bits in the mask." 

396 " To have them interpolated make sure doSaturationInterpolation=True", 

397 default=True, 

398 ) 

399 saturatedMaskName = pexConfig.Field( 

400 dtype=str, 

401 doc="Name of mask plane to use in saturation detection and interpolation", 

402 default="SAT", 

403 ) 

404 saturation = pexConfig.Field( 

405 dtype=float, 

406 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

407 default=float("NaN"), 

408 ) 

409 growSaturationFootprintSize = pexConfig.Field( 

410 dtype=int, 

411 doc="Number of pixels by which to grow the saturation footprints", 

412 default=1, 

413 ) 

414 

415 # Suspect pixel handling. 

416 doSuspect = pexConfig.Field( 

417 dtype=bool, 

418 doc="Mask suspect pixels?", 

419 default=False, 

420 ) 

421 suspectMaskName = pexConfig.Field( 

422 dtype=str, 

423 doc="Name of mask plane to use for suspect pixels", 

424 default="SUSPECT", 

425 ) 

426 numEdgeSuspect = pexConfig.Field( 

427 dtype=int, 

428 doc="Number of edge pixels to be flagged as untrustworthy.", 

429 default=0, 

430 ) 

431 edgeMaskLevel = pexConfig.ChoiceField( 

432 dtype=str, 

433 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

434 default="DETECTOR", 

435 allowed={ 

436 'DETECTOR': 'Mask only the edges of the full detector.', 

437 'AMP': 'Mask edges of each amplifier.', 

438 }, 

439 ) 

440 

441 # Initial masking options. 

442 doSetBadRegions = pexConfig.Field( 

443 dtype=bool, 

444 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

445 default=True, 

446 ) 

447 badStatistic = pexConfig.ChoiceField( 

448 dtype=str, 

449 doc="How to estimate the average value for BAD regions.", 

450 default='MEANCLIP', 

451 allowed={ 

452 "MEANCLIP": "Correct using the (clipped) mean of good data", 

453 "MEDIAN": "Correct using the median of the good data", 

454 }, 

455 ) 

456 

457 # Overscan subtraction configuration. 

458 doOverscan = pexConfig.Field( 

459 dtype=bool, 

460 doc="Do overscan subtraction?", 

461 default=True, 

462 ) 

463 overscan = pexConfig.ConfigurableField( 

464 target=OverscanCorrectionTask, 

465 doc="Overscan subtraction task for image segments.", 

466 ) 

467 overscanFitType = pexConfig.ChoiceField( 

468 dtype=str, 

469 doc="The method for fitting the overscan bias level.", 

470 default='MEDIAN', 

471 allowed={ 

472 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

473 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

474 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

475 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

476 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

477 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

478 "MEAN": "Correct using the mean of the overscan region", 

479 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

480 "MEDIAN": "Correct using the median of the overscan region", 

481 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

482 }, 

483 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

484 " This option will no longer be used, and will be removed after v20.") 

485 ) 

486 overscanOrder = pexConfig.Field( 

487 dtype=int, 

488 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

489 "or number of spline knots if overscan fit type is a spline."), 

490 default=1, 

491 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

492 " This option will no longer be used, and will be removed after v20.") 

493 ) 

494 overscanNumSigmaClip = pexConfig.Field( 

495 dtype=float, 

496 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

497 default=3.0, 

498 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

499 " This option will no longer be used, and will be removed after v20.") 

500 ) 

501 overscanIsInt = pexConfig.Field( 

502 dtype=bool, 

503 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

504 " and overscan.FitType=MEDIAN_PER_ROW.", 

505 default=True, 

506 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

507 " This option will no longer be used, and will be removed after v20.") 

508 ) 

509 # These options do not get deprecated, as they define how we slice up the 

510 # image data. 

511 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

512 dtype=int, 

513 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

514 default=0, 

515 ) 

516 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

517 dtype=int, 

518 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

519 default=0, 

520 ) 

521 overscanMaxDev = pexConfig.Field( 521 ↛ exitline 521 didn't jump to the function exit

522 dtype=float, 

523 doc="Maximum deviation from the median for overscan", 

524 default=1000.0, check=lambda x: x > 0 

525 ) 

526 overscanBiasJump = pexConfig.Field( 

527 dtype=bool, 

528 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

529 default=False, 

530 ) 

531 overscanBiasJumpKeyword = pexConfig.Field( 

532 dtype=str, 

533 doc="Header keyword containing information about devices.", 

534 default="NO_SUCH_KEY", 

535 ) 

536 overscanBiasJumpDevices = pexConfig.ListField( 

537 dtype=str, 

538 doc="List of devices that need piecewise overscan correction.", 

539 default=(), 

540 ) 

541 overscanBiasJumpLocation = pexConfig.Field( 

542 dtype=int, 

543 doc="Location of bias jump along y-axis.", 

544 default=0, 

545 ) 

546 

547 # Amplifier to CCD assembly configuration 

548 doAssembleCcd = pexConfig.Field( 

549 dtype=bool, 

550 default=True, 

551 doc="Assemble amp-level exposures into a ccd-level exposure?" 

552 ) 

553 assembleCcd = pexConfig.ConfigurableField( 

554 target=AssembleCcdTask, 

555 doc="CCD assembly task", 

556 ) 

557 

558 # General calibration configuration. 

559 doAssembleIsrExposures = pexConfig.Field( 

560 dtype=bool, 

561 default=False, 

562 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

563 ) 

564 doTrimToMatchCalib = pexConfig.Field( 

565 dtype=bool, 

566 default=False, 

567 doc="Trim raw data to match calibration bounding boxes?" 

568 ) 

569 

570 # Bias subtraction. 

571 doBias = pexConfig.Field( 

572 dtype=bool, 

573 doc="Apply bias frame correction?", 

574 default=True, 

575 ) 

576 biasDataProductName = pexConfig.Field( 

577 dtype=str, 

578 doc="Name of the bias data product", 

579 default="bias", 

580 ) 

581 doBiasBeforeOverscan = pexConfig.Field( 

582 dtype=bool, 

583 doc="Reverse order of overscan and bias correction.", 

584 default=False 

585 ) 

586 

587 # Deferred charge correction. 

588 doDeferredCharge = pexConfig.Field( 

589 dtype=bool, 

590 doc="Apply deferred charge correction?", 

591 default=False, 

592 ) 

593 deferredChargeCorrection = pexConfig.ConfigurableField( 

594 target=DeferredChargeTask, 

595 doc="Deferred charge correction task.", 

596 ) 

597 

598 # Variance construction 

599 doVariance = pexConfig.Field( 

600 dtype=bool, 

601 doc="Calculate variance?", 

602 default=True 

603 ) 

604 gain = pexConfig.Field( 

605 dtype=float, 

606 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

607 default=float("NaN"), 

608 ) 

609 readNoise = pexConfig.Field( 

610 dtype=float, 

611 doc="The read noise to use if no Detector is present in the Exposure", 

612 default=0.0, 

613 ) 

614 doEmpiricalReadNoise = pexConfig.Field( 

615 dtype=bool, 

616 default=False, 

617 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

618 ) 

619 usePtcReadNoise = pexConfig.Field( 

620 dtype=bool, 

621 default=False, 

622 doc="Use readnoise values from the Photon Transfer Curve?" 

623 ) 

624 maskNegativeVariance = pexConfig.Field( 

625 dtype=bool, 

626 default=True, 

627 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

628 "in the measurement of the overscan at an edge due to the data falling off faster " 

629 "than the overscan model can account for it." 

630 ) 

631 negativeVarianceMaskName = pexConfig.Field( 

632 dtype=str, 

633 default="BAD", 

634 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

635 ) 

636 # Linearization. 

637 doLinearize = pexConfig.Field( 

638 dtype=bool, 

639 doc="Correct for nonlinearity of the detector's response?", 

640 default=True, 

641 ) 

642 

643 # Crosstalk. 

644 doCrosstalk = pexConfig.Field( 

645 dtype=bool, 

646 doc="Apply intra-CCD crosstalk correction?", 

647 default=False, 

648 ) 

649 doCrosstalkBeforeAssemble = pexConfig.Field( 

650 dtype=bool, 

651 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

652 default=False, 

653 ) 

654 crosstalk = pexConfig.ConfigurableField( 

655 target=CrosstalkTask, 

656 doc="Intra-CCD crosstalk correction", 

657 ) 

658 

659 # Masking options. 

660 doDefect = pexConfig.Field( 

661 dtype=bool, 

662 doc="Apply correction for CCD defects, e.g. hot pixels?", 

663 default=True, 

664 ) 

665 doNanMasking = pexConfig.Field( 

666 dtype=bool, 

667 doc="Mask non-finite (NAN, inf) pixels?", 

668 default=True, 

669 ) 

670 doWidenSaturationTrails = pexConfig.Field( 

671 dtype=bool, 

672 doc="Widen bleed trails based on their width?", 

673 default=True 

674 ) 

675 

676 # Brighter-Fatter correction. 

677 doBrighterFatter = pexConfig.Field( 

678 dtype=bool, 

679 default=False, 

680 doc="Apply the brighter-fatter correction?" 

681 ) 

682 brighterFatterLevel = pexConfig.ChoiceField( 

683 dtype=str, 

684 default="DETECTOR", 

685 doc="The level at which to correct for brighter-fatter.", 

686 allowed={ 

687 "AMP": "Every amplifier treated separately.", 

688 "DETECTOR": "One kernel per detector", 

689 } 

690 ) 

691 brighterFatterMaxIter = pexConfig.Field( 

692 dtype=int, 

693 default=10, 

694 doc="Maximum number of iterations for the brighter-fatter correction" 

695 ) 

696 brighterFatterThreshold = pexConfig.Field( 

697 dtype=float, 

698 default=1000, 

699 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

700 "absolute value of the difference between the current corrected image and the one " 

701 "from the previous iteration summed over all the pixels." 

702 ) 

703 brighterFatterApplyGain = pexConfig.Field( 

704 dtype=bool, 

705 default=True, 

706 doc="Should the gain be applied when applying the brighter-fatter correction?" 

707 ) 

708 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

709 dtype=str, 

710 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

711 "correction.", 

712 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

713 ) 

714 brighterFatterMaskGrowSize = pexConfig.Field( 

715 dtype=int, 

716 default=0, 

717 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

718 "when brighter-fatter correction is applied." 

719 ) 

720 

721 # Dark subtraction. 

722 doDark = pexConfig.Field( 

723 dtype=bool, 

724 doc="Apply dark frame correction?", 

725 default=True, 

726 ) 

727 darkDataProductName = pexConfig.Field( 

728 dtype=str, 

729 doc="Name of the dark data product", 

730 default="dark", 

731 ) 

732 

733 # Camera-specific stray light removal. 

734 doStrayLight = pexConfig.Field( 

735 dtype=bool, 

736 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

737 default=False, 

738 ) 

739 strayLight = pexConfig.ConfigurableField( 

740 target=StrayLightTask, 

741 doc="y-band stray light correction" 

742 ) 

743 

744 # Flat correction. 

745 doFlat = pexConfig.Field( 

746 dtype=bool, 

747 doc="Apply flat field correction?", 

748 default=True, 

749 ) 

750 flatDataProductName = pexConfig.Field( 

751 dtype=str, 

752 doc="Name of the flat data product", 

753 default="flat", 

754 ) 

755 flatScalingType = pexConfig.ChoiceField( 

756 dtype=str, 

757 doc="The method for scaling the flat on the fly.", 

758 default='USER', 

759 allowed={ 

760 "USER": "Scale by flatUserScale", 

761 "MEAN": "Scale by the inverse of the mean", 

762 "MEDIAN": "Scale by the inverse of the median", 

763 }, 

764 ) 

765 flatUserScale = pexConfig.Field( 

766 dtype=float, 

767 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

768 default=1.0, 

769 ) 

770 doTweakFlat = pexConfig.Field( 

771 dtype=bool, 

772 doc="Tweak flats to match observed amplifier ratios?", 

773 default=False 

774 ) 

775 

776 # Amplifier normalization based on gains instead of using flats 

777 # configuration. 

778 doApplyGains = pexConfig.Field( 

779 dtype=bool, 

780 doc="Correct the amplifiers for their gains instead of applying flat correction", 

781 default=False, 

782 ) 

783 usePtcGains = pexConfig.Field( 

784 dtype=bool, 

785 doc="Use the gain values from the Photon Transfer Curve?", 

786 default=False, 

787 ) 

788 normalizeGains = pexConfig.Field( 

789 dtype=bool, 

790 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

791 default=False, 

792 ) 

793 

794 # Fringe correction. 

795 doFringe = pexConfig.Field( 

796 dtype=bool, 

797 doc="Apply fringe correction?", 

798 default=True, 

799 ) 

800 fringe = pexConfig.ConfigurableField( 

801 target=FringeTask, 

802 doc="Fringe subtraction task", 

803 ) 

804 fringeAfterFlat = pexConfig.Field( 

805 dtype=bool, 

806 doc="Do fringe subtraction after flat-fielding?", 

807 default=True, 

808 ) 

809 

810 # Amp offset correction. 

811 doAmpOffset = pexConfig.Field( 

812 doc="Calculate and apply amp offset corrections?", 

813 dtype=bool, 

814 default=False, 

815 ) 

816 ampOffset = pexConfig.ConfigurableField( 

817 doc="Amp offset correction task.", 

818 target=AmpOffsetTask, 

819 ) 

820 

821 # Initial CCD-level background statistics options. 

822 doMeasureBackground = pexConfig.Field( 

823 dtype=bool, 

824 doc="Measure the background level on the reduced image?", 

825 default=False, 

826 ) 

827 

828 # Camera-specific masking configuration. 

829 doCameraSpecificMasking = pexConfig.Field( 

830 dtype=bool, 

831 doc="Mask camera-specific bad regions?", 

832 default=False, 

833 ) 

834 masking = pexConfig.ConfigurableField( 

835 target=MaskingTask, 

836 doc="Masking task." 

837 ) 

838 

839 # Interpolation options. 

840 doInterpolate = pexConfig.Field( 

841 dtype=bool, 

842 doc="Interpolate masked pixels?", 

843 default=True, 

844 ) 

845 doSaturationInterpolation = pexConfig.Field( 

846 dtype=bool, 

847 doc="Perform interpolation over pixels masked as saturated?" 

848 " NB: This is independent of doSaturation; if that is False this plane" 

849 " will likely be blank, resulting in a no-op here.", 

850 default=True, 

851 ) 

852 doNanInterpolation = pexConfig.Field( 

853 dtype=bool, 

854 doc="Perform interpolation over pixels masked as NaN?" 

855 " NB: This is independent of doNanMasking; if that is False this plane" 

856 " will likely be blank, resulting in a no-op here.", 

857 default=True, 

858 ) 

859 doNanInterpAfterFlat = pexConfig.Field( 

860 dtype=bool, 

861 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

862 "also have to interpolate them before flat-fielding."), 

863 default=False, 

864 ) 

865 maskListToInterpolate = pexConfig.ListField( 

866 dtype=str, 

867 doc="List of mask planes that should be interpolated.", 

868 default=['SAT', 'BAD'], 

869 ) 

870 doSaveInterpPixels = pexConfig.Field( 

871 dtype=bool, 

872 doc="Save a copy of the pre-interpolated pixel values?", 

873 default=False, 

874 ) 

875 

876 # Default photometric calibration options. 

877 fluxMag0T1 = pexConfig.DictField( 

878 keytype=str, 

879 itemtype=float, 

880 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

881 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

882 )) 

883 ) 

884 defaultFluxMag0T1 = pexConfig.Field( 

885 dtype=float, 

886 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

887 default=pow(10.0, 0.4*28.0) 

888 ) 

889 

890 # Vignette correction configuration. 

891 doVignette = pexConfig.Field( 

892 dtype=bool, 

893 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure " 

894 "according to vignetting parameters?"), 

895 default=False, 

896 ) 

897 doMaskVignettePolygon = pexConfig.Field( 

898 dtype=bool, 

899 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette " 

900 "is False"), 

901 default=True, 

902 ) 

903 vignetteValue = pexConfig.Field( 

904 dtype=float, 

905 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.", 

906 optional=True, 

907 default=None, 

908 ) 

909 vignette = pexConfig.ConfigurableField( 

910 target=VignetteTask, 

911 doc="Vignetting task.", 

912 ) 

913 

914 # Transmission curve configuration. 

915 doAttachTransmissionCurve = pexConfig.Field( 

916 dtype=bool, 

917 default=False, 

918 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

919 ) 

920 doUseOpticsTransmission = pexConfig.Field( 

921 dtype=bool, 

922 default=True, 

923 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

924 ) 

925 doUseFilterTransmission = pexConfig.Field( 

926 dtype=bool, 

927 default=True, 

928 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

929 ) 

930 doUseSensorTransmission = pexConfig.Field( 

931 dtype=bool, 

932 default=True, 

933 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

934 ) 

935 doUseAtmosphereTransmission = pexConfig.Field( 

936 dtype=bool, 

937 default=True, 

938 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

939 ) 

940 

941 # Illumination correction. 

942 doIlluminationCorrection = pexConfig.Field( 

943 dtype=bool, 

944 default=False, 

945 doc="Perform illumination correction?" 

946 ) 

947 illuminationCorrectionDataProductName = pexConfig.Field( 

948 dtype=str, 

949 doc="Name of the illumination correction data product.", 

950 default="illumcor", 

951 ) 

952 illumScale = pexConfig.Field( 

953 dtype=float, 

954 doc="Scale factor for the illumination correction.", 

955 default=1.0, 

956 ) 

957 illumFilters = pexConfig.ListField( 

958 dtype=str, 

959 default=[], 

960 doc="Only perform illumination correction for these filters." 

961 ) 

962 

963 # Calculate additional statistics? 

964 doCalculateStatistics = pexConfig.Field( 

965 dtype=bool, 

966 doc="Should additional ISR statistics be calculated?", 

967 default=False, 

968 ) 

969 isrStats = pexConfig.ConfigurableField( 

970 target=IsrStatisticsTask, 

971 doc="Task to calculate additional statistics.", 

972 ) 

973 

974 # Write the outputs to disk. If ISR is run as a subtask, this may not 

975 # be needed. 

976 doWrite = pexConfig.Field( 

977 dtype=bool, 

978 doc="Persist postISRCCD?", 

979 default=True, 

980 ) 

981 

982 def validate(self): 

983 super().validate() 

984 if self.doFlat and self.doApplyGains: 

985 raise ValueError("You may not specify both doFlat and doApplyGains") 

986 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

987 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

988 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

989 self.maskListToInterpolate.append(self.saturatedMaskName) 

990 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

991 self.maskListToInterpolate.remove(self.saturatedMaskName) 

992 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

993 self.maskListToInterpolate.append("UNMASKEDNAN") 

994 

995 

996class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

997 """Apply common instrument signature correction algorithms to a raw frame. 

998 

999 The process for correcting imaging data is very similar from 

1000 camera to camera. This task provides a vanilla implementation of 

1001 doing these corrections, including the ability to turn certain 

1002 corrections off if they are not needed. The inputs to the primary 

1003 method, `run()`, are a raw exposure to be corrected and the 

1004 calibration data products. The raw input is a single chip sized 

1005 mosaic of all amps including overscans and other non-science 

1006 pixels. The method `runDataRef()` identifies and defines the 

1007 calibration data products, and is intended for use by a 

1008 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

1009 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

1010 subclassed for different camera, although the most camera specific 

1011 methods have been split into subtasks that can be redirected 

1012 appropriately. 

1013 

1014 The __init__ method sets up the subtasks for ISR processing, using 

1015 the defaults from `lsst.ip.isr`. 

1016 

1017 Parameters 

1018 ---------- 

1019 args : `list` 

1020 Positional arguments passed to the Task constructor. 

1021 None used at this time. 

1022 kwargs : `dict`, optional 

1023 Keyword arguments passed on to the Task constructor. 

1024 None used at this time. 

1025 """ 

1026 ConfigClass = IsrTaskConfig 

1027 _DefaultName = "isr" 

1028 

1029 def __init__(self, **kwargs): 

1030 super().__init__(**kwargs) 

1031 self.makeSubtask("assembleCcd") 

1032 self.makeSubtask("crosstalk") 

1033 self.makeSubtask("strayLight") 

1034 self.makeSubtask("fringe") 

1035 self.makeSubtask("masking") 

1036 self.makeSubtask("overscan") 

1037 self.makeSubtask("vignette") 

1038 self.makeSubtask("ampOffset") 

1039 self.makeSubtask("deferredChargeCorrection") 

1040 self.makeSubtask("isrStats") 

1041 

1042 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1043 inputs = butlerQC.get(inputRefs) 

1044 

1045 try: 

1046 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

1047 except Exception as e: 

1048 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

1049 (inputRefs, e)) 

1050 

1051 inputs['isGen3'] = True 

1052 

1053 detector = inputs['ccdExposure'].getDetector() 

1054 

1055 if self.config.doCrosstalk is True: 

1056 # Crosstalk sources need to be defined by the pipeline 

1057 # yaml if they exist. 

1058 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

1059 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

1060 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

1061 else: 

1062 coeffVector = (self.config.crosstalk.crosstalkValues 

1063 if self.config.crosstalk.useConfigCoefficients else None) 

1064 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

1065 inputs['crosstalk'] = crosstalkCalib 

1066 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

1067 if 'crosstalkSources' not in inputs: 

1068 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

1069 

1070 if self.doLinearize(detector) is True: 

1071 if 'linearizer' in inputs: 

1072 if isinstance(inputs['linearizer'], dict): 

1073 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1074 linearizer.fromYaml(inputs['linearizer']) 

1075 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1076 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1077 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1078 detector=detector, 

1079 log=self.log) 

1080 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1081 else: 

1082 linearizer = inputs['linearizer'] 

1083 linearizer.log = self.log 

1084 inputs['linearizer'] = linearizer 

1085 else: 

1086 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1087 self.log.warning("Constructing linearizer from cameraGeom information.") 

1088 

1089 if self.config.doDefect is True: 

1090 if "defects" in inputs and inputs['defects'] is not None: 

1091 # defects is loaded as a BaseCatalog with columns 

1092 # x0, y0, width, height. Masking expects a list of defects 

1093 # defined by their bounding box 

1094 if not isinstance(inputs["defects"], Defects): 

1095 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1096 

1097 # Load the correct style of brighter-fatter kernel, and repack 

1098 # the information as a numpy array. 

1099 if self.config.doBrighterFatter: 

1100 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1101 if brighterFatterKernel is None: 

1102 brighterFatterKernel = inputs.get('bfKernel', None) 

1103 

1104 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1105 # This is a ISR calib kernel 

1106 detName = detector.getName() 

1107 level = brighterFatterKernel.level 

1108 

1109 # This is expected to be a dictionary of amp-wise gains. 

1110 inputs['bfGains'] = brighterFatterKernel.gain 

1111 if self.config.brighterFatterLevel == 'DETECTOR': 

1112 if level == 'DETECTOR': 

1113 if detName in brighterFatterKernel.detKernels: 

1114 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1115 else: 

1116 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1117 elif level == 'AMP': 

1118 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1119 "fatter kernels.") 

1120 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1121 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1122 elif self.config.brighterFatterLevel == 'AMP': 

1123 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1124 

1125 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1126 expId = inputs['ccdExposure'].info.id 

1127 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1128 expId=expId, 

1129 assembler=self.assembleCcd 

1130 if self.config.doAssembleIsrExposures else None) 

1131 else: 

1132 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1133 

1134 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1135 if 'strayLightData' not in inputs: 

1136 inputs['strayLightData'] = None 

1137 

1138 outputs = self.run(**inputs) 

1139 butlerQC.put(outputs, outputRefs) 

1140 

1141 def readIsrData(self, dataRef, rawExposure): 

1142 """Retrieve necessary frames for instrument signature removal. 

1143 

1144 Pre-fetching all required ISR data products limits the IO 

1145 required by the ISR. Any conflict between the calibration data 

1146 available and that needed for ISR is also detected prior to 

1147 doing processing, allowing it to fail quickly. 

1148 

1149 Parameters 

1150 ---------- 

1151 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1152 Butler reference of the detector data to be processed 

1153 rawExposure : `afw.image.Exposure` 

1154 The raw exposure that will later be corrected with the 

1155 retrieved calibration data; should not be modified in this 

1156 method. 

1157 

1158 Returns 

1159 ------- 

1160 result : `lsst.pipe.base.Struct` 

1161 Result struct with components (which may be `None`): 

1162 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1163 - ``linearizer``: functor for linearization 

1164 (`ip.isr.linearize.LinearizeBase`) 

1165 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1166 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1167 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1168 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1169 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1170 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1171 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1172 - ``seed``: random seed derived from the ccdExposureId for random 

1173 number generator (`uint32`). 

1174 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1175 A ``TransmissionCurve`` that represents the throughput of the 

1176 optics, to be evaluated in focal-plane coordinates. 

1177 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1178 A ``TransmissionCurve`` that represents the throughput of the 

1179 filter itself, to be evaluated in focal-plane coordinates. 

1180 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1181 A ``TransmissionCurve`` that represents the throughput of the 

1182 sensor itself, to be evaluated in post-assembly trimmed 

1183 detector coordinates. 

1184 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1185 A ``TransmissionCurve`` that represents the throughput of the 

1186 atmosphere, assumed to be spatially constant. 

1187 - ``strayLightData`` : `object` 

1188 An opaque object containing calibration information for 

1189 stray-light correction. If `None`, no correction will be 

1190 performed. 

1191 - ``illumMaskedImage`` : illumination correction image 

1192 (`lsst.afw.image.MaskedImage`) 

1193 

1194 Raises 

1195 ------ 

1196 NotImplementedError : 

1197 Raised if a per-amplifier brighter-fatter kernel is requested by 

1198 the configuration. 

1199 """ 

1200 try: 

1201 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1202 dateObs = dateObs.toPython().isoformat() 

1203 except RuntimeError: 

1204 self.log.warning("Unable to identify dateObs for rawExposure.") 

1205 dateObs = None 

1206 

1207 ccd = rawExposure.getDetector() 

1208 filterLabel = rawExposure.getFilter() 

1209 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1210 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1211 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1212 if self.config.doBias else None) 

1213 # immediate=True required for functors and linearizers are functors 

1214 # see ticket DM-6515 

1215 linearizer = (dataRef.get("linearizer", immediate=True) 

1216 if self.doLinearize(ccd) else None) 

1217 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1218 linearizer.log = self.log 

1219 if isinstance(linearizer, numpy.ndarray): 

1220 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1221 

1222 crosstalkCalib = None 

1223 if self.config.doCrosstalk: 

1224 try: 

1225 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1226 except NoResults: 

1227 coeffVector = (self.config.crosstalk.crosstalkValues 

1228 if self.config.crosstalk.useConfigCoefficients else None) 

1229 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1230 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1231 if self.config.doCrosstalk else None) 

1232 

1233 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1234 if self.config.doDark else None) 

1235 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1236 dateObs=dateObs) 

1237 if self.config.doFlat else None) 

1238 

1239 brighterFatterKernel = None 

1240 brighterFatterGains = None 

1241 if self.config.doBrighterFatter is True: 

1242 try: 

1243 # Use the new-style cp_pipe version of the kernel if it exists 

1244 # If using a new-style kernel, always use the self-consistent 

1245 # gains, i.e. the ones inside the kernel object itself 

1246 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1247 brighterFatterGains = brighterFatterKernel.gain 

1248 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded") 

1249 except NoResults: 

1250 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1251 brighterFatterKernel = dataRef.get("bfKernel") 

1252 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded") 

1253 except NoResults: 

1254 brighterFatterKernel = None 

1255 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1256 # If the kernel is not an ndarray, it's the cp_pipe version 

1257 # so extract the kernel for this detector, or raise an error 

1258 if self.config.brighterFatterLevel == 'DETECTOR': 

1259 if brighterFatterKernel.detKernels: 

1260 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()] 

1261 else: 

1262 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1263 else: 

1264 # TODO DM-15631 for implementing this 

1265 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1266 

1267 defectList = (dataRef.get("defects") 

1268 if self.config.doDefect else None) 

1269 expId = rawExposure.info.id 

1270 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd 

1271 if self.config.doAssembleIsrExposures else None) 

1272 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1273 else pipeBase.Struct(fringes=None)) 

1274 

1275 if self.config.doAttachTransmissionCurve: 

1276 opticsTransmission = (dataRef.get("transmission_optics") 

1277 if self.config.doUseOpticsTransmission else None) 

1278 filterTransmission = (dataRef.get("transmission_filter") 

1279 if self.config.doUseFilterTransmission else None) 

1280 sensorTransmission = (dataRef.get("transmission_sensor") 

1281 if self.config.doUseSensorTransmission else None) 

1282 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1283 if self.config.doUseAtmosphereTransmission else None) 

1284 else: 

1285 opticsTransmission = None 

1286 filterTransmission = None 

1287 sensorTransmission = None 

1288 atmosphereTransmission = None 

1289 

1290 if self.config.doStrayLight: 

1291 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1292 else: 

1293 strayLightData = None 

1294 

1295 illumMaskedImage = (self.getIsrExposure(dataRef, 

1296 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1297 if (self.config.doIlluminationCorrection 

1298 and physicalFilter in self.config.illumFilters) 

1299 else None) 

1300 

1301 # Struct should include only kwargs to run() 

1302 return pipeBase.Struct(bias=biasExposure, 

1303 linearizer=linearizer, 

1304 crosstalk=crosstalkCalib, 

1305 crosstalkSources=crosstalkSources, 

1306 dark=darkExposure, 

1307 flat=flatExposure, 

1308 bfKernel=brighterFatterKernel, 

1309 bfGains=brighterFatterGains, 

1310 defects=defectList, 

1311 fringes=fringeStruct, 

1312 opticsTransmission=opticsTransmission, 

1313 filterTransmission=filterTransmission, 

1314 sensorTransmission=sensorTransmission, 

1315 atmosphereTransmission=atmosphereTransmission, 

1316 strayLightData=strayLightData, 

1317 illumMaskedImage=illumMaskedImage 

1318 ) 

1319 

1320 @timeMethod 

1321 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1322 crosstalk=None, crosstalkSources=None, 

1323 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1324 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1325 sensorTransmission=None, atmosphereTransmission=None, 

1326 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1327 deferredCharge=None, isGen3=False, 

1328 ): 

1329 """Perform instrument signature removal on an exposure. 

1330 

1331 Steps included in the ISR processing, in order performed, are: 

1332 - saturation and suspect pixel masking 

1333 - overscan subtraction 

1334 - CCD assembly of individual amplifiers 

1335 - bias subtraction 

1336 - variance image construction 

1337 - linearization of non-linear response 

1338 - crosstalk masking 

1339 - brighter-fatter correction 

1340 - dark subtraction 

1341 - fringe correction 

1342 - stray light subtraction 

1343 - flat correction 

1344 - masking of known defects and camera specific features 

1345 - vignette calculation 

1346 - appending transmission curve and distortion model 

1347 

1348 Parameters 

1349 ---------- 

1350 ccdExposure : `lsst.afw.image.Exposure` 

1351 The raw exposure that is to be run through ISR. The 

1352 exposure is modified by this method. 

1353 camera : `lsst.afw.cameraGeom.Camera`, optional 

1354 The camera geometry for this exposure. Required if 

1355 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1356 ``flat`` does not have an associated detector. 

1357 bias : `lsst.afw.image.Exposure`, optional 

1358 Bias calibration frame. 

1359 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1360 Functor for linearization. 

1361 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1362 Calibration for crosstalk. 

1363 crosstalkSources : `list`, optional 

1364 List of possible crosstalk sources. 

1365 dark : `lsst.afw.image.Exposure`, optional 

1366 Dark calibration frame. 

1367 flat : `lsst.afw.image.Exposure`, optional 

1368 Flat calibration frame. 

1369 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1370 Photon transfer curve dataset, with, e.g., gains 

1371 and read noise. 

1372 bfKernel : `numpy.ndarray`, optional 

1373 Brighter-fatter kernel. 

1374 bfGains : `dict` of `float`, optional 

1375 Gains used to override the detector's nominal gains for the 

1376 brighter-fatter correction. A dict keyed by amplifier name for 

1377 the detector in question. 

1378 defects : `lsst.ip.isr.Defects`, optional 

1379 List of defects. 

1380 fringes : `lsst.pipe.base.Struct`, optional 

1381 Struct containing the fringe correction data, with 

1382 elements: 

1383 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1384 - ``seed``: random seed derived from the ccdExposureId for random 

1385 number generator (`uint32`) 

1386 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1387 A ``TransmissionCurve`` that represents the throughput of the, 

1388 optics, to be evaluated in focal-plane coordinates. 

1389 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1390 A ``TransmissionCurve`` that represents the throughput of the 

1391 filter itself, to be evaluated in focal-plane coordinates. 

1392 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1393 A ``TransmissionCurve`` that represents the throughput of the 

1394 sensor itself, to be evaluated in post-assembly trimmed detector 

1395 coordinates. 

1396 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1397 A ``TransmissionCurve`` that represents the throughput of the 

1398 atmosphere, assumed to be spatially constant. 

1399 detectorNum : `int`, optional 

1400 The integer number for the detector to process. 

1401 isGen3 : bool, optional 

1402 Flag this call to run() as using the Gen3 butler environment. 

1403 strayLightData : `object`, optional 

1404 Opaque object containing calibration information for stray-light 

1405 correction. If `None`, no correction will be performed. 

1406 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1407 Illumination correction image. 

1408 

1409 Returns 

1410 ------- 

1411 result : `lsst.pipe.base.Struct` 

1412 Result struct with component: 

1413 - ``exposure`` : `afw.image.Exposure` 

1414 The fully ISR corrected exposure. 

1415 - ``outputExposure`` : `afw.image.Exposure` 

1416 An alias for `exposure` 

1417 - ``ossThumb`` : `numpy.ndarray` 

1418 Thumbnail image of the exposure after overscan subtraction. 

1419 - ``flattenedThumb`` : `numpy.ndarray` 

1420 Thumbnail image of the exposure after flat-field correction. 

1421 - ``outputStatistics`` : `` 

1422 Values of the additional statistics calculated. 

1423 

1424 Raises 

1425 ------ 

1426 RuntimeError 

1427 Raised if a configuration option is set to True, but the 

1428 required calibration data has not been specified. 

1429 

1430 Notes 

1431 ----- 

1432 The current processed exposure can be viewed by setting the 

1433 appropriate lsstDebug entries in the `debug.display` 

1434 dictionary. The names of these entries correspond to some of 

1435 the IsrTaskConfig Boolean options, with the value denoting the 

1436 frame to use. The exposure is shown inside the matching 

1437 option check and after the processing of that step has 

1438 finished. The steps with debug points are: 

1439 

1440 doAssembleCcd 

1441 doBias 

1442 doCrosstalk 

1443 doBrighterFatter 

1444 doDark 

1445 doFringe 

1446 doStrayLight 

1447 doFlat 

1448 

1449 In addition, setting the "postISRCCD" entry displays the 

1450 exposure after all ISR processing has finished. 

1451 

1452 """ 

1453 

1454 if isGen3 is True: 

1455 # Gen3 currently cannot automatically do configuration overrides. 

1456 # DM-15257 looks to discuss this issue. 

1457 # Configure input exposures; 

1458 

1459 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1460 bias = self.ensureExposure(bias, camera, detectorNum) 

1461 dark = self.ensureExposure(dark, camera, detectorNum) 

1462 flat = self.ensureExposure(flat, camera, detectorNum) 

1463 else: 

1464 if isinstance(ccdExposure, ButlerDataRef): 

1465 return self.runDataRef(ccdExposure) 

1466 

1467 ccd = ccdExposure.getDetector() 

1468 filterLabel = ccdExposure.getFilter() 

1469 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1470 

1471 if not ccd: 

1472 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1473 ccd = [FakeAmp(ccdExposure, self.config)] 

1474 

1475 # Validate Input 

1476 if self.config.doBias and bias is None: 

1477 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1478 if self.doLinearize(ccd) and linearizer is None: 

1479 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1480 if self.config.doBrighterFatter and bfKernel is None: 

1481 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1482 if self.config.doDark and dark is None: 

1483 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1484 if self.config.doFlat and flat is None: 

1485 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1486 if self.config.doDefect and defects is None: 

1487 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1488 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1489 and fringes.fringes is None): 

1490 # The `fringes` object needs to be a pipeBase.Struct, as 

1491 # we use it as a `dict` for the parameters of 

1492 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1493 # not be `None` if `doFringe=True`. Otherwise, raise. 

1494 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1495 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1496 and illumMaskedImage is None): 

1497 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1498 if (self.config.doDeferredCharge and deferredCharge is None): 

1499 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.") 

1500 

1501 # Begin ISR processing. 

1502 if self.config.doConvertIntToFloat: 

1503 self.log.info("Converting exposure to floating point values.") 

1504 ccdExposure = self.convertIntToFloat(ccdExposure) 

1505 

1506 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1507 self.log.info("Applying bias correction.") 

1508 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1509 trimToFit=self.config.doTrimToMatchCalib) 

1510 self.debugView(ccdExposure, "doBias") 

1511 

1512 # Amplifier level processing. 

1513 overscans = [] 

1514 for amp in ccd: 

1515 # if ccdExposure is one amp, 

1516 # check for coverage to prevent performing ops multiple times 

1517 if ccdExposure.getBBox().contains(amp.getBBox()): 

1518 # Check for fully masked bad amplifiers, 

1519 # and generate masks for SUSPECT and SATURATED values. 

1520 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1521 

1522 if self.config.doOverscan and not badAmp: 

1523 # Overscan correction on amp-by-amp basis. 

1524 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1525 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1526 if overscanResults is not None and \ 

1527 self.config.qa is not None and self.config.qa.saveStats is True: 

1528 if isinstance(overscanResults.overscanFit, float): 

1529 qaMedian = overscanResults.overscanFit 

1530 qaStdev = float("NaN") 

1531 else: 

1532 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1533 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1534 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1535 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1536 

1537 self.metadata[f"FIT MEDIAN {amp.getName()}"] = qaMedian 

1538 self.metadata[f"FIT STDEV {amp.getName()}"] = qaStdev 

1539 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1540 amp.getName(), qaMedian, qaStdev) 

1541 

1542 # Residuals after overscan correction 

1543 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1544 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1545 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1546 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1547 

1548 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter 

1549 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter 

1550 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1551 amp.getName(), qaMedianAfter, qaStdevAfter) 

1552 

1553 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1554 else: 

1555 if badAmp: 

1556 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1557 overscanResults = None 

1558 

1559 overscans.append(overscanResults if overscanResults is not None else None) 

1560 else: 

1561 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1562 

1563 if self.config.doDeferredCharge: 

1564 self.log.info("Applying deferred charge/CTI correction.") 

1565 self.deferredChargeCorrection.run(ccdExposure, deferredCharge) 

1566 self.debugView(ccdExposure, "doDeferredCharge") 

1567 

1568 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1569 self.log.info("Applying crosstalk correction.") 

1570 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1571 crosstalkSources=crosstalkSources, camera=camera) 

1572 self.debugView(ccdExposure, "doCrosstalk") 

1573 

1574 if self.config.doAssembleCcd: 

1575 self.log.info("Assembling CCD from amplifiers.") 

1576 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1577 

1578 if self.config.expectWcs and not ccdExposure.getWcs(): 

1579 self.log.warning("No WCS found in input exposure.") 

1580 self.debugView(ccdExposure, "doAssembleCcd") 

1581 

1582 ossThumb = None 

1583 if self.config.qa.doThumbnailOss: 

1584 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1585 

1586 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1587 self.log.info("Applying bias correction.") 

1588 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1589 trimToFit=self.config.doTrimToMatchCalib) 

1590 self.debugView(ccdExposure, "doBias") 

1591 

1592 if self.config.doVariance: 

1593 for amp, overscanResults in zip(ccd, overscans): 

1594 if ccdExposure.getBBox().contains(amp.getBBox()): 

1595 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1596 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1597 if overscanResults is not None: 

1598 self.updateVariance(ampExposure, amp, 

1599 overscanImage=overscanResults.overscanImage, 

1600 ptcDataset=ptc) 

1601 else: 

1602 self.updateVariance(ampExposure, amp, 

1603 overscanImage=None, 

1604 ptcDataset=ptc) 

1605 if self.config.qa is not None and self.config.qa.saveStats is True: 

1606 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1607 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1608 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \ 

1609 qaStats.getValue(afwMath.MEDIAN) 

1610 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \ 

1611 qaStats.getValue(afwMath.STDEVCLIP) 

1612 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1613 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1614 qaStats.getValue(afwMath.STDEVCLIP)) 

1615 if self.config.maskNegativeVariance: 

1616 self.maskNegativeVariance(ccdExposure) 

1617 

1618 if self.doLinearize(ccd): 

1619 self.log.info("Applying linearizer.") 

1620 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1621 detector=ccd, log=self.log) 

1622 

1623 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1624 self.log.info("Applying crosstalk correction.") 

1625 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1626 crosstalkSources=crosstalkSources, isTrimmed=True) 

1627 self.debugView(ccdExposure, "doCrosstalk") 

1628 

1629 # Masking block. Optionally mask known defects, NAN/inf pixels, 

1630 # widen trails, and do anything else the camera needs. Saturated and 

1631 # suspect pixels have already been masked. 

1632 if self.config.doDefect: 

1633 self.log.info("Masking defects.") 

1634 self.maskDefect(ccdExposure, defects) 

1635 

1636 if self.config.numEdgeSuspect > 0: 

1637 self.log.info("Masking edges as SUSPECT.") 

1638 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1639 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1640 

1641 if self.config.doNanMasking: 

1642 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1643 self.maskNan(ccdExposure) 

1644 

1645 if self.config.doWidenSaturationTrails: 

1646 self.log.info("Widening saturation trails.") 

1647 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1648 

1649 if self.config.doCameraSpecificMasking: 

1650 self.log.info("Masking regions for camera specific reasons.") 

1651 self.masking.run(ccdExposure) 

1652 

1653 if self.config.doBrighterFatter: 

1654 # We need to apply flats and darks before we can interpolate, and 

1655 # we need to interpolate before we do B-F, but we do B-F without 

1656 # the flats and darks applied so we can work in units of electrons 

1657 # or holes. This context manager applies and then removes the darks 

1658 # and flats. 

1659 # 

1660 # We also do not want to interpolate values here, so operate on 

1661 # temporary images so we can apply only the BF-correction and roll 

1662 # back the interpolation. 

1663 interpExp = ccdExposure.clone() 

1664 with self.flatContext(interpExp, flat, dark): 

1665 isrFunctions.interpolateFromMask( 

1666 maskedImage=interpExp.getMaskedImage(), 

1667 fwhm=self.config.fwhm, 

1668 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1669 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1670 ) 

1671 bfExp = interpExp.clone() 

1672 

1673 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1674 type(bfKernel), type(bfGains)) 

1675 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1676 self.config.brighterFatterMaxIter, 

1677 self.config.brighterFatterThreshold, 

1678 self.config.brighterFatterApplyGain, 

1679 bfGains) 

1680 if bfResults[1] == self.config.brighterFatterMaxIter: 

1681 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1682 bfResults[0]) 

1683 else: 

1684 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1685 bfResults[1]) 

1686 image = ccdExposure.getMaskedImage().getImage() 

1687 bfCorr = bfExp.getMaskedImage().getImage() 

1688 bfCorr -= interpExp.getMaskedImage().getImage() 

1689 image += bfCorr 

1690 

1691 # Applying the brighter-fatter correction applies a 

1692 # convolution to the science image. At the edges this 

1693 # convolution may not have sufficient valid pixels to 

1694 # produce a valid correction. Mark pixels within the size 

1695 # of the brighter-fatter kernel as EDGE to warn of this 

1696 # fact. 

1697 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1698 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1699 maskPlane="EDGE") 

1700 

1701 if self.config.brighterFatterMaskGrowSize > 0: 

1702 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1703 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1704 isrFunctions.growMasks(ccdExposure.getMask(), 

1705 radius=self.config.brighterFatterMaskGrowSize, 

1706 maskNameList=maskPlane, 

1707 maskValue=maskPlane) 

1708 

1709 self.debugView(ccdExposure, "doBrighterFatter") 

1710 

1711 if self.config.doDark: 

1712 self.log.info("Applying dark correction.") 

1713 self.darkCorrection(ccdExposure, dark) 

1714 self.debugView(ccdExposure, "doDark") 

1715 

1716 if self.config.doFringe and not self.config.fringeAfterFlat: 

1717 self.log.info("Applying fringe correction before flat.") 

1718 self.fringe.run(ccdExposure, **fringes.getDict()) 

1719 self.debugView(ccdExposure, "doFringe") 

1720 

1721 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1722 self.log.info("Checking strayLight correction.") 

1723 self.strayLight.run(ccdExposure, strayLightData) 

1724 self.debugView(ccdExposure, "doStrayLight") 

1725 

1726 if self.config.doFlat: 

1727 self.log.info("Applying flat correction.") 

1728 self.flatCorrection(ccdExposure, flat) 

1729 self.debugView(ccdExposure, "doFlat") 

1730 

1731 if self.config.doApplyGains: 

1732 self.log.info("Applying gain correction instead of flat.") 

1733 if self.config.usePtcGains: 

1734 self.log.info("Using gains from the Photon Transfer Curve.") 

1735 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1736 ptcGains=ptc.gain) 

1737 else: 

1738 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1739 

1740 if self.config.doFringe and self.config.fringeAfterFlat: 

1741 self.log.info("Applying fringe correction after flat.") 

1742 self.fringe.run(ccdExposure, **fringes.getDict()) 

1743 

1744 if self.config.doVignette: 

1745 if self.config.doMaskVignettePolygon: 

1746 self.log.info("Constructing, attaching, and masking vignette polygon.") 

1747 else: 

1748 self.log.info("Constructing and attaching vignette polygon.") 

1749 self.vignettePolygon = self.vignette.run( 

1750 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon, 

1751 vignetteValue=self.config.vignetteValue, log=self.log) 

1752 

1753 if self.config.doAttachTransmissionCurve: 

1754 self.log.info("Adding transmission curves.") 

1755 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1756 filterTransmission=filterTransmission, 

1757 sensorTransmission=sensorTransmission, 

1758 atmosphereTransmission=atmosphereTransmission) 

1759 

1760 flattenedThumb = None 

1761 if self.config.qa.doThumbnailFlattened: 

1762 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1763 

1764 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1765 self.log.info("Performing illumination correction.") 

1766 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1767 illumMaskedImage, illumScale=self.config.illumScale, 

1768 trimToFit=self.config.doTrimToMatchCalib) 

1769 

1770 preInterpExp = None 

1771 if self.config.doSaveInterpPixels: 

1772 preInterpExp = ccdExposure.clone() 

1773 

1774 # Reset and interpolate bad pixels. 

1775 # 

1776 # Large contiguous bad regions (which should have the BAD mask 

1777 # bit set) should have their values set to the image median. 

1778 # This group should include defects and bad amplifiers. As the 

1779 # area covered by these defects are large, there's little 

1780 # reason to expect that interpolation would provide a more 

1781 # useful value. 

1782 # 

1783 # Smaller defects can be safely interpolated after the larger 

1784 # regions have had their pixel values reset. This ensures 

1785 # that the remaining defects adjacent to bad amplifiers (as an 

1786 # example) do not attempt to interpolate extreme values. 

1787 if self.config.doSetBadRegions: 

1788 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1789 if badPixelCount > 0: 

1790 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1791 

1792 if self.config.doInterpolate: 

1793 self.log.info("Interpolating masked pixels.") 

1794 isrFunctions.interpolateFromMask( 

1795 maskedImage=ccdExposure.getMaskedImage(), 

1796 fwhm=self.config.fwhm, 

1797 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1798 maskNameList=list(self.config.maskListToInterpolate) 

1799 ) 

1800 

1801 self.roughZeroPoint(ccdExposure) 

1802 

1803 # correct for amp offsets within the CCD 

1804 if self.config.doAmpOffset: 

1805 self.log.info("Correcting amp offsets.") 

1806 self.ampOffset.run(ccdExposure) 

1807 

1808 if self.config.doMeasureBackground: 

1809 self.log.info("Measuring background level.") 

1810 self.measureBackground(ccdExposure, self.config.qa) 

1811 

1812 if self.config.qa is not None and self.config.qa.saveStats is True: 

1813 for amp in ccd: 

1814 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1815 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1816 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1817 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN) 

1818 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \ 

1819 qaStats.getValue(afwMath.STDEVCLIP) 

1820 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1821 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1822 qaStats.getValue(afwMath.STDEVCLIP)) 

1823 

1824 # calculate additional statistics. 

1825 outputStatistics = None 

1826 if self.config.doCalculateStatistics: 

1827 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans, 

1828 ptc=ptc).results 

1829 

1830 self.debugView(ccdExposure, "postISRCCD") 

1831 

1832 return pipeBase.Struct( 

1833 exposure=ccdExposure, 

1834 ossThumb=ossThumb, 

1835 flattenedThumb=flattenedThumb, 

1836 

1837 preInterpExposure=preInterpExp, 

1838 outputExposure=ccdExposure, 

1839 outputOssThumbnail=ossThumb, 

1840 outputFlattenedThumbnail=flattenedThumb, 

1841 outputStatistics=outputStatistics, 

1842 ) 

1843 

1844 @timeMethod 

1845 def runDataRef(self, sensorRef): 

1846 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1847 

1848 This method contains the `CmdLineTask` interface to the ISR 

1849 processing. All IO is handled here, freeing the `run()` method 

1850 to manage only pixel-level calculations. The steps performed 

1851 are: 

1852 - Read in necessary detrending/isr/calibration data. 

1853 - Process raw exposure in `run()`. 

1854 - Persist the ISR-corrected exposure as "postISRCCD" if 

1855 config.doWrite=True. 

1856 

1857 Parameters 

1858 ---------- 

1859 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1860 DataRef of the detector data to be processed 

1861 

1862 Returns 

1863 ------- 

1864 result : `lsst.pipe.base.Struct` 

1865 Result struct with component: 

1866 - ``exposure`` : `afw.image.Exposure` 

1867 The fully ISR corrected exposure. 

1868 

1869 Raises 

1870 ------ 

1871 RuntimeError 

1872 Raised if a configuration option is set to True, but the 

1873 required calibration data does not exist. 

1874 

1875 """ 

1876 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1877 

1878 ccdExposure = sensorRef.get(self.config.datasetType) 

1879 

1880 camera = sensorRef.get("camera") 

1881 isrData = self.readIsrData(sensorRef, ccdExposure) 

1882 

1883 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1884 

1885 if self.config.doWrite: 

1886 sensorRef.put(result.exposure, "postISRCCD") 

1887 if result.preInterpExposure is not None: 

1888 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated") 

1889 if result.ossThumb is not None: 

1890 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1891 if result.flattenedThumb is not None: 

1892 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1893 

1894 return result 

1895 

1896 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1897 """Retrieve a calibration dataset for removing instrument signature. 

1898 

1899 Parameters 

1900 ---------- 

1901 

1902 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1903 DataRef of the detector data to find calibration datasets 

1904 for. 

1905 datasetType : `str` 

1906 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1907 dateObs : `str`, optional 

1908 Date of the observation. Used to correct butler failures 

1909 when using fallback filters. 

1910 immediate : `Bool` 

1911 If True, disable butler proxies to enable error handling 

1912 within this routine. 

1913 

1914 Returns 

1915 ------- 

1916 exposure : `lsst.afw.image.Exposure` 

1917 Requested calibration frame. 

1918 

1919 Raises 

1920 ------ 

1921 RuntimeError 

1922 Raised if no matching calibration frame can be found. 

1923 """ 

1924 try: 

1925 exp = dataRef.get(datasetType, immediate=immediate) 

1926 except Exception as exc1: 

1927 if not self.config.fallbackFilterName: 

1928 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1929 try: 

1930 if self.config.useFallbackDate and dateObs: 

1931 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1932 dateObs=dateObs, immediate=immediate) 

1933 else: 

1934 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1935 except Exception as exc2: 

1936 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1937 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1938 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1939 

1940 if self.config.doAssembleIsrExposures: 

1941 exp = self.assembleCcd.assembleCcd(exp) 

1942 return exp 

1943 

1944 def ensureExposure(self, inputExp, camera=None, detectorNum=None): 

1945 """Ensure that the data returned by Butler is a fully constructed exp. 

1946 

1947 ISR requires exposure-level image data for historical reasons, so if we 

1948 did not recieve that from Butler, construct it from what we have, 

1949 modifying the input in place. 

1950 

1951 Parameters 

1952 ---------- 

1953 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, 

1954 or `lsst.afw.image.ImageF` 

1955 The input data structure obtained from Butler. 

1956 camera : `lsst.afw.cameraGeom.camera`, optional 

1957 The camera associated with the image. Used to find the appropriate 

1958 detector if detector is not already set. 

1959 detectorNum : `int`, optional 

1960 The detector in the camera to attach, if the detector is not 

1961 already set. 

1962 

1963 Returns 

1964 ------- 

1965 inputExp : `lsst.afw.image.Exposure` 

1966 The re-constructed exposure, with appropriate detector parameters. 

1967 

1968 Raises 

1969 ------ 

1970 TypeError 

1971 Raised if the input data cannot be used to construct an exposure. 

1972 """ 

1973 if isinstance(inputExp, afwImage.DecoratedImageU): 

1974 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1975 elif isinstance(inputExp, afwImage.ImageF): 

1976 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1977 elif isinstance(inputExp, afwImage.MaskedImageF): 

1978 inputExp = afwImage.makeExposure(inputExp) 

1979 elif isinstance(inputExp, afwImage.Exposure): 

1980 pass 

1981 elif inputExp is None: 

1982 # Assume this will be caught by the setup if it is a problem. 

1983 return inputExp 

1984 else: 

1985 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1986 (type(inputExp), )) 

1987 

1988 if inputExp.getDetector() is None: 

1989 if camera is None or detectorNum is None: 

1990 raise RuntimeError('Must supply both a camera and detector number when using exposures ' 

1991 'without a detector set.') 

1992 inputExp.setDetector(camera[detectorNum]) 

1993 

1994 return inputExp 

1995 

1996 def convertIntToFloat(self, exposure): 

1997 """Convert exposure image from uint16 to float. 

1998 

1999 If the exposure does not need to be converted, the input is 

2000 immediately returned. For exposures that are converted to use 

2001 floating point pixels, the variance is set to unity and the 

2002 mask to zero. 

2003 

2004 Parameters 

2005 ---------- 

2006 exposure : `lsst.afw.image.Exposure` 

2007 The raw exposure to be converted. 

2008 

2009 Returns 

2010 ------- 

2011 newexposure : `lsst.afw.image.Exposure` 

2012 The input ``exposure``, converted to floating point pixels. 

2013 

2014 Raises 

2015 ------ 

2016 RuntimeError 

2017 Raised if the exposure type cannot be converted to float. 

2018 

2019 """ 

2020 if isinstance(exposure, afwImage.ExposureF): 

2021 # Nothing to be done 

2022 self.log.debug("Exposure already of type float.") 

2023 return exposure 

2024 if not hasattr(exposure, "convertF"): 

2025 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

2026 

2027 newexposure = exposure.convertF() 

2028 newexposure.variance[:] = 1 

2029 newexposure.mask[:] = 0x0 

2030 

2031 return newexposure 

2032 

2033 def maskAmplifier(self, ccdExposure, amp, defects): 

2034 """Identify bad amplifiers, saturated and suspect pixels. 

2035 

2036 Parameters 

2037 ---------- 

2038 ccdExposure : `lsst.afw.image.Exposure` 

2039 Input exposure to be masked. 

2040 amp : `lsst.afw.table.AmpInfoCatalog` 

2041 Catalog of parameters defining the amplifier on this 

2042 exposure to mask. 

2043 defects : `lsst.ip.isr.Defects` 

2044 List of defects. Used to determine if the entire 

2045 amplifier is bad. 

2046 

2047 Returns 

2048 ------- 

2049 badAmp : `Bool` 

2050 If this is true, the entire amplifier area is covered by 

2051 defects and unusable. 

2052 

2053 """ 

2054 maskedImage = ccdExposure.getMaskedImage() 

2055 

2056 badAmp = False 

2057 

2058 # Check if entire amp region is defined as a defect 

2059 # NB: need to use amp.getBBox() for correct comparison with current 

2060 # defects definition. 

2061 if defects is not None: 

2062 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

2063 

2064 # In the case of a bad amp, we will set mask to "BAD" 

2065 # (here use amp.getRawBBox() for correct association with pixels in 

2066 # current ccdExposure). 

2067 if badAmp: 

2068 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

2069 afwImage.PARENT) 

2070 maskView = dataView.getMask() 

2071 maskView |= maskView.getPlaneBitMask("BAD") 

2072 del maskView 

2073 return badAmp 

2074 

2075 # Mask remaining defects after assembleCcd() to allow for defects that 

2076 # cross amplifier boundaries. Saturation and suspect pixels can be 

2077 # masked now, though. 

2078 limits = dict() 

2079 if self.config.doSaturation and not badAmp: 

2080 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

2081 if self.config.doSuspect and not badAmp: 

2082 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

2083 if math.isfinite(self.config.saturation): 

2084 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

2085 

2086 for maskName, maskThreshold in limits.items(): 

2087 if not math.isnan(maskThreshold): 

2088 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2089 isrFunctions.makeThresholdMask( 

2090 maskedImage=dataView, 

2091 threshold=maskThreshold, 

2092 growFootprints=0, 

2093 maskName=maskName 

2094 ) 

2095 

2096 # Determine if we've fully masked this amplifier with SUSPECT and 

2097 # SAT pixels. 

2098 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

2099 afwImage.PARENT) 

2100 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

2101 self.config.suspectMaskName]) 

2102 if numpy.all(maskView.getArray() & maskVal > 0): 

2103 badAmp = True 

2104 maskView |= maskView.getPlaneBitMask("BAD") 

2105 

2106 return badAmp 

2107 

2108 def overscanCorrection(self, ccdExposure, amp): 

2109 """Apply overscan correction in place. 

2110 

2111 This method does initial pixel rejection of the overscan 

2112 region. The overscan can also be optionally segmented to 

2113 allow for discontinuous overscan responses to be fit 

2114 separately. The actual overscan subtraction is performed by 

2115 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

2116 which is called here after the amplifier is preprocessed. 

2117 

2118 Parameters 

2119 ---------- 

2120 ccdExposure : `lsst.afw.image.Exposure` 

2121 Exposure to have overscan correction performed. 

2122 amp : `lsst.afw.cameraGeom.Amplifer` 

2123 The amplifier to consider while correcting the overscan. 

2124 

2125 Returns 

2126 ------- 

2127 overscanResults : `lsst.pipe.base.Struct` 

2128 Result struct with components: 

2129 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

2130 Value or fit subtracted from the amplifier image data. 

2131 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

2132 Value or fit subtracted from the overscan image data. 

2133 - ``overscanImage`` : `lsst.afw.image.Image` 

2134 Image of the overscan region with the overscan 

2135 correction applied. This quantity is used to estimate 

2136 the amplifier read noise empirically. 

2137 

2138 Raises 

2139 ------ 

2140 RuntimeError 

2141 Raised if the ``amp`` does not contain raw pixel information. 

2142 

2143 See Also 

2144 -------- 

2145 lsst.ip.isr.isrFunctions.overscanCorrection 

2146 """ 

2147 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2148 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2149 return None 

2150 

2151 statControl = afwMath.StatisticsControl() 

2152 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2153 

2154 # Determine the bounding boxes 

2155 dataBBox = amp.getRawDataBBox() 

2156 oscanBBox = amp.getRawHorizontalOverscanBBox() 

2157 dx0 = 0 

2158 dx1 = 0 

2159 

2160 prescanBBox = amp.getRawPrescanBBox() 

2161 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

2162 dx0 += self.config.overscanNumLeadingColumnsToSkip 

2163 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

2164 else: 

2165 dx0 += self.config.overscanNumTrailingColumnsToSkip 

2166 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

2167 

2168 # Determine if we need to work on subregions of the amplifier 

2169 # and overscan. 

2170 imageBBoxes = [] 

2171 overscanBBoxes = [] 

2172 

2173 if ((self.config.overscanBiasJump 

2174 and self.config.overscanBiasJumpLocation) 

2175 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

2176 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

2177 self.config.overscanBiasJumpDevices)): 

2178 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

2179 yLower = self.config.overscanBiasJumpLocation 

2180 yUpper = dataBBox.getHeight() - yLower 

2181 else: 

2182 yUpper = self.config.overscanBiasJumpLocation 

2183 yLower = dataBBox.getHeight() - yUpper 

2184 

2185 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2186 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

2187 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2188 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2189 yLower))) 

2190 

2191 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

2192 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

2193 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

2194 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2195 yUpper))) 

2196 else: 

2197 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2198 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

2199 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2200 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2201 oscanBBox.getHeight()))) 

2202 

2203 # Perform overscan correction on subregions, ensuring saturated 

2204 # pixels are masked. 

2205 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

2206 ampImage = ccdExposure.maskedImage[imageBBox] 

2207 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2208 

2209 overscanArray = overscanImage.image.array 

2210 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2211 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2212 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2213 

2214 statControl = afwMath.StatisticsControl() 

2215 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2216 

2217 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2218 

2219 # If we trimmed columns, we need to restore them. 

2220 if dx0 != 0 or dx1 != 0: 

2221 fullOverscan = ccdExposure.maskedImage[oscanBBox] 

2222 overscanVector = overscanResults.overscanFit.array[:, 0] 

2223 overscanModel = afwImage.ImageF(fullOverscan.getDimensions()) 

2224 overscanModel.array[:, :] = 0.0 

2225 overscanModel.array[:, 0:dx0] = overscanVector[:, numpy.newaxis] 

2226 overscanModel.array[:, dx1:] = overscanVector[:, numpy.newaxis] 

2227 fullOverscanImage = fullOverscan.getImage() 

2228 fullOverscanImage -= overscanModel 

2229 overscanResults = pipeBase.Struct(imageFit=overscanResults.imageFit, 

2230 overscanFit=overscanModel, 

2231 overscanImage=fullOverscan, 

2232 edgeMask=overscanResults.edgeMask) 

2233 

2234 # Measure average overscan levels and record them in the metadata. 

2235 levelStat = afwMath.MEDIAN 

2236 sigmaStat = afwMath.STDEVCLIP 

2237 

2238 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2239 self.config.qa.flatness.nIter) 

2240 metadata = ccdExposure.getMetadata() 

2241 ampNum = amp.getName() 

2242 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2243 if isinstance(overscanResults.overscanFit, float): 

2244 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit 

2245 metadata[f"ISR_OSCAN_SIGMA{ampNum}"] = 0.0 

2246 else: 

2247 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2248 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat) 

2249 metadata[f"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat) 

2250 

2251 return overscanResults 

2252 

2253 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2254 """Set the variance plane using the gain and read noise 

2255 

2256 The read noise is calculated from the ``overscanImage`` if the 

2257 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2258 the value from the amplifier data is used. 

2259 

2260 Parameters 

2261 ---------- 

2262 ampExposure : `lsst.afw.image.Exposure` 

2263 Exposure to process. 

2264 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2265 Amplifier detector data. 

2266 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2267 Image of overscan, required only for empirical read noise. 

2268 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2269 PTC dataset containing the gains and read noise. 

2270 

2271 

2272 Raises 

2273 ------ 

2274 RuntimeError 

2275 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2276 are ``True``, but ptcDataset is not provided. 

2277 

2278 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2279 ``overscanImage`` is ``None``. 

2280 

2281 See also 

2282 -------- 

2283 lsst.ip.isr.isrFunctions.updateVariance 

2284 """ 

2285 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2286 if self.config.usePtcGains: 

2287 if ptcDataset is None: 

2288 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2289 else: 

2290 gain = ptcDataset.gain[amp.getName()] 

2291 self.log.info("Using gain from Photon Transfer Curve.") 

2292 else: 

2293 gain = amp.getGain() 

2294 

2295 if math.isnan(gain): 

2296 gain = 1.0 

2297 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2298 elif gain <= 0: 

2299 patchedGain = 1.0 

2300 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2301 amp.getName(), gain, patchedGain) 

2302 gain = patchedGain 

2303 

2304 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2305 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2306 

2307 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2308 stats = afwMath.StatisticsControl() 

2309 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2310 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2311 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2312 amp.getName(), readNoise) 

2313 elif self.config.usePtcReadNoise: 

2314 if ptcDataset is None: 

2315 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2316 else: 

2317 readNoise = ptcDataset.noise[amp.getName()] 

2318 self.log.info("Using read noise from Photon Transfer Curve.") 

2319 else: 

2320 readNoise = amp.getReadNoise() 

2321 

2322 isrFunctions.updateVariance( 

2323 maskedImage=ampExposure.getMaskedImage(), 

2324 gain=gain, 

2325 readNoise=readNoise, 

2326 ) 

2327 

2328 def maskNegativeVariance(self, exposure): 

2329 """Identify and mask pixels with negative variance values. 

2330 

2331 Parameters 

2332 ---------- 

2333 exposure : `lsst.afw.image.Exposure` 

2334 Exposure to process. 

2335 

2336 See Also 

2337 -------- 

2338 lsst.ip.isr.isrFunctions.updateVariance 

2339 """ 

2340 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2341 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2342 exposure.mask.array[bad] |= maskPlane 

2343 

2344 def darkCorrection(self, exposure, darkExposure, invert=False): 

2345 """Apply dark correction in place. 

2346 

2347 Parameters 

2348 ---------- 

2349 exposure : `lsst.afw.image.Exposure` 

2350 Exposure to process. 

2351 darkExposure : `lsst.afw.image.Exposure` 

2352 Dark exposure of the same size as ``exposure``. 

2353 invert : `Bool`, optional 

2354 If True, re-add the dark to an already corrected image. 

2355 

2356 Raises 

2357 ------ 

2358 RuntimeError 

2359 Raised if either ``exposure`` or ``darkExposure`` do not 

2360 have their dark time defined. 

2361 

2362 See Also 

2363 -------- 

2364 lsst.ip.isr.isrFunctions.darkCorrection 

2365 """ 

2366 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2367 if math.isnan(expScale): 

2368 raise RuntimeError("Exposure darktime is NAN.") 

2369 if darkExposure.getInfo().getVisitInfo() is not None \ 

2370 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2371 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2372 else: 

2373 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2374 # so getDarkTime() does not exist. 

2375 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2376 darkScale = 1.0 

2377 

2378 isrFunctions.darkCorrection( 

2379 maskedImage=exposure.getMaskedImage(), 

2380 darkMaskedImage=darkExposure.getMaskedImage(), 

2381 expScale=expScale, 

2382 darkScale=darkScale, 

2383 invert=invert, 

2384 trimToFit=self.config.doTrimToMatchCalib 

2385 ) 

2386 

2387 def doLinearize(self, detector): 

2388 """Check if linearization is needed for the detector cameraGeom. 

2389 

2390 Checks config.doLinearize and the linearity type of the first 

2391 amplifier. 

2392 

2393 Parameters 

2394 ---------- 

2395 detector : `lsst.afw.cameraGeom.Detector` 

2396 Detector to get linearity type from. 

2397 

2398 Returns 

2399 ------- 

2400 doLinearize : `Bool` 

2401 If True, linearization should be performed. 

2402 """ 

2403 return self.config.doLinearize and \ 

2404 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2405 

2406 def flatCorrection(self, exposure, flatExposure, invert=False): 

2407 """Apply flat correction in place. 

2408 

2409 Parameters 

2410 ---------- 

2411 exposure : `lsst.afw.image.Exposure` 

2412 Exposure to process. 

2413 flatExposure : `lsst.afw.image.Exposure` 

2414 Flat exposure of the same size as ``exposure``. 

2415 invert : `Bool`, optional 

2416 If True, unflatten an already flattened image. 

2417 

2418 See Also 

2419 -------- 

2420 lsst.ip.isr.isrFunctions.flatCorrection 

2421 """ 

2422 isrFunctions.flatCorrection( 

2423 maskedImage=exposure.getMaskedImage(), 

2424 flatMaskedImage=flatExposure.getMaskedImage(), 

2425 scalingType=self.config.flatScalingType, 

2426 userScale=self.config.flatUserScale, 

2427 invert=invert, 

2428 trimToFit=self.config.doTrimToMatchCalib 

2429 ) 

2430 

2431 def saturationDetection(self, exposure, amp): 

2432 """Detect and mask saturated pixels in config.saturatedMaskName. 

2433 

2434 Parameters 

2435 ---------- 

2436 exposure : `lsst.afw.image.Exposure` 

2437 Exposure to process. Only the amplifier DataSec is processed. 

2438 amp : `lsst.afw.table.AmpInfoCatalog` 

2439 Amplifier detector data. 

2440 

2441 See Also 

2442 -------- 

2443 lsst.ip.isr.isrFunctions.makeThresholdMask 

2444 """ 

2445 if not math.isnan(amp.getSaturation()): 

2446 maskedImage = exposure.getMaskedImage() 

2447 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2448 isrFunctions.makeThresholdMask( 

2449 maskedImage=dataView, 

2450 threshold=amp.getSaturation(), 

2451 growFootprints=0, 

2452 maskName=self.config.saturatedMaskName, 

2453 ) 

2454 

2455 def saturationInterpolation(self, exposure): 

2456 """Interpolate over saturated pixels, in place. 

2457 

2458 This method should be called after `saturationDetection`, to 

2459 ensure that the saturated pixels have been identified in the 

2460 SAT mask. It should also be called after `assembleCcd`, since 

2461 saturated regions may cross amplifier boundaries. 

2462 

2463 Parameters 

2464 ---------- 

2465 exposure : `lsst.afw.image.Exposure` 

2466 Exposure to process. 

2467 

2468 See Also 

2469 -------- 

2470 lsst.ip.isr.isrTask.saturationDetection 

2471 lsst.ip.isr.isrFunctions.interpolateFromMask 

2472 """ 

2473 isrFunctions.interpolateFromMask( 

2474 maskedImage=exposure.getMaskedImage(), 

2475 fwhm=self.config.fwhm, 

2476 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2477 maskNameList=list(self.config.saturatedMaskName), 

2478 ) 

2479 

2480 def suspectDetection(self, exposure, amp): 

2481 """Detect and mask suspect pixels in config.suspectMaskName. 

2482 

2483 Parameters 

2484 ---------- 

2485 exposure : `lsst.afw.image.Exposure` 

2486 Exposure to process. Only the amplifier DataSec is processed. 

2487 amp : `lsst.afw.table.AmpInfoCatalog` 

2488 Amplifier detector data. 

2489 

2490 See Also 

2491 -------- 

2492 lsst.ip.isr.isrFunctions.makeThresholdMask 

2493 

2494 Notes 

2495 ----- 

2496 Suspect pixels are pixels whose value is greater than 

2497 amp.getSuspectLevel(). This is intended to indicate pixels that may be 

2498 affected by unknown systematics; for example if non-linearity 

2499 corrections above a certain level are unstable then that would be a 

2500 useful value for suspectLevel. A value of `nan` indicates that no such 

2501 level exists and no pixels are to be masked as suspicious. 

2502 """ 

2503 suspectLevel = amp.getSuspectLevel() 

2504 if math.isnan(suspectLevel): 

2505 return 

2506 

2507 maskedImage = exposure.getMaskedImage() 

2508 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2509 isrFunctions.makeThresholdMask( 

2510 maskedImage=dataView, 

2511 threshold=suspectLevel, 

2512 growFootprints=0, 

2513 maskName=self.config.suspectMaskName, 

2514 ) 

2515 

2516 def maskDefect(self, exposure, defectBaseList): 

2517 """Mask defects using mask plane "BAD", in place. 

2518 

2519 Parameters 

2520 ---------- 

2521 exposure : `lsst.afw.image.Exposure` 

2522 Exposure to process. 

2523 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2524 `lsst.afw.image.DefectBase`. 

2525 List of defects to mask. 

2526 

2527 Notes 

2528 ----- 

2529 Call this after CCD assembly, since defects may cross amplifier 

2530 boundaries. 

2531 """ 

2532 maskedImage = exposure.getMaskedImage() 

2533 if not isinstance(defectBaseList, Defects): 

2534 # Promotes DefectBase to Defect 

2535 defectList = Defects(defectBaseList) 

2536 else: 

2537 defectList = defectBaseList 

2538 defectList.maskPixels(maskedImage, maskName="BAD") 

2539 

2540 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2541 """Mask edge pixels with applicable mask plane. 

2542 

2543 Parameters 

2544 ---------- 

2545 exposure : `lsst.afw.image.Exposure` 

2546 Exposure to process. 

2547 numEdgePixels : `int`, optional 

2548 Number of edge pixels to mask. 

2549 maskPlane : `str`, optional 

2550 Mask plane name to use. 

2551 level : `str`, optional 

2552 Level at which to mask edges. 

2553 """ 

2554 maskedImage = exposure.getMaskedImage() 

2555 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2556 

2557 if numEdgePixels > 0: 

2558 if level == 'DETECTOR': 

2559 boxes = [maskedImage.getBBox()] 

2560 elif level == 'AMP': 

2561 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2562 

2563 for box in boxes: 

2564 # This makes a bbox numEdgeSuspect pixels smaller than the 

2565 # image on each side 

2566 subImage = maskedImage[box] 

2567 box.grow(-numEdgePixels) 

2568 # Mask pixels outside box 

2569 SourceDetectionTask.setEdgeBits( 

2570 subImage, 

2571 box, 

2572 maskBitMask) 

2573 

2574 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2575 """Mask and interpolate defects using mask plane "BAD", in place. 

2576 

2577 Parameters 

2578 ---------- 

2579 exposure : `lsst.afw.image.Exposure` 

2580 Exposure to process. 

2581 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2582 `lsst.afw.image.DefectBase`. 

2583 List of defects to mask and interpolate. 

2584 

2585 See Also 

2586 -------- 

2587 lsst.ip.isr.isrTask.maskDefect 

2588 """ 

2589 self.maskDefect(exposure, defectBaseList) 

2590 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2591 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2592 isrFunctions.interpolateFromMask( 

2593 maskedImage=exposure.getMaskedImage(), 

2594 fwhm=self.config.fwhm, 

2595 growSaturatedFootprints=0, 

2596 maskNameList=["BAD"], 

2597 ) 

2598 

2599 def maskNan(self, exposure): 

2600 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2601 

2602 Parameters 

2603 ---------- 

2604 exposure : `lsst.afw.image.Exposure` 

2605 Exposure to process. 

2606 

2607 Notes 

2608 ----- 

2609 We mask over all non-finite values (NaN, inf), including those 

2610 that are masked with other bits (because those may or may not be 

2611 interpolated over later, and we want to remove all NaN/infs). 

2612 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2613 preserve the historical name. 

2614 """ 

2615 maskedImage = exposure.getMaskedImage() 

2616 

2617 # Find and mask NaNs 

2618 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2619 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2620 numNans = maskNans(maskedImage, maskVal) 

2621 self.metadata["NUMNANS"] = numNans 

2622 if numNans > 0: 

2623 self.log.warning("There were %d unmasked NaNs.", numNans) 

2624 

2625 def maskAndInterpolateNan(self, exposure): 

2626 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2627 in place. 

2628 

2629 Parameters 

2630 ---------- 

2631 exposure : `lsst.afw.image.Exposure` 

2632 Exposure to process. 

2633 

2634 See Also 

2635 -------- 

2636 lsst.ip.isr.isrTask.maskNan 

2637 """ 

2638 self.maskNan(exposure) 

2639 isrFunctions.interpolateFromMask( 

2640 maskedImage=exposure.getMaskedImage(), 

2641 fwhm=self.config.fwhm, 

2642 growSaturatedFootprints=0, 

2643 maskNameList=["UNMASKEDNAN"], 

2644 ) 

2645 

2646 def measureBackground(self, exposure, IsrQaConfig=None): 

2647 """Measure the image background in subgrids, for quality control. 

2648 

2649 Parameters 

2650 ---------- 

2651 exposure : `lsst.afw.image.Exposure` 

2652 Exposure to process. 

2653 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2654 Configuration object containing parameters on which background 

2655 statistics and subgrids to use. 

2656 """ 

2657 if IsrQaConfig is not None: 

2658 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2659 IsrQaConfig.flatness.nIter) 

2660 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2661 statsControl.setAndMask(maskVal) 

2662 maskedImage = exposure.getMaskedImage() 

2663 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2664 skyLevel = stats.getValue(afwMath.MEDIAN) 

2665 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2666 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2667 metadata = exposure.getMetadata() 

2668 metadata["SKYLEVEL"] = skyLevel 

2669 metadata["SKYSIGMA"] = skySigma 

2670 

2671 # calcluating flatlevel over the subgrids 

2672 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2673 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2674 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2675 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2676 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2677 skyLevels = numpy.zeros((nX, nY)) 

2678 

2679 for j in range(nY): 

2680 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2681 for i in range(nX): 

2682 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2683 

2684 xLLC = xc - meshXHalf 

2685 yLLC = yc - meshYHalf 

2686 xURC = xc + meshXHalf - 1 

2687 yURC = yc + meshYHalf - 1 

2688 

2689 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2690 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2691 

2692 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2693 

2694 good = numpy.where(numpy.isfinite(skyLevels)) 

2695 skyMedian = numpy.median(skyLevels[good]) 

2696 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2697 flatness_rms = numpy.std(flatness) 

2698 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2699 

2700 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2701 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2702 nX, nY, flatness_pp, flatness_rms) 

2703 

2704 metadata["FLATNESS_PP"] = float(flatness_pp) 

2705 metadata["FLATNESS_RMS"] = float(flatness_rms) 

2706 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY) 

2707 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX 

2708 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY 

2709 

2710 def roughZeroPoint(self, exposure): 

2711 """Set an approximate magnitude zero point for the exposure. 

2712 

2713 Parameters 

2714 ---------- 

2715 exposure : `lsst.afw.image.Exposure` 

2716 Exposure to process. 

2717 """ 

2718 filterLabel = exposure.getFilter() 

2719 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2720 

2721 if physicalFilter in self.config.fluxMag0T1: 

2722 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2723 else: 

2724 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2725 fluxMag0 = self.config.defaultFluxMag0T1 

2726 

2727 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2728 if not expTime > 0: # handle NaN as well as <= 0 

2729 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2730 return 

2731 

2732 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2733 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2734 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2735 

2736 @contextmanager 

2737 def flatContext(self, exp, flat, dark=None): 

2738 """Context manager that applies and removes flats and darks, 

2739 if the task is configured to apply them. 

2740 

2741 Parameters 

2742 ---------- 

2743 exp : `lsst.afw.image.Exposure` 

2744 Exposure to process. 

2745 flat : `lsst.afw.image.Exposure` 

2746 Flat exposure the same size as ``exp``. 

2747 dark : `lsst.afw.image.Exposure`, optional 

2748 Dark exposure the same size as ``exp``. 

2749 

2750 Yields 

2751 ------ 

2752 exp : `lsst.afw.image.Exposure` 

2753 The flat and dark corrected exposure. 

2754 """ 

2755 if self.config.doDark and dark is not None: 

2756 self.darkCorrection(exp, dark) 

2757 if self.config.doFlat: 

2758 self.flatCorrection(exp, flat) 

2759 try: 

2760 yield exp 

2761 finally: 

2762 if self.config.doFlat: 

2763 self.flatCorrection(exp, flat, invert=True) 

2764 if self.config.doDark and dark is not None: 

2765 self.darkCorrection(exp, dark, invert=True) 

2766 

2767 def debugView(self, exposure, stepname): 

2768 """Utility function to examine ISR exposure at different stages. 

2769 

2770 Parameters 

2771 ---------- 

2772 exposure : `lsst.afw.image.Exposure` 

2773 Exposure to view. 

2774 stepname : `str` 

2775 State of processing to view. 

2776 """ 

2777 frame = getDebugFrame(self._display, stepname) 

2778 if frame: 

2779 display = getDisplay(frame) 

2780 display.scale('asinh', 'zscale') 

2781 display.mtv(exposure) 

2782 prompt = "Press Enter to continue [c]... " 

2783 while True: 

2784 ans = input(prompt).lower() 

2785 if ans in ("", "c",): 

2786 break 

2787 

2788 

2789class FakeAmp(object): 

2790 """A Detector-like object that supports returning gain and saturation level 

2791 

2792 This is used when the input exposure does not have a detector. 

2793 

2794 Parameters 

2795 ---------- 

2796 exposure : `lsst.afw.image.Exposure` 

2797 Exposure to generate a fake amplifier for. 

2798 config : `lsst.ip.isr.isrTaskConfig` 

2799 Configuration to apply to the fake amplifier. 

2800 """ 

2801 

2802 def __init__(self, exposure, config): 

2803 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2804 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2805 self._gain = config.gain 

2806 self._readNoise = config.readNoise 

2807 self._saturation = config.saturation 

2808 

2809 def getBBox(self): 

2810 return self._bbox 

2811 

2812 def getRawBBox(self): 

2813 return self._bbox 

2814 

2815 def getRawHorizontalOverscanBBox(self): 

2816 return self._RawHorizontalOverscanBBox 

2817 

2818 def getGain(self): 

2819 return self._gain 

2820 

2821 def getReadNoise(self): 

2822 return self._readNoise 

2823 

2824 def getSaturation(self): 

2825 return self._saturation 

2826 

2827 def getSuspectLevel(self): 

2828 return float("NaN") 

2829 

2830 

2831class RunIsrConfig(pexConfig.Config): 

2832 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2833 

2834 

2835class RunIsrTask(pipeBase.CmdLineTask): 

2836 """Task to wrap the default IsrTask to allow it to be retargeted. 

2837 

2838 The standard IsrTask can be called directly from a command line 

2839 program, but doing so removes the ability of the task to be 

2840 retargeted. As most cameras override some set of the IsrTask 

2841 methods, this would remove those data-specific methods in the 

2842 output post-ISR images. This wrapping class fixes the issue, 

2843 allowing identical post-ISR images to be generated by both the 

2844 processCcd and isrTask code. 

2845 """ 

2846 ConfigClass = RunIsrConfig 

2847 _DefaultName = "runIsr" 

2848 

2849 def __init__(self, *args, **kwargs): 

2850 super().__init__(*args, **kwargs) 

2851 self.makeSubtask("isr") 

2852 

2853 def runDataRef(self, dataRef): 

2854 """ 

2855 Parameters 

2856 ---------- 

2857 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2858 data reference of the detector data to be processed 

2859 

2860 Returns 

2861 ------- 

2862 result : `pipeBase.Struct` 

2863 Result struct with component: 

2864 

2865 - exposure : `lsst.afw.image.Exposure` 

2866 Post-ISR processed exposure. 

2867 """ 

2868 return self.isr.runDataRef(dataRef)