Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42 

43from . import isrFunctions 

44from . import isrQa 

45from . import linearize 

46from .defects import Defects 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 minimum=0, # can fall back to cameraGeom 

123 ) 

124 # TODO: DM-25348. This does not work yet to correctly load 

125 # possible crosstalk sources. 

126 crosstalkSources = cT.PrerequisiteInput( 

127 name="isrOverscanCorrected", 

128 doc="Overscan corrected input images.", 

129 storageClass="Exposure", 

130 dimensions=["instrument", "exposure", "detector"], 

131 deferLoad=True, 

132 multiple=True, 

133 lookupFunction=crosstalkSourceLookup, 

134 minimum=0, # not needed for all instruments, no config to control this 

135 ) 

136 bias = cT.PrerequisiteInput( 

137 name="bias", 

138 doc="Input bias calibration.", 

139 storageClass="ExposureF", 

140 dimensions=["instrument", "detector"], 

141 isCalibration=True, 

142 ) 

143 dark = cT.PrerequisiteInput( 

144 name='dark', 

145 doc="Input dark calibration.", 

146 storageClass="ExposureF", 

147 dimensions=["instrument", "detector"], 

148 isCalibration=True, 

149 ) 

150 flat = cT.PrerequisiteInput( 

151 name="flat", 

152 doc="Input flat calibration.", 

153 storageClass="ExposureF", 

154 dimensions=["instrument", "physical_filter", "detector"], 

155 isCalibration=True, 

156 ) 

157 ptc = cT.PrerequisiteInput( 

158 name="ptc", 

159 doc="Input Photon Transfer Curve dataset", 

160 storageClass="PhotonTransferCurveDataset", 

161 dimensions=["instrument", "detector"], 

162 isCalibration=True, 

163 ) 

164 fringes = cT.PrerequisiteInput( 

165 name="fringe", 

166 doc="Input fringe calibration.", 

167 storageClass="ExposureF", 

168 dimensions=["instrument", "physical_filter", "detector"], 

169 isCalibration=True, 

170 minimum=0, # only needed for some bands, even when enabled 

171 ) 

172 strayLightData = cT.PrerequisiteInput( 

173 name='yBackground', 

174 doc="Input stray light calibration.", 

175 storageClass="StrayLightData", 

176 dimensions=["instrument", "physical_filter", "detector"], 

177 isCalibration=True, 

178 minimum=0, # only needed for some bands, even when enabled 

179 ) 

180 bfKernel = cT.PrerequisiteInput( 

181 name='bfKernel', 

182 doc="Input brighter-fatter kernel.", 

183 storageClass="NumpyArray", 

184 dimensions=["instrument"], 

185 isCalibration=True, 

186 minimum=0, # can use either bfKernel or newBFKernel 

187 ) 

188 newBFKernel = cT.PrerequisiteInput( 

189 name='brighterFatterKernel', 

190 doc="Newer complete kernel + gain solutions.", 

191 storageClass="BrighterFatterKernel", 

192 dimensions=["instrument", "detector"], 

193 isCalibration=True, 

194 minimum=0, # can use either bfKernel or newBFKernel 

195 ) 

196 defects = cT.PrerequisiteInput( 

197 name='defects', 

198 doc="Input defect tables.", 

199 storageClass="Defects", 

200 dimensions=["instrument", "detector"], 

201 isCalibration=True, 

202 ) 

203 linearizer = cT.PrerequisiteInput( 

204 name='linearizer', 

205 storageClass="Linearizer", 

206 doc="Linearity correction calibration.", 

207 dimensions=["instrument", "detector"], 

208 isCalibration=True, 

209 minimum=0, # can fall back to cameraGeom 

210 ) 

211 opticsTransmission = cT.PrerequisiteInput( 

212 name="transmission_optics", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the optics.", 

215 dimensions=["instrument"], 

216 isCalibration=True, 

217 ) 

218 filterTransmission = cT.PrerequisiteInput( 

219 name="transmission_filter", 

220 storageClass="TransmissionCurve", 

221 doc="Transmission curve due to the filter.", 

222 dimensions=["instrument", "physical_filter"], 

223 isCalibration=True, 

224 ) 

225 sensorTransmission = cT.PrerequisiteInput( 

226 name="transmission_sensor", 

227 storageClass="TransmissionCurve", 

228 doc="Transmission curve due to the sensor.", 

229 dimensions=["instrument", "detector"], 

230 isCalibration=True, 

231 ) 

232 atmosphereTransmission = cT.PrerequisiteInput( 

233 name="transmission_atmosphere", 

234 storageClass="TransmissionCurve", 

235 doc="Transmission curve due to the atmosphere.", 

236 dimensions=["instrument"], 

237 isCalibration=True, 

238 ) 

239 illumMaskedImage = cT.PrerequisiteInput( 

240 name="illum", 

241 doc="Input illumination correction.", 

242 storageClass="MaskedImageF", 

243 dimensions=["instrument", "physical_filter", "detector"], 

244 isCalibration=True, 

245 ) 

246 

247 outputExposure = cT.Output( 

248 name='postISRCCD', 

249 doc="Output ISR processed exposure.", 

250 storageClass="Exposure", 

251 dimensions=["instrument", "exposure", "detector"], 

252 ) 

253 preInterpExposure = cT.Output( 

254 name='preInterpISRCCD', 

255 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

256 storageClass="ExposureF", 

257 dimensions=["instrument", "exposure", "detector"], 

258 ) 

259 outputOssThumbnail = cT.Output( 

260 name="OssThumb", 

261 doc="Output Overscan-subtracted thumbnail image.", 

262 storageClass="Thumbnail", 

263 dimensions=["instrument", "exposure", "detector"], 

264 ) 

265 outputFlattenedThumbnail = cT.Output( 

266 name="FlattenedThumb", 

267 doc="Output flat-corrected thumbnail image.", 

268 storageClass="Thumbnail", 

269 dimensions=["instrument", "exposure", "detector"], 

270 ) 

271 

272 def __init__(self, *, config=None): 

273 super().__init__(config=config) 

274 

275 if config.doBias is not True: 

276 self.prerequisiteInputs.discard("bias") 

277 if config.doLinearize is not True: 

278 self.prerequisiteInputs.discard("linearizer") 

279 if config.doCrosstalk is not True: 

280 self.inputs.discard("crosstalkSources") 

281 self.prerequisiteInputs.discard("crosstalk") 

282 if config.doBrighterFatter is not True: 

283 self.prerequisiteInputs.discard("bfKernel") 

284 self.prerequisiteInputs.discard("newBFKernel") 

285 if config.doDefect is not True: 

286 self.prerequisiteInputs.discard("defects") 

287 if config.doDark is not True: 

288 self.prerequisiteInputs.discard("dark") 

289 if config.doFlat is not True: 

290 self.prerequisiteInputs.discard("flat") 

291 if config.doFringe is not True: 

292 self.prerequisiteInputs.discard("fringe") 

293 if config.doStrayLight is not True: 

294 self.prerequisiteInputs.discard("strayLightData") 

295 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

296 self.prerequisiteInputs.discard("ptc") 

297 if config.doAttachTransmissionCurve is not True: 

298 self.prerequisiteInputs.discard("opticsTransmission") 

299 self.prerequisiteInputs.discard("filterTransmission") 

300 self.prerequisiteInputs.discard("sensorTransmission") 

301 self.prerequisiteInputs.discard("atmosphereTransmission") 

302 if config.doUseOpticsTransmission is not True: 

303 self.prerequisiteInputs.discard("opticsTransmission") 

304 if config.doUseFilterTransmission is not True: 

305 self.prerequisiteInputs.discard("filterTransmission") 

306 if config.doUseSensorTransmission is not True: 

307 self.prerequisiteInputs.discard("sensorTransmission") 

308 if config.doUseAtmosphereTransmission is not True: 

309 self.prerequisiteInputs.discard("atmosphereTransmission") 

310 if config.doIlluminationCorrection is not True: 

311 self.prerequisiteInputs.discard("illumMaskedImage") 

312 

313 if config.doWrite is not True: 

314 self.outputs.discard("outputExposure") 

315 self.outputs.discard("preInterpExposure") 

316 self.outputs.discard("outputFlattenedThumbnail") 

317 self.outputs.discard("outputOssThumbnail") 

318 if config.doSaveInterpPixels is not True: 

319 self.outputs.discard("preInterpExposure") 

320 if config.qa.doThumbnailOss is not True: 

321 self.outputs.discard("outputOssThumbnail") 

322 if config.qa.doThumbnailFlattened is not True: 

323 self.outputs.discard("outputFlattenedThumbnail") 

324 

325 

326class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

327 pipelineConnections=IsrTaskConnections): 

328 """Configuration parameters for IsrTask. 

329 

330 Items are grouped in the order in which they are executed by the task. 

331 """ 

332 datasetType = pexConfig.Field( 

333 dtype=str, 

334 doc="Dataset type for input data; users will typically leave this alone, " 

335 "but camera-specific ISR tasks will override it", 

336 default="raw", 

337 ) 

338 

339 fallbackFilterName = pexConfig.Field( 

340 dtype=str, 

341 doc="Fallback default filter name for calibrations.", 

342 optional=True 

343 ) 

344 useFallbackDate = pexConfig.Field( 

345 dtype=bool, 

346 doc="Pass observation date when using fallback filter.", 

347 default=False, 

348 ) 

349 expectWcs = pexConfig.Field( 

350 dtype=bool, 

351 default=True, 

352 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

353 ) 

354 fwhm = pexConfig.Field( 

355 dtype=float, 

356 doc="FWHM of PSF in arcseconds.", 

357 default=1.0, 

358 ) 

359 qa = pexConfig.ConfigField( 

360 dtype=isrQa.IsrQaConfig, 

361 doc="QA related configuration options.", 

362 ) 

363 

364 # Image conversion configuration 

365 doConvertIntToFloat = pexConfig.Field( 

366 dtype=bool, 

367 doc="Convert integer raw images to floating point values?", 

368 default=True, 

369 ) 

370 

371 # Saturated pixel handling. 

372 doSaturation = pexConfig.Field( 

373 dtype=bool, 

374 doc="Mask saturated pixels? NB: this is totally independent of the" 

375 " interpolation option - this is ONLY setting the bits in the mask." 

376 " To have them interpolated make sure doSaturationInterpolation=True", 

377 default=True, 

378 ) 

379 saturatedMaskName = pexConfig.Field( 

380 dtype=str, 

381 doc="Name of mask plane to use in saturation detection and interpolation", 

382 default="SAT", 

383 ) 

384 saturation = pexConfig.Field( 

385 dtype=float, 

386 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

387 default=float("NaN"), 

388 ) 

389 growSaturationFootprintSize = pexConfig.Field( 

390 dtype=int, 

391 doc="Number of pixels by which to grow the saturation footprints", 

392 default=1, 

393 ) 

394 

395 # Suspect pixel handling. 

396 doSuspect = pexConfig.Field( 

397 dtype=bool, 

398 doc="Mask suspect pixels?", 

399 default=False, 

400 ) 

401 suspectMaskName = pexConfig.Field( 

402 dtype=str, 

403 doc="Name of mask plane to use for suspect pixels", 

404 default="SUSPECT", 

405 ) 

406 numEdgeSuspect = pexConfig.Field( 

407 dtype=int, 

408 doc="Number of edge pixels to be flagged as untrustworthy.", 

409 default=0, 

410 ) 

411 edgeMaskLevel = pexConfig.ChoiceField( 

412 dtype=str, 

413 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

414 default="DETECTOR", 

415 allowed={ 

416 'DETECTOR': 'Mask only the edges of the full detector.', 

417 'AMP': 'Mask edges of each amplifier.', 

418 }, 

419 ) 

420 

421 # Initial masking options. 

422 doSetBadRegions = pexConfig.Field( 

423 dtype=bool, 

424 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

425 default=True, 

426 ) 

427 badStatistic = pexConfig.ChoiceField( 

428 dtype=str, 

429 doc="How to estimate the average value for BAD regions.", 

430 default='MEANCLIP', 

431 allowed={ 

432 "MEANCLIP": "Correct using the (clipped) mean of good data", 

433 "MEDIAN": "Correct using the median of the good data", 

434 }, 

435 ) 

436 

437 # Overscan subtraction configuration. 

438 doOverscan = pexConfig.Field( 

439 dtype=bool, 

440 doc="Do overscan subtraction?", 

441 default=True, 

442 ) 

443 overscan = pexConfig.ConfigurableField( 

444 target=OverscanCorrectionTask, 

445 doc="Overscan subtraction task for image segments.", 

446 ) 

447 

448 overscanFitType = pexConfig.ChoiceField( 

449 dtype=str, 

450 doc="The method for fitting the overscan bias level.", 

451 default='MEDIAN', 

452 allowed={ 

453 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

454 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

455 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

456 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

457 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

458 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

459 "MEAN": "Correct using the mean of the overscan region", 

460 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

461 "MEDIAN": "Correct using the median of the overscan region", 

462 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

463 }, 

464 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

465 " This option will no longer be used, and will be removed after v20.") 

466 ) 

467 overscanOrder = pexConfig.Field( 

468 dtype=int, 

469 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

470 "or number of spline knots if overscan fit type is a spline."), 

471 default=1, 

472 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

473 " This option will no longer be used, and will be removed after v20.") 

474 ) 

475 overscanNumSigmaClip = pexConfig.Field( 

476 dtype=float, 

477 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

478 default=3.0, 

479 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

480 " This option will no longer be used, and will be removed after v20.") 

481 ) 

482 overscanIsInt = pexConfig.Field( 

483 dtype=bool, 

484 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

485 " and overscan.FitType=MEDIAN_PER_ROW.", 

486 default=True, 

487 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

488 " This option will no longer be used, and will be removed after v20.") 

489 ) 

490 # These options do not get deprecated, as they define how we slice up the image data. 

491 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

492 dtype=int, 

493 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

494 default=0, 

495 ) 

496 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

497 dtype=int, 

498 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

499 default=0, 

500 ) 

501 overscanMaxDev = pexConfig.Field( 501 ↛ exitline 501 didn't jump to the function exit

502 dtype=float, 

503 doc="Maximum deviation from the median for overscan", 

504 default=1000.0, check=lambda x: x > 0 

505 ) 

506 overscanBiasJump = pexConfig.Field( 

507 dtype=bool, 

508 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

509 default=False, 

510 ) 

511 overscanBiasJumpKeyword = pexConfig.Field( 

512 dtype=str, 

513 doc="Header keyword containing information about devices.", 

514 default="NO_SUCH_KEY", 

515 ) 

516 overscanBiasJumpDevices = pexConfig.ListField( 

517 dtype=str, 

518 doc="List of devices that need piecewise overscan correction.", 

519 default=(), 

520 ) 

521 overscanBiasJumpLocation = pexConfig.Field( 

522 dtype=int, 

523 doc="Location of bias jump along y-axis.", 

524 default=0, 

525 ) 

526 

527 # Amplifier to CCD assembly configuration 

528 doAssembleCcd = pexConfig.Field( 

529 dtype=bool, 

530 default=True, 

531 doc="Assemble amp-level exposures into a ccd-level exposure?" 

532 ) 

533 assembleCcd = pexConfig.ConfigurableField( 

534 target=AssembleCcdTask, 

535 doc="CCD assembly task", 

536 ) 

537 

538 # General calibration configuration. 

539 doAssembleIsrExposures = pexConfig.Field( 

540 dtype=bool, 

541 default=False, 

542 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

543 ) 

544 doTrimToMatchCalib = pexConfig.Field( 

545 dtype=bool, 

546 default=False, 

547 doc="Trim raw data to match calibration bounding boxes?" 

548 ) 

549 

550 # Bias subtraction. 

551 doBias = pexConfig.Field( 

552 dtype=bool, 

553 doc="Apply bias frame correction?", 

554 default=True, 

555 ) 

556 biasDataProductName = pexConfig.Field( 

557 dtype=str, 

558 doc="Name of the bias data product", 

559 default="bias", 

560 ) 

561 doBiasBeforeOverscan = pexConfig.Field( 

562 dtype=bool, 

563 doc="Reverse order of overscan and bias correction.", 

564 default=False 

565 ) 

566 

567 # Variance construction 

568 doVariance = pexConfig.Field( 

569 dtype=bool, 

570 doc="Calculate variance?", 

571 default=True 

572 ) 

573 gain = pexConfig.Field( 

574 dtype=float, 

575 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

576 default=float("NaN"), 

577 ) 

578 readNoise = pexConfig.Field( 

579 dtype=float, 

580 doc="The read noise to use if no Detector is present in the Exposure", 

581 default=0.0, 

582 ) 

583 doEmpiricalReadNoise = pexConfig.Field( 

584 dtype=bool, 

585 default=False, 

586 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

587 ) 

588 usePtcReadNoise = pexConfig.Field( 

589 dtype=bool, 

590 default=False, 

591 doc="Use readnoise values from the Photon Transfer Curve?" 

592 ) 

593 # Linearization. 

594 doLinearize = pexConfig.Field( 

595 dtype=bool, 

596 doc="Correct for nonlinearity of the detector's response?", 

597 default=True, 

598 ) 

599 

600 # Crosstalk. 

601 doCrosstalk = pexConfig.Field( 

602 dtype=bool, 

603 doc="Apply intra-CCD crosstalk correction?", 

604 default=False, 

605 ) 

606 doCrosstalkBeforeAssemble = pexConfig.Field( 

607 dtype=bool, 

608 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

609 default=False, 

610 ) 

611 crosstalk = pexConfig.ConfigurableField( 

612 target=CrosstalkTask, 

613 doc="Intra-CCD crosstalk correction", 

614 ) 

615 

616 # Masking options. 

617 doDefect = pexConfig.Field( 

618 dtype=bool, 

619 doc="Apply correction for CCD defects, e.g. hot pixels?", 

620 default=True, 

621 ) 

622 doNanMasking = pexConfig.Field( 

623 dtype=bool, 

624 doc="Mask non-finite (NAN, inf) pixels?", 

625 default=True, 

626 ) 

627 doWidenSaturationTrails = pexConfig.Field( 

628 dtype=bool, 

629 doc="Widen bleed trails based on their width?", 

630 default=True 

631 ) 

632 

633 # Brighter-Fatter correction. 

634 doBrighterFatter = pexConfig.Field( 

635 dtype=bool, 

636 default=False, 

637 doc="Apply the brighter-fatter correction?" 

638 ) 

639 brighterFatterLevel = pexConfig.ChoiceField( 

640 dtype=str, 

641 default="DETECTOR", 

642 doc="The level at which to correct for brighter-fatter.", 

643 allowed={ 

644 "AMP": "Every amplifier treated separately.", 

645 "DETECTOR": "One kernel per detector", 

646 } 

647 ) 

648 brighterFatterMaxIter = pexConfig.Field( 

649 dtype=int, 

650 default=10, 

651 doc="Maximum number of iterations for the brighter-fatter correction" 

652 ) 

653 brighterFatterThreshold = pexConfig.Field( 

654 dtype=float, 

655 default=1000, 

656 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

657 "absolute value of the difference between the current corrected image and the one " 

658 "from the previous iteration summed over all the pixels." 

659 ) 

660 brighterFatterApplyGain = pexConfig.Field( 

661 dtype=bool, 

662 default=True, 

663 doc="Should the gain be applied when applying the brighter-fatter correction?" 

664 ) 

665 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

666 dtype=str, 

667 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

668 "correction.", 

669 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

670 ) 

671 brighterFatterMaskGrowSize = pexConfig.Field( 

672 dtype=int, 

673 default=0, 

674 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

675 "when brighter-fatter correction is applied." 

676 ) 

677 

678 # Dark subtraction. 

679 doDark = pexConfig.Field( 

680 dtype=bool, 

681 doc="Apply dark frame correction?", 

682 default=True, 

683 ) 

684 darkDataProductName = pexConfig.Field( 

685 dtype=str, 

686 doc="Name of the dark data product", 

687 default="dark", 

688 ) 

689 

690 # Camera-specific stray light removal. 

691 doStrayLight = pexConfig.Field( 

692 dtype=bool, 

693 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

694 default=False, 

695 ) 

696 strayLight = pexConfig.ConfigurableField( 

697 target=StrayLightTask, 

698 doc="y-band stray light correction" 

699 ) 

700 

701 # Flat correction. 

702 doFlat = pexConfig.Field( 

703 dtype=bool, 

704 doc="Apply flat field correction?", 

705 default=True, 

706 ) 

707 flatDataProductName = pexConfig.Field( 

708 dtype=str, 

709 doc="Name of the flat data product", 

710 default="flat", 

711 ) 

712 flatScalingType = pexConfig.ChoiceField( 

713 dtype=str, 

714 doc="The method for scaling the flat on the fly.", 

715 default='USER', 

716 allowed={ 

717 "USER": "Scale by flatUserScale", 

718 "MEAN": "Scale by the inverse of the mean", 

719 "MEDIAN": "Scale by the inverse of the median", 

720 }, 

721 ) 

722 flatUserScale = pexConfig.Field( 

723 dtype=float, 

724 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

725 default=1.0, 

726 ) 

727 doTweakFlat = pexConfig.Field( 

728 dtype=bool, 

729 doc="Tweak flats to match observed amplifier ratios?", 

730 default=False 

731 ) 

732 

733 # Amplifier normalization based on gains instead of using flats configuration. 

734 doApplyGains = pexConfig.Field( 

735 dtype=bool, 

736 doc="Correct the amplifiers for their gains instead of applying flat correction", 

737 default=False, 

738 ) 

739 usePtcGains = pexConfig.Field( 

740 dtype=bool, 

741 doc="Use the gain values from the Photon Transfer Curve?", 

742 default=False, 

743 ) 

744 normalizeGains = pexConfig.Field( 

745 dtype=bool, 

746 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

747 default=False, 

748 ) 

749 

750 # Fringe correction. 

751 doFringe = pexConfig.Field( 

752 dtype=bool, 

753 doc="Apply fringe correction?", 

754 default=True, 

755 ) 

756 fringe = pexConfig.ConfigurableField( 

757 target=FringeTask, 

758 doc="Fringe subtraction task", 

759 ) 

760 fringeAfterFlat = pexConfig.Field( 

761 dtype=bool, 

762 doc="Do fringe subtraction after flat-fielding?", 

763 default=True, 

764 ) 

765 

766 # Initial CCD-level background statistics options. 

767 doMeasureBackground = pexConfig.Field( 

768 dtype=bool, 

769 doc="Measure the background level on the reduced image?", 

770 default=False, 

771 ) 

772 

773 # Camera-specific masking configuration. 

774 doCameraSpecificMasking = pexConfig.Field( 

775 dtype=bool, 

776 doc="Mask camera-specific bad regions?", 

777 default=False, 

778 ) 

779 masking = pexConfig.ConfigurableField( 

780 target=MaskingTask, 

781 doc="Masking task." 

782 ) 

783 

784 # Interpolation options. 

785 

786 doInterpolate = pexConfig.Field( 

787 dtype=bool, 

788 doc="Interpolate masked pixels?", 

789 default=True, 

790 ) 

791 doSaturationInterpolation = pexConfig.Field( 

792 dtype=bool, 

793 doc="Perform interpolation over pixels masked as saturated?" 

794 " NB: This is independent of doSaturation; if that is False this plane" 

795 " will likely be blank, resulting in a no-op here.", 

796 default=True, 

797 ) 

798 doNanInterpolation = pexConfig.Field( 

799 dtype=bool, 

800 doc="Perform interpolation over pixels masked as NaN?" 

801 " NB: This is independent of doNanMasking; if that is False this plane" 

802 " will likely be blank, resulting in a no-op here.", 

803 default=True, 

804 ) 

805 doNanInterpAfterFlat = pexConfig.Field( 

806 dtype=bool, 

807 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

808 "also have to interpolate them before flat-fielding."), 

809 default=False, 

810 ) 

811 maskListToInterpolate = pexConfig.ListField( 

812 dtype=str, 

813 doc="List of mask planes that should be interpolated.", 

814 default=['SAT', 'BAD'], 

815 ) 

816 doSaveInterpPixels = pexConfig.Field( 

817 dtype=bool, 

818 doc="Save a copy of the pre-interpolated pixel values?", 

819 default=False, 

820 ) 

821 

822 # Default photometric calibration options. 

823 fluxMag0T1 = pexConfig.DictField( 

824 keytype=str, 

825 itemtype=float, 

826 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

827 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

828 )) 

829 ) 

830 defaultFluxMag0T1 = pexConfig.Field( 

831 dtype=float, 

832 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

833 default=pow(10.0, 0.4*28.0) 

834 ) 

835 

836 # Vignette correction configuration. 

837 doVignette = pexConfig.Field( 

838 dtype=bool, 

839 doc="Apply vignetting parameters?", 

840 default=False, 

841 ) 

842 vignette = pexConfig.ConfigurableField( 

843 target=VignetteTask, 

844 doc="Vignetting task.", 

845 ) 

846 

847 # Transmission curve configuration. 

848 doAttachTransmissionCurve = pexConfig.Field( 

849 dtype=bool, 

850 default=False, 

851 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

852 ) 

853 doUseOpticsTransmission = pexConfig.Field( 

854 dtype=bool, 

855 default=True, 

856 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

857 ) 

858 doUseFilterTransmission = pexConfig.Field( 

859 dtype=bool, 

860 default=True, 

861 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

862 ) 

863 doUseSensorTransmission = pexConfig.Field( 

864 dtype=bool, 

865 default=True, 

866 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

867 ) 

868 doUseAtmosphereTransmission = pexConfig.Field( 

869 dtype=bool, 

870 default=True, 

871 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

872 ) 

873 

874 # Illumination correction. 

875 doIlluminationCorrection = pexConfig.Field( 

876 dtype=bool, 

877 default=False, 

878 doc="Perform illumination correction?" 

879 ) 

880 illuminationCorrectionDataProductName = pexConfig.Field( 

881 dtype=str, 

882 doc="Name of the illumination correction data product.", 

883 default="illumcor", 

884 ) 

885 illumScale = pexConfig.Field( 

886 dtype=float, 

887 doc="Scale factor for the illumination correction.", 

888 default=1.0, 

889 ) 

890 illumFilters = pexConfig.ListField( 

891 dtype=str, 

892 default=[], 

893 doc="Only perform illumination correction for these filters." 

894 ) 

895 

896 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

897 doWrite = pexConfig.Field( 

898 dtype=bool, 

899 doc="Persist postISRCCD?", 

900 default=True, 

901 ) 

902 

903 def validate(self): 

904 super().validate() 

905 if self.doFlat and self.doApplyGains: 

906 raise ValueError("You may not specify both doFlat and doApplyGains") 

907 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

908 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

909 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

910 self.maskListToInterpolate.append(self.saturatedMaskName) 

911 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

912 self.maskListToInterpolate.remove(self.saturatedMaskName) 

913 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

914 self.maskListToInterpolate.append("UNMASKEDNAN") 

915 

916 

917class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

918 """Apply common instrument signature correction algorithms to a raw frame. 

919 

920 The process for correcting imaging data is very similar from 

921 camera to camera. This task provides a vanilla implementation of 

922 doing these corrections, including the ability to turn certain 

923 corrections off if they are not needed. The inputs to the primary 

924 method, `run()`, are a raw exposure to be corrected and the 

925 calibration data products. The raw input is a single chip sized 

926 mosaic of all amps including overscans and other non-science 

927 pixels. The method `runDataRef()` identifies and defines the 

928 calibration data products, and is intended for use by a 

929 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

930 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

931 subclassed for different camera, although the most camera specific 

932 methods have been split into subtasks that can be redirected 

933 appropriately. 

934 

935 The __init__ method sets up the subtasks for ISR processing, using 

936 the defaults from `lsst.ip.isr`. 

937 

938 Parameters 

939 ---------- 

940 args : `list` 

941 Positional arguments passed to the Task constructor. None used at this time. 

942 kwargs : `dict`, optional 

943 Keyword arguments passed on to the Task constructor. None used at this time. 

944 """ 

945 ConfigClass = IsrTaskConfig 

946 _DefaultName = "isr" 

947 

948 def __init__(self, **kwargs): 

949 super().__init__(**kwargs) 

950 self.makeSubtask("assembleCcd") 

951 self.makeSubtask("crosstalk") 

952 self.makeSubtask("strayLight") 

953 self.makeSubtask("fringe") 

954 self.makeSubtask("masking") 

955 self.makeSubtask("overscan") 

956 self.makeSubtask("vignette") 

957 

958 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

959 inputs = butlerQC.get(inputRefs) 

960 

961 try: 

962 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

963 except Exception as e: 

964 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

965 (inputRefs, e)) 

966 

967 inputs['isGen3'] = True 

968 

969 detector = inputs['ccdExposure'].getDetector() 

970 

971 if self.config.doCrosstalk is True: 

972 # Crosstalk sources need to be defined by the pipeline 

973 # yaml if they exist. 

974 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

975 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

976 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

977 else: 

978 coeffVector = (self.config.crosstalk.crosstalkValues 

979 if self.config.crosstalk.useConfigCoefficients else None) 

980 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

981 inputs['crosstalk'] = crosstalkCalib 

982 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

983 if 'crosstalkSources' not in inputs: 

984 self.log.warn("No crosstalkSources found for chip with interChip terms!") 

985 

986 if self.doLinearize(detector) is True: 

987 if 'linearizer' in inputs: 

988 if isinstance(inputs['linearizer'], dict): 

989 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

990 linearizer.fromYaml(inputs['linearizer']) 

991 self.log.warn("Dictionary linearizers will be deprecated in DM-28741.") 

992 elif isinstance(inputs['linearizer'], numpy.ndarray): 

993 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

994 detector=detector, 

995 log=self.log) 

996 self.log.warn("Bare lookup table linearizers will be deprecated in DM-28741.") 

997 else: 

998 linearizer = inputs['linearizer'] 

999 linearizer.log = self.log 

1000 inputs['linearizer'] = linearizer 

1001 else: 

1002 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1003 self.log.warn("Constructing linearizer from cameraGeom information.") 

1004 

1005 if self.config.doDefect is True: 

1006 if "defects" in inputs and inputs['defects'] is not None: 

1007 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

1008 # masking expects a list of defects defined by their bounding box 

1009 if not isinstance(inputs["defects"], Defects): 

1010 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1011 

1012 # Load the correct style of brighter-fatter kernel, and repack 

1013 # the information as a numpy array. 

1014 if self.config.doBrighterFatter: 

1015 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1016 if brighterFatterKernel is None: 

1017 brighterFatterKernel = inputs.get('bfKernel', None) 

1018 

1019 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1020 # This is a ISR calib kernel 

1021 detName = detector.getName() 

1022 level = brighterFatterKernel.level 

1023 

1024 # This is expected to be a dictionary of amp-wise gains. 

1025 inputs['bfGains'] = brighterFatterKernel.gain 

1026 if self.config.brighterFatterLevel == 'DETECTOR': 

1027 if level == 'DETECTOR': 

1028 if detName in brighterFatterKernel.detKernels: 

1029 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1030 else: 

1031 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1032 elif level == 'AMP': 

1033 self.log.warn("Making DETECTOR level kernel from AMP based brighter fatter kernels.") 

1034 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1035 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1036 elif self.config.brighterFatterLevel == 'AMP': 

1037 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1038 

1039 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1040 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

1041 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1042 expId=expId, 

1043 assembler=self.assembleCcd 

1044 if self.config.doAssembleIsrExposures else None) 

1045 else: 

1046 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1047 

1048 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1049 if 'strayLightData' not in inputs: 

1050 inputs['strayLightData'] = None 

1051 

1052 outputs = self.run(**inputs) 

1053 butlerQC.put(outputs, outputRefs) 

1054 

1055 def readIsrData(self, dataRef, rawExposure): 

1056 """Retrieve necessary frames for instrument signature removal. 

1057 

1058 Pre-fetching all required ISR data products limits the IO 

1059 required by the ISR. Any conflict between the calibration data 

1060 available and that needed for ISR is also detected prior to 

1061 doing processing, allowing it to fail quickly. 

1062 

1063 Parameters 

1064 ---------- 

1065 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1066 Butler reference of the detector data to be processed 

1067 rawExposure : `afw.image.Exposure` 

1068 The raw exposure that will later be corrected with the 

1069 retrieved calibration data; should not be modified in this 

1070 method. 

1071 

1072 Returns 

1073 ------- 

1074 result : `lsst.pipe.base.Struct` 

1075 Result struct with components (which may be `None`): 

1076 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1077 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1078 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1079 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1080 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1081 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1082 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1083 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1084 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1085 - ``seed``: random seed derived from the ccdExposureId for random 

1086 number generator (`uint32`). 

1087 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1088 A ``TransmissionCurve`` that represents the throughput of the optics, 

1089 to be evaluated in focal-plane coordinates. 

1090 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1091 A ``TransmissionCurve`` that represents the throughput of the filter 

1092 itself, to be evaluated in focal-plane coordinates. 

1093 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1094 A ``TransmissionCurve`` that represents the throughput of the sensor 

1095 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1096 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1097 A ``TransmissionCurve`` that represents the throughput of the 

1098 atmosphere, assumed to be spatially constant. 

1099 - ``strayLightData`` : `object` 

1100 An opaque object containing calibration information for 

1101 stray-light correction. If `None`, no correction will be 

1102 performed. 

1103 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1104 

1105 Raises 

1106 ------ 

1107 NotImplementedError : 

1108 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1109 """ 

1110 try: 

1111 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1112 dateObs = dateObs.toPython().isoformat() 

1113 except RuntimeError: 

1114 self.log.warn("Unable to identify dateObs for rawExposure.") 

1115 dateObs = None 

1116 

1117 ccd = rawExposure.getDetector() 

1118 filterLabel = rawExposure.getFilterLabel() 

1119 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1120 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1121 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1122 if self.config.doBias else None) 

1123 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1124 linearizer = (dataRef.get("linearizer", immediate=True) 

1125 if self.doLinearize(ccd) else None) 

1126 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1127 linearizer.log = self.log 

1128 if isinstance(linearizer, numpy.ndarray): 

1129 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1130 

1131 crosstalkCalib = None 

1132 if self.config.doCrosstalk: 

1133 try: 

1134 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1135 except NoResults: 

1136 coeffVector = (self.config.crosstalk.crosstalkValues 

1137 if self.config.crosstalk.useConfigCoefficients else None) 

1138 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1139 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1140 if self.config.doCrosstalk else None) 

1141 

1142 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1143 if self.config.doDark else None) 

1144 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1145 dateObs=dateObs) 

1146 if self.config.doFlat else None) 

1147 

1148 brighterFatterKernel = None 

1149 brighterFatterGains = None 

1150 if self.config.doBrighterFatter is True: 

1151 try: 

1152 # Use the new-style cp_pipe version of the kernel if it exists 

1153 # If using a new-style kernel, always use the self-consistent 

1154 # gains, i.e. the ones inside the kernel object itself 

1155 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1156 brighterFatterGains = brighterFatterKernel.gain 

1157 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded") 

1158 except NoResults: 

1159 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1160 brighterFatterKernel = dataRef.get("bfKernel") 

1161 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded") 

1162 except NoResults: 

1163 brighterFatterKernel = None 

1164 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1165 # If the kernel is not an ndarray, it's the cp_pipe version 

1166 # so extract the kernel for this detector, or raise an error 

1167 if self.config.brighterFatterLevel == 'DETECTOR': 

1168 if brighterFatterKernel.detKernels: 

1169 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()] 

1170 else: 

1171 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1172 else: 

1173 # TODO DM-15631 for implementing this 

1174 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1175 

1176 defectList = (dataRef.get("defects") 

1177 if self.config.doDefect else None) 

1178 expId = rawExposure.getInfo().getVisitInfo().getExposureId() 

1179 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd 

1180 if self.config.doAssembleIsrExposures else None) 

1181 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1182 else pipeBase.Struct(fringes=None)) 

1183 

1184 if self.config.doAttachTransmissionCurve: 

1185 opticsTransmission = (dataRef.get("transmission_optics") 

1186 if self.config.doUseOpticsTransmission else None) 

1187 filterTransmission = (dataRef.get("transmission_filter") 

1188 if self.config.doUseFilterTransmission else None) 

1189 sensorTransmission = (dataRef.get("transmission_sensor") 

1190 if self.config.doUseSensorTransmission else None) 

1191 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1192 if self.config.doUseAtmosphereTransmission else None) 

1193 else: 

1194 opticsTransmission = None 

1195 filterTransmission = None 

1196 sensorTransmission = None 

1197 atmosphereTransmission = None 

1198 

1199 if self.config.doStrayLight: 

1200 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1201 else: 

1202 strayLightData = None 

1203 

1204 illumMaskedImage = (self.getIsrExposure(dataRef, 

1205 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1206 if (self.config.doIlluminationCorrection 

1207 and physicalFilter in self.config.illumFilters) 

1208 else None) 

1209 

1210 # Struct should include only kwargs to run() 

1211 return pipeBase.Struct(bias=biasExposure, 

1212 linearizer=linearizer, 

1213 crosstalk=crosstalkCalib, 

1214 crosstalkSources=crosstalkSources, 

1215 dark=darkExposure, 

1216 flat=flatExposure, 

1217 bfKernel=brighterFatterKernel, 

1218 bfGains=brighterFatterGains, 

1219 defects=defectList, 

1220 fringes=fringeStruct, 

1221 opticsTransmission=opticsTransmission, 

1222 filterTransmission=filterTransmission, 

1223 sensorTransmission=sensorTransmission, 

1224 atmosphereTransmission=atmosphereTransmission, 

1225 strayLightData=strayLightData, 

1226 illumMaskedImage=illumMaskedImage 

1227 ) 

1228 

1229 @pipeBase.timeMethod 

1230 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1231 crosstalk=None, crosstalkSources=None, 

1232 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1233 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1234 sensorTransmission=None, atmosphereTransmission=None, 

1235 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1236 isGen3=False, 

1237 ): 

1238 """Perform instrument signature removal on an exposure. 

1239 

1240 Steps included in the ISR processing, in order performed, are: 

1241 - saturation and suspect pixel masking 

1242 - overscan subtraction 

1243 - CCD assembly of individual amplifiers 

1244 - bias subtraction 

1245 - variance image construction 

1246 - linearization of non-linear response 

1247 - crosstalk masking 

1248 - brighter-fatter correction 

1249 - dark subtraction 

1250 - fringe correction 

1251 - stray light subtraction 

1252 - flat correction 

1253 - masking of known defects and camera specific features 

1254 - vignette calculation 

1255 - appending transmission curve and distortion model 

1256 

1257 Parameters 

1258 ---------- 

1259 ccdExposure : `lsst.afw.image.Exposure` 

1260 The raw exposure that is to be run through ISR. The 

1261 exposure is modified by this method. 

1262 camera : `lsst.afw.cameraGeom.Camera`, optional 

1263 The camera geometry for this exposure. Required if ``isGen3`` is 

1264 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1265 ``flat`` does not have an associated detector. 

1266 bias : `lsst.afw.image.Exposure`, optional 

1267 Bias calibration frame. 

1268 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1269 Functor for linearization. 

1270 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1271 Calibration for crosstalk. 

1272 crosstalkSources : `list`, optional 

1273 List of possible crosstalk sources. 

1274 dark : `lsst.afw.image.Exposure`, optional 

1275 Dark calibration frame. 

1276 flat : `lsst.afw.image.Exposure`, optional 

1277 Flat calibration frame. 

1278 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1279 Photon transfer curve dataset, with, e.g., gains 

1280 and read noise. 

1281 bfKernel : `numpy.ndarray`, optional 

1282 Brighter-fatter kernel. 

1283 bfGains : `dict` of `float`, optional 

1284 Gains used to override the detector's nominal gains for the 

1285 brighter-fatter correction. A dict keyed by amplifier name for 

1286 the detector in question. 

1287 defects : `lsst.ip.isr.Defects`, optional 

1288 List of defects. 

1289 fringes : `lsst.pipe.base.Struct`, optional 

1290 Struct containing the fringe correction data, with 

1291 elements: 

1292 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1293 - ``seed``: random seed derived from the ccdExposureId for random 

1294 number generator (`uint32`) 

1295 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1296 A ``TransmissionCurve`` that represents the throughput of the optics, 

1297 to be evaluated in focal-plane coordinates. 

1298 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1299 A ``TransmissionCurve`` that represents the throughput of the filter 

1300 itself, to be evaluated in focal-plane coordinates. 

1301 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1302 A ``TransmissionCurve`` that represents the throughput of the sensor 

1303 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1304 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1305 A ``TransmissionCurve`` that represents the throughput of the 

1306 atmosphere, assumed to be spatially constant. 

1307 detectorNum : `int`, optional 

1308 The integer number for the detector to process. 

1309 isGen3 : bool, optional 

1310 Flag this call to run() as using the Gen3 butler environment. 

1311 strayLightData : `object`, optional 

1312 Opaque object containing calibration information for stray-light 

1313 correction. If `None`, no correction will be performed. 

1314 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1315 Illumination correction image. 

1316 

1317 Returns 

1318 ------- 

1319 result : `lsst.pipe.base.Struct` 

1320 Result struct with component: 

1321 - ``exposure`` : `afw.image.Exposure` 

1322 The fully ISR corrected exposure. 

1323 - ``outputExposure`` : `afw.image.Exposure` 

1324 An alias for `exposure` 

1325 - ``ossThumb`` : `numpy.ndarray` 

1326 Thumbnail image of the exposure after overscan subtraction. 

1327 - ``flattenedThumb`` : `numpy.ndarray` 

1328 Thumbnail image of the exposure after flat-field correction. 

1329 

1330 Raises 

1331 ------ 

1332 RuntimeError 

1333 Raised if a configuration option is set to True, but the 

1334 required calibration data has not been specified. 

1335 

1336 Notes 

1337 ----- 

1338 The current processed exposure can be viewed by setting the 

1339 appropriate lsstDebug entries in the `debug.display` 

1340 dictionary. The names of these entries correspond to some of 

1341 the IsrTaskConfig Boolean options, with the value denoting the 

1342 frame to use. The exposure is shown inside the matching 

1343 option check and after the processing of that step has 

1344 finished. The steps with debug points are: 

1345 

1346 doAssembleCcd 

1347 doBias 

1348 doCrosstalk 

1349 doBrighterFatter 

1350 doDark 

1351 doFringe 

1352 doStrayLight 

1353 doFlat 

1354 

1355 In addition, setting the "postISRCCD" entry displays the 

1356 exposure after all ISR processing has finished. 

1357 

1358 """ 

1359 

1360 if isGen3 is True: 

1361 # Gen3 currently cannot automatically do configuration overrides. 

1362 # DM-15257 looks to discuss this issue. 

1363 # Configure input exposures; 

1364 if detectorNum is None: 

1365 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1366 

1367 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1368 bias = self.ensureExposure(bias, camera, detectorNum) 

1369 dark = self.ensureExposure(dark, camera, detectorNum) 

1370 flat = self.ensureExposure(flat, camera, detectorNum) 

1371 else: 

1372 if isinstance(ccdExposure, ButlerDataRef): 

1373 return self.runDataRef(ccdExposure) 

1374 

1375 ccd = ccdExposure.getDetector() 

1376 filterLabel = ccdExposure.getFilterLabel() 

1377 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1378 

1379 if not ccd: 

1380 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1381 ccd = [FakeAmp(ccdExposure, self.config)] 

1382 

1383 # Validate Input 

1384 if self.config.doBias and bias is None: 

1385 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1386 if self.doLinearize(ccd) and linearizer is None: 

1387 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1388 if self.config.doBrighterFatter and bfKernel is None: 

1389 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1390 if self.config.doDark and dark is None: 

1391 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1392 if self.config.doFlat and flat is None: 

1393 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1394 if self.config.doDefect and defects is None: 

1395 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1396 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1397 and fringes.fringes is None): 

1398 # The `fringes` object needs to be a pipeBase.Struct, as 

1399 # we use it as a `dict` for the parameters of 

1400 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1401 # not be `None` if `doFringe=True`. Otherwise, raise. 

1402 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1403 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1404 and illumMaskedImage is None): 

1405 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1406 

1407 # Begin ISR processing. 

1408 if self.config.doConvertIntToFloat: 

1409 self.log.info("Converting exposure to floating point values.") 

1410 ccdExposure = self.convertIntToFloat(ccdExposure) 

1411 

1412 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1413 self.log.info("Applying bias correction.") 

1414 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1415 trimToFit=self.config.doTrimToMatchCalib) 

1416 self.debugView(ccdExposure, "doBias") 

1417 

1418 # Amplifier level processing. 

1419 overscans = [] 

1420 for amp in ccd: 

1421 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1422 if ccdExposure.getBBox().contains(amp.getBBox()): 

1423 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1424 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1425 

1426 if self.config.doOverscan and not badAmp: 

1427 # Overscan correction on amp-by-amp basis. 

1428 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1429 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1430 if overscanResults is not None and \ 

1431 self.config.qa is not None and self.config.qa.saveStats is True: 

1432 if isinstance(overscanResults.overscanFit, float): 

1433 qaMedian = overscanResults.overscanFit 

1434 qaStdev = float("NaN") 

1435 else: 

1436 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1437 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1438 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1439 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1440 

1441 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian) 

1442 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev) 

1443 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1444 amp.getName(), qaMedian, qaStdev) 

1445 

1446 # Residuals after overscan correction 

1447 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1448 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1449 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1450 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1451 

1452 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter) 

1453 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter) 

1454 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1455 amp.getName(), qaMedianAfter, qaStdevAfter) 

1456 

1457 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1458 else: 

1459 if badAmp: 

1460 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1461 overscanResults = None 

1462 

1463 overscans.append(overscanResults if overscanResults is not None else None) 

1464 else: 

1465 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1466 

1467 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1468 self.log.info("Applying crosstalk correction.") 

1469 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1470 crosstalkSources=crosstalkSources, camera=camera) 

1471 self.debugView(ccdExposure, "doCrosstalk") 

1472 

1473 if self.config.doAssembleCcd: 

1474 self.log.info("Assembling CCD from amplifiers.") 

1475 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1476 

1477 if self.config.expectWcs and not ccdExposure.getWcs(): 

1478 self.log.warn("No WCS found in input exposure.") 

1479 self.debugView(ccdExposure, "doAssembleCcd") 

1480 

1481 ossThumb = None 

1482 if self.config.qa.doThumbnailOss: 

1483 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1484 

1485 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1486 self.log.info("Applying bias correction.") 

1487 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1488 trimToFit=self.config.doTrimToMatchCalib) 

1489 self.debugView(ccdExposure, "doBias") 

1490 

1491 if self.config.doVariance: 

1492 for amp, overscanResults in zip(ccd, overscans): 

1493 if ccdExposure.getBBox().contains(amp.getBBox()): 

1494 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1495 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1496 if overscanResults is not None: 

1497 self.updateVariance(ampExposure, amp, 

1498 overscanImage=overscanResults.overscanImage, 

1499 ptcDataset=ptc) 

1500 else: 

1501 self.updateVariance(ampExposure, amp, 

1502 overscanImage=None, 

1503 ptcDataset=ptc) 

1504 if self.config.qa is not None and self.config.qa.saveStats is True: 

1505 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1506 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1507 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1508 qaStats.getValue(afwMath.MEDIAN)) 

1509 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1510 qaStats.getValue(afwMath.STDEVCLIP)) 

1511 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1512 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1513 qaStats.getValue(afwMath.STDEVCLIP)) 

1514 

1515 if self.doLinearize(ccd): 

1516 self.log.info("Applying linearizer.") 

1517 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1518 detector=ccd, log=self.log) 

1519 

1520 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1521 self.log.info("Applying crosstalk correction.") 

1522 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1523 crosstalkSources=crosstalkSources, isTrimmed=True) 

1524 self.debugView(ccdExposure, "doCrosstalk") 

1525 

1526 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do 

1527 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1528 if self.config.doDefect: 

1529 self.log.info("Masking defects.") 

1530 self.maskDefect(ccdExposure, defects) 

1531 

1532 if self.config.numEdgeSuspect > 0: 

1533 self.log.info("Masking edges as SUSPECT.") 

1534 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1535 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1536 

1537 if self.config.doNanMasking: 

1538 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1539 self.maskNan(ccdExposure) 

1540 

1541 if self.config.doWidenSaturationTrails: 

1542 self.log.info("Widening saturation trails.") 

1543 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1544 

1545 if self.config.doCameraSpecificMasking: 

1546 self.log.info("Masking regions for camera specific reasons.") 

1547 self.masking.run(ccdExposure) 

1548 

1549 if self.config.doBrighterFatter: 

1550 # We need to apply flats and darks before we can interpolate, and we 

1551 # need to interpolate before we do B-F, but we do B-F without the 

1552 # flats and darks applied so we can work in units of electrons or holes. 

1553 # This context manager applies and then removes the darks and flats. 

1554 # 

1555 # We also do not want to interpolate values here, so operate on temporary 

1556 # images so we can apply only the BF-correction and roll back the 

1557 # interpolation. 

1558 interpExp = ccdExposure.clone() 

1559 with self.flatContext(interpExp, flat, dark): 

1560 isrFunctions.interpolateFromMask( 

1561 maskedImage=interpExp.getMaskedImage(), 

1562 fwhm=self.config.fwhm, 

1563 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1564 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1565 ) 

1566 bfExp = interpExp.clone() 

1567 

1568 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1569 type(bfKernel), type(bfGains)) 

1570 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1571 self.config.brighterFatterMaxIter, 

1572 self.config.brighterFatterThreshold, 

1573 self.config.brighterFatterApplyGain, 

1574 bfGains) 

1575 if bfResults[1] == self.config.brighterFatterMaxIter: 

1576 self.log.warn("Brighter-fatter correction did not converge, final difference %f.", 

1577 bfResults[0]) 

1578 else: 

1579 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1580 bfResults[1]) 

1581 image = ccdExposure.getMaskedImage().getImage() 

1582 bfCorr = bfExp.getMaskedImage().getImage() 

1583 bfCorr -= interpExp.getMaskedImage().getImage() 

1584 image += bfCorr 

1585 

1586 # Applying the brighter-fatter correction applies a 

1587 # convolution to the science image. At the edges this 

1588 # convolution may not have sufficient valid pixels to 

1589 # produce a valid correction. Mark pixels within the size 

1590 # of the brighter-fatter kernel as EDGE to warn of this 

1591 # fact. 

1592 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1593 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1594 maskPlane="EDGE") 

1595 

1596 if self.config.brighterFatterMaskGrowSize > 0: 

1597 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1598 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1599 isrFunctions.growMasks(ccdExposure.getMask(), 

1600 radius=self.config.brighterFatterMaskGrowSize, 

1601 maskNameList=maskPlane, 

1602 maskValue=maskPlane) 

1603 

1604 self.debugView(ccdExposure, "doBrighterFatter") 

1605 

1606 if self.config.doDark: 

1607 self.log.info("Applying dark correction.") 

1608 self.darkCorrection(ccdExposure, dark) 

1609 self.debugView(ccdExposure, "doDark") 

1610 

1611 if self.config.doFringe and not self.config.fringeAfterFlat: 

1612 self.log.info("Applying fringe correction before flat.") 

1613 self.fringe.run(ccdExposure, **fringes.getDict()) 

1614 self.debugView(ccdExposure, "doFringe") 

1615 

1616 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1617 self.log.info("Checking strayLight correction.") 

1618 self.strayLight.run(ccdExposure, strayLightData) 

1619 self.debugView(ccdExposure, "doStrayLight") 

1620 

1621 if self.config.doFlat: 

1622 self.log.info("Applying flat correction.") 

1623 self.flatCorrection(ccdExposure, flat) 

1624 self.debugView(ccdExposure, "doFlat") 

1625 

1626 if self.config.doApplyGains: 

1627 self.log.info("Applying gain correction instead of flat.") 

1628 if self.config.usePtcGains: 

1629 self.log.info("Using gains from the Photon Transfer Curve.") 

1630 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1631 ptcGains=ptc.gain) 

1632 else: 

1633 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1634 

1635 if self.config.doFringe and self.config.fringeAfterFlat: 

1636 self.log.info("Applying fringe correction after flat.") 

1637 self.fringe.run(ccdExposure, **fringes.getDict()) 

1638 

1639 if self.config.doVignette: 

1640 self.log.info("Constructing Vignette polygon.") 

1641 self.vignettePolygon = self.vignette.run(ccdExposure) 

1642 

1643 if self.config.vignette.doWriteVignettePolygon: 

1644 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1645 

1646 if self.config.doAttachTransmissionCurve: 

1647 self.log.info("Adding transmission curves.") 

1648 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1649 filterTransmission=filterTransmission, 

1650 sensorTransmission=sensorTransmission, 

1651 atmosphereTransmission=atmosphereTransmission) 

1652 

1653 flattenedThumb = None 

1654 if self.config.qa.doThumbnailFlattened: 

1655 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1656 

1657 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1658 self.log.info("Performing illumination correction.") 

1659 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1660 illumMaskedImage, illumScale=self.config.illumScale, 

1661 trimToFit=self.config.doTrimToMatchCalib) 

1662 

1663 preInterpExp = None 

1664 if self.config.doSaveInterpPixels: 

1665 preInterpExp = ccdExposure.clone() 

1666 

1667 # Reset and interpolate bad pixels. 

1668 # 

1669 # Large contiguous bad regions (which should have the BAD mask 

1670 # bit set) should have their values set to the image median. 

1671 # This group should include defects and bad amplifiers. As the 

1672 # area covered by these defects are large, there's little 

1673 # reason to expect that interpolation would provide a more 

1674 # useful value. 

1675 # 

1676 # Smaller defects can be safely interpolated after the larger 

1677 # regions have had their pixel values reset. This ensures 

1678 # that the remaining defects adjacent to bad amplifiers (as an 

1679 # example) do not attempt to interpolate extreme values. 

1680 if self.config.doSetBadRegions: 

1681 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1682 if badPixelCount > 0: 

1683 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1684 

1685 if self.config.doInterpolate: 

1686 self.log.info("Interpolating masked pixels.") 

1687 isrFunctions.interpolateFromMask( 

1688 maskedImage=ccdExposure.getMaskedImage(), 

1689 fwhm=self.config.fwhm, 

1690 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1691 maskNameList=list(self.config.maskListToInterpolate) 

1692 ) 

1693 

1694 self.roughZeroPoint(ccdExposure) 

1695 

1696 if self.config.doMeasureBackground: 

1697 self.log.info("Measuring background level.") 

1698 self.measureBackground(ccdExposure, self.config.qa) 

1699 

1700 if self.config.qa is not None and self.config.qa.saveStats is True: 

1701 for amp in ccd: 

1702 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1703 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1704 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1705 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1706 qaStats.getValue(afwMath.MEDIAN)) 

1707 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1708 qaStats.getValue(afwMath.STDEVCLIP)) 

1709 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1710 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1711 qaStats.getValue(afwMath.STDEVCLIP)) 

1712 

1713 self.debugView(ccdExposure, "postISRCCD") 

1714 

1715 return pipeBase.Struct( 

1716 exposure=ccdExposure, 

1717 ossThumb=ossThumb, 

1718 flattenedThumb=flattenedThumb, 

1719 

1720 preInterpolatedExposure=preInterpExp, 

1721 outputExposure=ccdExposure, 

1722 outputOssThumbnail=ossThumb, 

1723 outputFlattenedThumbnail=flattenedThumb, 

1724 ) 

1725 

1726 @pipeBase.timeMethod 

1727 def runDataRef(self, sensorRef): 

1728 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1729 

1730 This method contains the `CmdLineTask` interface to the ISR 

1731 processing. All IO is handled here, freeing the `run()` method 

1732 to manage only pixel-level calculations. The steps performed 

1733 are: 

1734 - Read in necessary detrending/isr/calibration data. 

1735 - Process raw exposure in `run()`. 

1736 - Persist the ISR-corrected exposure as "postISRCCD" if 

1737 config.doWrite=True. 

1738 

1739 Parameters 

1740 ---------- 

1741 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1742 DataRef of the detector data to be processed 

1743 

1744 Returns 

1745 ------- 

1746 result : `lsst.pipe.base.Struct` 

1747 Result struct with component: 

1748 - ``exposure`` : `afw.image.Exposure` 

1749 The fully ISR corrected exposure. 

1750 

1751 Raises 

1752 ------ 

1753 RuntimeError 

1754 Raised if a configuration option is set to True, but the 

1755 required calibration data does not exist. 

1756 

1757 """ 

1758 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1759 

1760 ccdExposure = sensorRef.get(self.config.datasetType) 

1761 

1762 camera = sensorRef.get("camera") 

1763 isrData = self.readIsrData(sensorRef, ccdExposure) 

1764 

1765 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1766 

1767 if self.config.doWrite: 

1768 sensorRef.put(result.exposure, "postISRCCD") 

1769 if result.preInterpolatedExposure is not None: 

1770 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1771 if result.ossThumb is not None: 

1772 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1773 if result.flattenedThumb is not None: 

1774 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1775 

1776 return result 

1777 

1778 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1779 """Retrieve a calibration dataset for removing instrument signature. 

1780 

1781 Parameters 

1782 ---------- 

1783 

1784 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1785 DataRef of the detector data to find calibration datasets 

1786 for. 

1787 datasetType : `str` 

1788 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1789 dateObs : `str`, optional 

1790 Date of the observation. Used to correct butler failures 

1791 when using fallback filters. 

1792 immediate : `Bool` 

1793 If True, disable butler proxies to enable error handling 

1794 within this routine. 

1795 

1796 Returns 

1797 ------- 

1798 exposure : `lsst.afw.image.Exposure` 

1799 Requested calibration frame. 

1800 

1801 Raises 

1802 ------ 

1803 RuntimeError 

1804 Raised if no matching calibration frame can be found. 

1805 """ 

1806 try: 

1807 exp = dataRef.get(datasetType, immediate=immediate) 

1808 except Exception as exc1: 

1809 if not self.config.fallbackFilterName: 

1810 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1811 try: 

1812 if self.config.useFallbackDate and dateObs: 

1813 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1814 dateObs=dateObs, immediate=immediate) 

1815 else: 

1816 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1817 except Exception as exc2: 

1818 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1819 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1820 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1821 

1822 if self.config.doAssembleIsrExposures: 

1823 exp = self.assembleCcd.assembleCcd(exp) 

1824 return exp 

1825 

1826 def ensureExposure(self, inputExp, camera, detectorNum): 

1827 """Ensure that the data returned by Butler is a fully constructed exposure. 

1828 

1829 ISR requires exposure-level image data for historical reasons, so if we did 

1830 not recieve that from Butler, construct it from what we have, modifying the 

1831 input in place. 

1832 

1833 Parameters 

1834 ---------- 

1835 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1836 `lsst.afw.image.ImageF` 

1837 The input data structure obtained from Butler. 

1838 camera : `lsst.afw.cameraGeom.camera` 

1839 The camera associated with the image. Used to find the appropriate 

1840 detector. 

1841 detectorNum : `int` 

1842 The detector this exposure should match. 

1843 

1844 Returns 

1845 ------- 

1846 inputExp : `lsst.afw.image.Exposure` 

1847 The re-constructed exposure, with appropriate detector parameters. 

1848 

1849 Raises 

1850 ------ 

1851 TypeError 

1852 Raised if the input data cannot be used to construct an exposure. 

1853 """ 

1854 if isinstance(inputExp, afwImage.DecoratedImageU): 

1855 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1856 elif isinstance(inputExp, afwImage.ImageF): 

1857 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1858 elif isinstance(inputExp, afwImage.MaskedImageF): 

1859 inputExp = afwImage.makeExposure(inputExp) 

1860 elif isinstance(inputExp, afwImage.Exposure): 

1861 pass 

1862 elif inputExp is None: 

1863 # Assume this will be caught by the setup if it is a problem. 

1864 return inputExp 

1865 else: 

1866 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1867 (type(inputExp), )) 

1868 

1869 if inputExp.getDetector() is None: 

1870 inputExp.setDetector(camera[detectorNum]) 

1871 

1872 return inputExp 

1873 

1874 def convertIntToFloat(self, exposure): 

1875 """Convert exposure image from uint16 to float. 

1876 

1877 If the exposure does not need to be converted, the input is 

1878 immediately returned. For exposures that are converted to use 

1879 floating point pixels, the variance is set to unity and the 

1880 mask to zero. 

1881 

1882 Parameters 

1883 ---------- 

1884 exposure : `lsst.afw.image.Exposure` 

1885 The raw exposure to be converted. 

1886 

1887 Returns 

1888 ------- 

1889 newexposure : `lsst.afw.image.Exposure` 

1890 The input ``exposure``, converted to floating point pixels. 

1891 

1892 Raises 

1893 ------ 

1894 RuntimeError 

1895 Raised if the exposure type cannot be converted to float. 

1896 

1897 """ 

1898 if isinstance(exposure, afwImage.ExposureF): 

1899 # Nothing to be done 

1900 self.log.debug("Exposure already of type float.") 

1901 return exposure 

1902 if not hasattr(exposure, "convertF"): 

1903 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1904 

1905 newexposure = exposure.convertF() 

1906 newexposure.variance[:] = 1 

1907 newexposure.mask[:] = 0x0 

1908 

1909 return newexposure 

1910 

1911 def maskAmplifier(self, ccdExposure, amp, defects): 

1912 """Identify bad amplifiers, saturated and suspect pixels. 

1913 

1914 Parameters 

1915 ---------- 

1916 ccdExposure : `lsst.afw.image.Exposure` 

1917 Input exposure to be masked. 

1918 amp : `lsst.afw.table.AmpInfoCatalog` 

1919 Catalog of parameters defining the amplifier on this 

1920 exposure to mask. 

1921 defects : `lsst.ip.isr.Defects` 

1922 List of defects. Used to determine if the entire 

1923 amplifier is bad. 

1924 

1925 Returns 

1926 ------- 

1927 badAmp : `Bool` 

1928 If this is true, the entire amplifier area is covered by 

1929 defects and unusable. 

1930 

1931 """ 

1932 maskedImage = ccdExposure.getMaskedImage() 

1933 

1934 badAmp = False 

1935 

1936 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1937 # comparison with current defects definition. 

1938 if defects is not None: 

1939 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1940 

1941 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1942 # association with pixels in current ccdExposure). 

1943 if badAmp: 

1944 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1945 afwImage.PARENT) 

1946 maskView = dataView.getMask() 

1947 maskView |= maskView.getPlaneBitMask("BAD") 

1948 del maskView 

1949 return badAmp 

1950 

1951 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1952 # Saturation and suspect pixels can be masked now, though. 

1953 limits = dict() 

1954 if self.config.doSaturation and not badAmp: 

1955 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1956 if self.config.doSuspect and not badAmp: 

1957 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1958 if math.isfinite(self.config.saturation): 

1959 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1960 

1961 for maskName, maskThreshold in limits.items(): 

1962 if not math.isnan(maskThreshold): 

1963 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1964 isrFunctions.makeThresholdMask( 

1965 maskedImage=dataView, 

1966 threshold=maskThreshold, 

1967 growFootprints=0, 

1968 maskName=maskName 

1969 ) 

1970 

1971 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1972 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1973 afwImage.PARENT) 

1974 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1975 self.config.suspectMaskName]) 

1976 if numpy.all(maskView.getArray() & maskVal > 0): 

1977 badAmp = True 

1978 maskView |= maskView.getPlaneBitMask("BAD") 

1979 

1980 return badAmp 

1981 

1982 def overscanCorrection(self, ccdExposure, amp): 

1983 """Apply overscan correction in place. 

1984 

1985 This method does initial pixel rejection of the overscan 

1986 region. The overscan can also be optionally segmented to 

1987 allow for discontinuous overscan responses to be fit 

1988 separately. The actual overscan subtraction is performed by 

1989 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1990 which is called here after the amplifier is preprocessed. 

1991 

1992 Parameters 

1993 ---------- 

1994 ccdExposure : `lsst.afw.image.Exposure` 

1995 Exposure to have overscan correction performed. 

1996 amp : `lsst.afw.cameraGeom.Amplifer` 

1997 The amplifier to consider while correcting the overscan. 

1998 

1999 Returns 

2000 ------- 

2001 overscanResults : `lsst.pipe.base.Struct` 

2002 Result struct with components: 

2003 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

2004 Value or fit subtracted from the amplifier image data. 

2005 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

2006 Value or fit subtracted from the overscan image data. 

2007 - ``overscanImage`` : `lsst.afw.image.Image` 

2008 Image of the overscan region with the overscan 

2009 correction applied. This quantity is used to estimate 

2010 the amplifier read noise empirically. 

2011 

2012 Raises 

2013 ------ 

2014 RuntimeError 

2015 Raised if the ``amp`` does not contain raw pixel information. 

2016 

2017 See Also 

2018 -------- 

2019 lsst.ip.isr.isrFunctions.overscanCorrection 

2020 """ 

2021 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2022 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2023 return None 

2024 

2025 statControl = afwMath.StatisticsControl() 

2026 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2027 

2028 # Determine the bounding boxes 

2029 dataBBox = amp.getRawDataBBox() 

2030 oscanBBox = amp.getRawHorizontalOverscanBBox() 

2031 dx0 = 0 

2032 dx1 = 0 

2033 

2034 prescanBBox = amp.getRawPrescanBBox() 

2035 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

2036 dx0 += self.config.overscanNumLeadingColumnsToSkip 

2037 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

2038 else: 

2039 dx0 += self.config.overscanNumTrailingColumnsToSkip 

2040 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

2041 

2042 # Determine if we need to work on subregions of the amplifier and overscan. 

2043 imageBBoxes = [] 

2044 overscanBBoxes = [] 

2045 

2046 if ((self.config.overscanBiasJump 

2047 and self.config.overscanBiasJumpLocation) 

2048 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

2049 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

2050 self.config.overscanBiasJumpDevices)): 

2051 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

2052 yLower = self.config.overscanBiasJumpLocation 

2053 yUpper = dataBBox.getHeight() - yLower 

2054 else: 

2055 yUpper = self.config.overscanBiasJumpLocation 

2056 yLower = dataBBox.getHeight() - yUpper 

2057 

2058 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2059 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

2060 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2061 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2062 yLower))) 

2063 

2064 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

2065 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

2066 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

2067 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2068 yUpper))) 

2069 else: 

2070 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2071 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

2072 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2073 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2074 oscanBBox.getHeight()))) 

2075 

2076 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

2077 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

2078 ampImage = ccdExposure.maskedImage[imageBBox] 

2079 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2080 

2081 overscanArray = overscanImage.image.array 

2082 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2083 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2084 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2085 

2086 statControl = afwMath.StatisticsControl() 

2087 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2088 

2089 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2090 

2091 # Measure average overscan levels and record them in the metadata. 

2092 levelStat = afwMath.MEDIAN 

2093 sigmaStat = afwMath.STDEVCLIP 

2094 

2095 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2096 self.config.qa.flatness.nIter) 

2097 metadata = ccdExposure.getMetadata() 

2098 ampNum = amp.getName() 

2099 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2100 if isinstance(overscanResults.overscanFit, float): 

2101 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2102 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2103 else: 

2104 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2105 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2106 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2107 

2108 return overscanResults 

2109 

2110 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2111 """Set the variance plane using the gain and read noise 

2112 

2113 The read noise is calculated from the ``overscanImage`` if the 

2114 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2115 the value from the amplifier data is used. 

2116 

2117 Parameters 

2118 ---------- 

2119 ampExposure : `lsst.afw.image.Exposure` 

2120 Exposure to process. 

2121 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2122 Amplifier detector data. 

2123 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2124 Image of overscan, required only for empirical read noise. 

2125 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2126 PTC dataset containing the gains and read noise. 

2127 

2128 

2129 Raises 

2130 ------ 

2131 RuntimeError 

2132 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2133 are ``True``, but ptcDataset is not provided. 

2134 

2135 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2136 ``overscanImage`` is ``None``. 

2137 

2138 See also 

2139 -------- 

2140 lsst.ip.isr.isrFunctions.updateVariance 

2141 """ 

2142 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2143 if self.config.usePtcGains: 

2144 if ptcDataset is None: 

2145 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2146 else: 

2147 gain = ptcDataset.gain[amp.getName()] 

2148 self.log.info("Using gain from Photon Transfer Curve.") 

2149 else: 

2150 gain = amp.getGain() 

2151 

2152 if math.isnan(gain): 

2153 gain = 1.0 

2154 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2155 elif gain <= 0: 

2156 patchedGain = 1.0 

2157 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

2158 amp.getName(), gain, patchedGain) 

2159 gain = patchedGain 

2160 

2161 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2162 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2163 

2164 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2165 stats = afwMath.StatisticsControl() 

2166 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2167 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2168 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2169 amp.getName(), readNoise) 

2170 elif self.config.usePtcReadNoise: 

2171 if ptcDataset is None: 

2172 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2173 else: 

2174 readNoise = ptcDataset.noise[amp.getName()] 

2175 self.log.info("Using read noise from Photon Transfer Curve.") 

2176 else: 

2177 readNoise = amp.getReadNoise() 

2178 

2179 isrFunctions.updateVariance( 

2180 maskedImage=ampExposure.getMaskedImage(), 

2181 gain=gain, 

2182 readNoise=readNoise, 

2183 ) 

2184 

2185 def darkCorrection(self, exposure, darkExposure, invert=False): 

2186 """Apply dark correction in place. 

2187 

2188 Parameters 

2189 ---------- 

2190 exposure : `lsst.afw.image.Exposure` 

2191 Exposure to process. 

2192 darkExposure : `lsst.afw.image.Exposure` 

2193 Dark exposure of the same size as ``exposure``. 

2194 invert : `Bool`, optional 

2195 If True, re-add the dark to an already corrected image. 

2196 

2197 Raises 

2198 ------ 

2199 RuntimeError 

2200 Raised if either ``exposure`` or ``darkExposure`` do not 

2201 have their dark time defined. 

2202 

2203 See Also 

2204 -------- 

2205 lsst.ip.isr.isrFunctions.darkCorrection 

2206 """ 

2207 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2208 if math.isnan(expScale): 

2209 raise RuntimeError("Exposure darktime is NAN.") 

2210 if darkExposure.getInfo().getVisitInfo() is not None \ 

2211 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2212 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2213 else: 

2214 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2215 # so getDarkTime() does not exist. 

2216 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2217 darkScale = 1.0 

2218 

2219 isrFunctions.darkCorrection( 

2220 maskedImage=exposure.getMaskedImage(), 

2221 darkMaskedImage=darkExposure.getMaskedImage(), 

2222 expScale=expScale, 

2223 darkScale=darkScale, 

2224 invert=invert, 

2225 trimToFit=self.config.doTrimToMatchCalib 

2226 ) 

2227 

2228 def doLinearize(self, detector): 

2229 """Check if linearization is needed for the detector cameraGeom. 

2230 

2231 Checks config.doLinearize and the linearity type of the first 

2232 amplifier. 

2233 

2234 Parameters 

2235 ---------- 

2236 detector : `lsst.afw.cameraGeom.Detector` 

2237 Detector to get linearity type from. 

2238 

2239 Returns 

2240 ------- 

2241 doLinearize : `Bool` 

2242 If True, linearization should be performed. 

2243 """ 

2244 return self.config.doLinearize and \ 

2245 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2246 

2247 def flatCorrection(self, exposure, flatExposure, invert=False): 

2248 """Apply flat correction in place. 

2249 

2250 Parameters 

2251 ---------- 

2252 exposure : `lsst.afw.image.Exposure` 

2253 Exposure to process. 

2254 flatExposure : `lsst.afw.image.Exposure` 

2255 Flat exposure of the same size as ``exposure``. 

2256 invert : `Bool`, optional 

2257 If True, unflatten an already flattened image. 

2258 

2259 See Also 

2260 -------- 

2261 lsst.ip.isr.isrFunctions.flatCorrection 

2262 """ 

2263 isrFunctions.flatCorrection( 

2264 maskedImage=exposure.getMaskedImage(), 

2265 flatMaskedImage=flatExposure.getMaskedImage(), 

2266 scalingType=self.config.flatScalingType, 

2267 userScale=self.config.flatUserScale, 

2268 invert=invert, 

2269 trimToFit=self.config.doTrimToMatchCalib 

2270 ) 

2271 

2272 def saturationDetection(self, exposure, amp): 

2273 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2274 

2275 Parameters 

2276 ---------- 

2277 exposure : `lsst.afw.image.Exposure` 

2278 Exposure to process. Only the amplifier DataSec is processed. 

2279 amp : `lsst.afw.table.AmpInfoCatalog` 

2280 Amplifier detector data. 

2281 

2282 See Also 

2283 -------- 

2284 lsst.ip.isr.isrFunctions.makeThresholdMask 

2285 """ 

2286 if not math.isnan(amp.getSaturation()): 

2287 maskedImage = exposure.getMaskedImage() 

2288 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2289 isrFunctions.makeThresholdMask( 

2290 maskedImage=dataView, 

2291 threshold=amp.getSaturation(), 

2292 growFootprints=0, 

2293 maskName=self.config.saturatedMaskName, 

2294 ) 

2295 

2296 def saturationInterpolation(self, exposure): 

2297 """Interpolate over saturated pixels, in place. 

2298 

2299 This method should be called after `saturationDetection`, to 

2300 ensure that the saturated pixels have been identified in the 

2301 SAT mask. It should also be called after `assembleCcd`, since 

2302 saturated regions may cross amplifier boundaries. 

2303 

2304 Parameters 

2305 ---------- 

2306 exposure : `lsst.afw.image.Exposure` 

2307 Exposure to process. 

2308 

2309 See Also 

2310 -------- 

2311 lsst.ip.isr.isrTask.saturationDetection 

2312 lsst.ip.isr.isrFunctions.interpolateFromMask 

2313 """ 

2314 isrFunctions.interpolateFromMask( 

2315 maskedImage=exposure.getMaskedImage(), 

2316 fwhm=self.config.fwhm, 

2317 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2318 maskNameList=list(self.config.saturatedMaskName), 

2319 ) 

2320 

2321 def suspectDetection(self, exposure, amp): 

2322 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2323 

2324 Parameters 

2325 ---------- 

2326 exposure : `lsst.afw.image.Exposure` 

2327 Exposure to process. Only the amplifier DataSec is processed. 

2328 amp : `lsst.afw.table.AmpInfoCatalog` 

2329 Amplifier detector data. 

2330 

2331 See Also 

2332 -------- 

2333 lsst.ip.isr.isrFunctions.makeThresholdMask 

2334 

2335 Notes 

2336 ----- 

2337 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2338 This is intended to indicate pixels that may be affected by unknown systematics; 

2339 for example if non-linearity corrections above a certain level are unstable 

2340 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2341 that no such level exists and no pixels are to be masked as suspicious. 

2342 """ 

2343 suspectLevel = amp.getSuspectLevel() 

2344 if math.isnan(suspectLevel): 

2345 return 

2346 

2347 maskedImage = exposure.getMaskedImage() 

2348 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2349 isrFunctions.makeThresholdMask( 

2350 maskedImage=dataView, 

2351 threshold=suspectLevel, 

2352 growFootprints=0, 

2353 maskName=self.config.suspectMaskName, 

2354 ) 

2355 

2356 def maskDefect(self, exposure, defectBaseList): 

2357 """Mask defects using mask plane "BAD", in place. 

2358 

2359 Parameters 

2360 ---------- 

2361 exposure : `lsst.afw.image.Exposure` 

2362 Exposure to process. 

2363 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2364 `lsst.afw.image.DefectBase`. 

2365 List of defects to mask. 

2366 

2367 Notes 

2368 ----- 

2369 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2370 """ 

2371 maskedImage = exposure.getMaskedImage() 

2372 if not isinstance(defectBaseList, Defects): 

2373 # Promotes DefectBase to Defect 

2374 defectList = Defects(defectBaseList) 

2375 else: 

2376 defectList = defectBaseList 

2377 defectList.maskPixels(maskedImage, maskName="BAD") 

2378 

2379 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2380 """Mask edge pixels with applicable mask plane. 

2381 

2382 Parameters 

2383 ---------- 

2384 exposure : `lsst.afw.image.Exposure` 

2385 Exposure to process. 

2386 numEdgePixels : `int`, optional 

2387 Number of edge pixels to mask. 

2388 maskPlane : `str`, optional 

2389 Mask plane name to use. 

2390 level : `str`, optional 

2391 Level at which to mask edges. 

2392 """ 

2393 maskedImage = exposure.getMaskedImage() 

2394 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2395 

2396 if numEdgePixels > 0: 

2397 if level == 'DETECTOR': 

2398 boxes = [maskedImage.getBBox()] 

2399 elif level == 'AMP': 

2400 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2401 

2402 for box in boxes: 

2403 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2404 subImage = maskedImage[box] 

2405 box.grow(-numEdgePixels) 

2406 # Mask pixels outside box 

2407 SourceDetectionTask.setEdgeBits( 

2408 subImage, 

2409 box, 

2410 maskBitMask) 

2411 

2412 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2413 """Mask and interpolate defects using mask plane "BAD", in place. 

2414 

2415 Parameters 

2416 ---------- 

2417 exposure : `lsst.afw.image.Exposure` 

2418 Exposure to process. 

2419 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2420 `lsst.afw.image.DefectBase`. 

2421 List of defects to mask and interpolate. 

2422 

2423 See Also 

2424 -------- 

2425 lsst.ip.isr.isrTask.maskDefect 

2426 """ 

2427 self.maskDefect(exposure, defectBaseList) 

2428 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2429 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2430 isrFunctions.interpolateFromMask( 

2431 maskedImage=exposure.getMaskedImage(), 

2432 fwhm=self.config.fwhm, 

2433 growSaturatedFootprints=0, 

2434 maskNameList=["BAD"], 

2435 ) 

2436 

2437 def maskNan(self, exposure): 

2438 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2439 

2440 Parameters 

2441 ---------- 

2442 exposure : `lsst.afw.image.Exposure` 

2443 Exposure to process. 

2444 

2445 Notes 

2446 ----- 

2447 We mask over all non-finite values (NaN, inf), including those 

2448 that are masked with other bits (because those may or may not be 

2449 interpolated over later, and we want to remove all NaN/infs). 

2450 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2451 preserve the historical name. 

2452 """ 

2453 maskedImage = exposure.getMaskedImage() 

2454 

2455 # Find and mask NaNs 

2456 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2457 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2458 numNans = maskNans(maskedImage, maskVal) 

2459 self.metadata.set("NUMNANS", numNans) 

2460 if numNans > 0: 

2461 self.log.warn("There were %d unmasked NaNs.", numNans) 

2462 

2463 def maskAndInterpolateNan(self, exposure): 

2464 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2465 in place. 

2466 

2467 Parameters 

2468 ---------- 

2469 exposure : `lsst.afw.image.Exposure` 

2470 Exposure to process. 

2471 

2472 See Also 

2473 -------- 

2474 lsst.ip.isr.isrTask.maskNan 

2475 """ 

2476 self.maskNan(exposure) 

2477 isrFunctions.interpolateFromMask( 

2478 maskedImage=exposure.getMaskedImage(), 

2479 fwhm=self.config.fwhm, 

2480 growSaturatedFootprints=0, 

2481 maskNameList=["UNMASKEDNAN"], 

2482 ) 

2483 

2484 def measureBackground(self, exposure, IsrQaConfig=None): 

2485 """Measure the image background in subgrids, for quality control purposes. 

2486 

2487 Parameters 

2488 ---------- 

2489 exposure : `lsst.afw.image.Exposure` 

2490 Exposure to process. 

2491 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2492 Configuration object containing parameters on which background 

2493 statistics and subgrids to use. 

2494 """ 

2495 if IsrQaConfig is not None: 

2496 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2497 IsrQaConfig.flatness.nIter) 

2498 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2499 statsControl.setAndMask(maskVal) 

2500 maskedImage = exposure.getMaskedImage() 

2501 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2502 skyLevel = stats.getValue(afwMath.MEDIAN) 

2503 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2504 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2505 metadata = exposure.getMetadata() 

2506 metadata.set('SKYLEVEL', skyLevel) 

2507 metadata.set('SKYSIGMA', skySigma) 

2508 

2509 # calcluating flatlevel over the subgrids 

2510 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2511 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2512 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2513 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2514 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2515 skyLevels = numpy.zeros((nX, nY)) 

2516 

2517 for j in range(nY): 

2518 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2519 for i in range(nX): 

2520 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2521 

2522 xLLC = xc - meshXHalf 

2523 yLLC = yc - meshYHalf 

2524 xURC = xc + meshXHalf - 1 

2525 yURC = yc + meshYHalf - 1 

2526 

2527 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2528 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2529 

2530 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2531 

2532 good = numpy.where(numpy.isfinite(skyLevels)) 

2533 skyMedian = numpy.median(skyLevels[good]) 

2534 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2535 flatness_rms = numpy.std(flatness) 

2536 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2537 

2538 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2539 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2540 nX, nY, flatness_pp, flatness_rms) 

2541 

2542 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2543 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2544 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2545 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2546 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2547 

2548 def roughZeroPoint(self, exposure): 

2549 """Set an approximate magnitude zero point for the exposure. 

2550 

2551 Parameters 

2552 ---------- 

2553 exposure : `lsst.afw.image.Exposure` 

2554 Exposure to process. 

2555 """ 

2556 filterLabel = exposure.getFilterLabel() 

2557 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2558 

2559 if physicalFilter in self.config.fluxMag0T1: 

2560 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2561 else: 

2562 self.log.warn("No rough magnitude zero point defined for filter {}.".format(physicalFilter)) 

2563 fluxMag0 = self.config.defaultFluxMag0T1 

2564 

2565 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2566 if not expTime > 0: # handle NaN as well as <= 0 

2567 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2568 return 

2569 

2570 self.log.info("Setting rough magnitude zero point for filter {}: {}". 

2571 format(physicalFilter, 2.5*math.log10(fluxMag0*expTime))) 

2572 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2573 

2574 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2575 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2576 

2577 Parameters 

2578 ---------- 

2579 ccdExposure : `lsst.afw.image.Exposure` 

2580 Exposure to process. 

2581 fpPolygon : `lsst.afw.geom.Polygon` 

2582 Polygon in focal plane coordinates. 

2583 """ 

2584 # Get ccd corners in focal plane coordinates 

2585 ccd = ccdExposure.getDetector() 

2586 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2587 ccdPolygon = Polygon(fpCorners) 

2588 

2589 # Get intersection of ccd corners with fpPolygon 

2590 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2591 

2592 # Transform back to pixel positions and build new polygon 

2593 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2594 validPolygon = Polygon(ccdPoints) 

2595 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2596 

2597 @contextmanager 

2598 def flatContext(self, exp, flat, dark=None): 

2599 """Context manager that applies and removes flats and darks, 

2600 if the task is configured to apply them. 

2601 

2602 Parameters 

2603 ---------- 

2604 exp : `lsst.afw.image.Exposure` 

2605 Exposure to process. 

2606 flat : `lsst.afw.image.Exposure` 

2607 Flat exposure the same size as ``exp``. 

2608 dark : `lsst.afw.image.Exposure`, optional 

2609 Dark exposure the same size as ``exp``. 

2610 

2611 Yields 

2612 ------ 

2613 exp : `lsst.afw.image.Exposure` 

2614 The flat and dark corrected exposure. 

2615 """ 

2616 if self.config.doDark and dark is not None: 

2617 self.darkCorrection(exp, dark) 

2618 if self.config.doFlat: 

2619 self.flatCorrection(exp, flat) 

2620 try: 

2621 yield exp 

2622 finally: 

2623 if self.config.doFlat: 

2624 self.flatCorrection(exp, flat, invert=True) 

2625 if self.config.doDark and dark is not None: 

2626 self.darkCorrection(exp, dark, invert=True) 

2627 

2628 def debugView(self, exposure, stepname): 

2629 """Utility function to examine ISR exposure at different stages. 

2630 

2631 Parameters 

2632 ---------- 

2633 exposure : `lsst.afw.image.Exposure` 

2634 Exposure to view. 

2635 stepname : `str` 

2636 State of processing to view. 

2637 """ 

2638 frame = getDebugFrame(self._display, stepname) 

2639 if frame: 

2640 display = getDisplay(frame) 

2641 display.scale('asinh', 'zscale') 

2642 display.mtv(exposure) 

2643 prompt = "Press Enter to continue [c]... " 

2644 while True: 

2645 ans = input(prompt).lower() 

2646 if ans in ("", "c",): 

2647 break 

2648 

2649 

2650class FakeAmp(object): 

2651 """A Detector-like object that supports returning gain and saturation level 

2652 

2653 This is used when the input exposure does not have a detector. 

2654 

2655 Parameters 

2656 ---------- 

2657 exposure : `lsst.afw.image.Exposure` 

2658 Exposure to generate a fake amplifier for. 

2659 config : `lsst.ip.isr.isrTaskConfig` 

2660 Configuration to apply to the fake amplifier. 

2661 """ 

2662 

2663 def __init__(self, exposure, config): 

2664 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2665 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2666 self._gain = config.gain 

2667 self._readNoise = config.readNoise 

2668 self._saturation = config.saturation 

2669 

2670 def getBBox(self): 

2671 return self._bbox 

2672 

2673 def getRawBBox(self): 

2674 return self._bbox 

2675 

2676 def getRawHorizontalOverscanBBox(self): 

2677 return self._RawHorizontalOverscanBBox 

2678 

2679 def getGain(self): 

2680 return self._gain 

2681 

2682 def getReadNoise(self): 

2683 return self._readNoise 

2684 

2685 def getSaturation(self): 

2686 return self._saturation 

2687 

2688 def getSuspectLevel(self): 

2689 return float("NaN") 

2690 

2691 

2692class RunIsrConfig(pexConfig.Config): 

2693 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2694 

2695 

2696class RunIsrTask(pipeBase.CmdLineTask): 

2697 """Task to wrap the default IsrTask to allow it to be retargeted. 

2698 

2699 The standard IsrTask can be called directly from a command line 

2700 program, but doing so removes the ability of the task to be 

2701 retargeted. As most cameras override some set of the IsrTask 

2702 methods, this would remove those data-specific methods in the 

2703 output post-ISR images. This wrapping class fixes the issue, 

2704 allowing identical post-ISR images to be generated by both the 

2705 processCcd and isrTask code. 

2706 """ 

2707 ConfigClass = RunIsrConfig 

2708 _DefaultName = "runIsr" 

2709 

2710 def __init__(self, *args, **kwargs): 

2711 super().__init__(*args, **kwargs) 

2712 self.makeSubtask("isr") 

2713 

2714 def runDataRef(self, dataRef): 

2715 """ 

2716 Parameters 

2717 ---------- 

2718 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2719 data reference of the detector data to be processed 

2720 

2721 Returns 

2722 ------- 

2723 result : `pipeBase.Struct` 

2724 Result struct with component: 

2725 

2726 - exposure : `lsst.afw.image.Exposure` 

2727 Post-ISR processed exposure. 

2728 """ 

2729 return self.isr.runDataRef(dataRef)