Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42 

43from . import isrFunctions 

44from . import isrQa 

45from . import linearize 

46from .defects import Defects 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 minimum=0, # can fall back to cameraGeom 

123 ) 

124 # TODO: DM-25348. This does not work yet to correctly load 

125 # possible crosstalk sources. 

126 crosstalkSources = cT.PrerequisiteInput( 

127 name="isrOverscanCorrected", 

128 doc="Overscan corrected input images.", 

129 storageClass="Exposure", 

130 dimensions=["instrument", "exposure", "detector"], 

131 deferLoad=True, 

132 multiple=True, 

133 lookupFunction=crosstalkSourceLookup, 

134 minimum=0, # not needed for all instruments, no config to control this 

135 ) 

136 bias = cT.PrerequisiteInput( 

137 name="bias", 

138 doc="Input bias calibration.", 

139 storageClass="ExposureF", 

140 dimensions=["instrument", "detector"], 

141 isCalibration=True, 

142 ) 

143 dark = cT.PrerequisiteInput( 

144 name='dark', 

145 doc="Input dark calibration.", 

146 storageClass="ExposureF", 

147 dimensions=["instrument", "detector"], 

148 isCalibration=True, 

149 ) 

150 flat = cT.PrerequisiteInput( 

151 name="flat", 

152 doc="Input flat calibration.", 

153 storageClass="ExposureF", 

154 dimensions=["instrument", "physical_filter", "detector"], 

155 isCalibration=True, 

156 ) 

157 ptc = cT.PrerequisiteInput( 

158 name="ptc", 

159 doc="Input Photon Transfer Curve dataset", 

160 storageClass="PhotonTransferCurveDataset", 

161 dimensions=["instrument", "detector"], 

162 isCalibration=True, 

163 ) 

164 fringes = cT.PrerequisiteInput( 

165 name="fringe", 

166 doc="Input fringe calibration.", 

167 storageClass="ExposureF", 

168 dimensions=["instrument", "physical_filter", "detector"], 

169 isCalibration=True, 

170 minimum=0, # only needed for some bands, even when enabled 

171 ) 

172 strayLightData = cT.PrerequisiteInput( 

173 name='yBackground', 

174 doc="Input stray light calibration.", 

175 storageClass="StrayLightData", 

176 dimensions=["instrument", "physical_filter", "detector"], 

177 isCalibration=True, 

178 minimum=0, # only needed for some bands, even when enabled 

179 ) 

180 bfKernel = cT.PrerequisiteInput( 

181 name='bfKernel', 

182 doc="Input brighter-fatter kernel.", 

183 storageClass="NumpyArray", 

184 dimensions=["instrument"], 

185 isCalibration=True, 

186 minimum=0, # can use either bfKernel or newBFKernel 

187 ) 

188 newBFKernel = cT.PrerequisiteInput( 

189 name='brighterFatterKernel', 

190 doc="Newer complete kernel + gain solutions.", 

191 storageClass="BrighterFatterKernel", 

192 dimensions=["instrument", "detector"], 

193 isCalibration=True, 

194 minimum=0, # can use either bfKernel or newBFKernel 

195 ) 

196 defects = cT.PrerequisiteInput( 

197 name='defects', 

198 doc="Input defect tables.", 

199 storageClass="Defects", 

200 dimensions=["instrument", "detector"], 

201 isCalibration=True, 

202 ) 

203 linearizer = cT.PrerequisiteInput( 

204 name='linearizer', 

205 storageClass="Linearizer", 

206 doc="Linearity correction calibration.", 

207 dimensions=["instrument", "detector"], 

208 isCalibration=True, 

209 minimum=0, # can fall back to cameraGeom 

210 ) 

211 opticsTransmission = cT.PrerequisiteInput( 

212 name="transmission_optics", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the optics.", 

215 dimensions=["instrument"], 

216 isCalibration=True, 

217 ) 

218 filterTransmission = cT.PrerequisiteInput( 

219 name="transmission_filter", 

220 storageClass="TransmissionCurve", 

221 doc="Transmission curve due to the filter.", 

222 dimensions=["instrument", "physical_filter"], 

223 isCalibration=True, 

224 ) 

225 sensorTransmission = cT.PrerequisiteInput( 

226 name="transmission_sensor", 

227 storageClass="TransmissionCurve", 

228 doc="Transmission curve due to the sensor.", 

229 dimensions=["instrument", "detector"], 

230 isCalibration=True, 

231 ) 

232 atmosphereTransmission = cT.PrerequisiteInput( 

233 name="transmission_atmosphere", 

234 storageClass="TransmissionCurve", 

235 doc="Transmission curve due to the atmosphere.", 

236 dimensions=["instrument"], 

237 isCalibration=True, 

238 ) 

239 illumMaskedImage = cT.PrerequisiteInput( 

240 name="illum", 

241 doc="Input illumination correction.", 

242 storageClass="MaskedImageF", 

243 dimensions=["instrument", "physical_filter", "detector"], 

244 isCalibration=True, 

245 ) 

246 

247 outputExposure = cT.Output( 

248 name='postISRCCD', 

249 doc="Output ISR processed exposure.", 

250 storageClass="Exposure", 

251 dimensions=["instrument", "exposure", "detector"], 

252 ) 

253 preInterpExposure = cT.Output( 

254 name='preInterpISRCCD', 

255 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

256 storageClass="ExposureF", 

257 dimensions=["instrument", "exposure", "detector"], 

258 ) 

259 outputOssThumbnail = cT.Output( 

260 name="OssThumb", 

261 doc="Output Overscan-subtracted thumbnail image.", 

262 storageClass="Thumbnail", 

263 dimensions=["instrument", "exposure", "detector"], 

264 ) 

265 outputFlattenedThumbnail = cT.Output( 

266 name="FlattenedThumb", 

267 doc="Output flat-corrected thumbnail image.", 

268 storageClass="Thumbnail", 

269 dimensions=["instrument", "exposure", "detector"], 

270 ) 

271 

272 def __init__(self, *, config=None): 

273 super().__init__(config=config) 

274 

275 if config.doBias is not True: 

276 self.prerequisiteInputs.discard("bias") 

277 if config.doLinearize is not True: 

278 self.prerequisiteInputs.discard("linearizer") 

279 if config.doCrosstalk is not True: 

280 self.inputs.discard("crosstalkSources") 

281 self.prerequisiteInputs.discard("crosstalk") 

282 if config.doBrighterFatter is not True: 

283 self.prerequisiteInputs.discard("bfKernel") 

284 self.prerequisiteInputs.discard("newBFKernel") 

285 if config.doDefect is not True: 

286 self.prerequisiteInputs.discard("defects") 

287 if config.doDark is not True: 

288 self.prerequisiteInputs.discard("dark") 

289 if config.doFlat is not True: 

290 self.prerequisiteInputs.discard("flat") 

291 if config.doFringe is not True: 

292 self.prerequisiteInputs.discard("fringe") 

293 if config.doStrayLight is not True: 

294 self.prerequisiteInputs.discard("strayLightData") 

295 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

296 self.prerequisiteInputs.discard("ptc") 

297 if config.doAttachTransmissionCurve is not True: 

298 self.prerequisiteInputs.discard("opticsTransmission") 

299 self.prerequisiteInputs.discard("filterTransmission") 

300 self.prerequisiteInputs.discard("sensorTransmission") 

301 self.prerequisiteInputs.discard("atmosphereTransmission") 

302 if config.doUseOpticsTransmission is not True: 

303 self.prerequisiteInputs.discard("opticsTransmission") 

304 if config.doUseFilterTransmission is not True: 

305 self.prerequisiteInputs.discard("filterTransmission") 

306 if config.doUseSensorTransmission is not True: 

307 self.prerequisiteInputs.discard("sensorTransmission") 

308 if config.doUseAtmosphereTransmission is not True: 

309 self.prerequisiteInputs.discard("atmosphereTransmission") 

310 if config.doIlluminationCorrection is not True: 

311 self.prerequisiteInputs.discard("illumMaskedImage") 

312 

313 if config.doWrite is not True: 

314 self.outputs.discard("outputExposure") 

315 self.outputs.discard("preInterpExposure") 

316 self.outputs.discard("outputFlattenedThumbnail") 

317 self.outputs.discard("outputOssThumbnail") 

318 if config.doSaveInterpPixels is not True: 

319 self.outputs.discard("preInterpExposure") 

320 if config.qa.doThumbnailOss is not True: 

321 self.outputs.discard("outputOssThumbnail") 

322 if config.qa.doThumbnailFlattened is not True: 

323 self.outputs.discard("outputFlattenedThumbnail") 

324 

325 

326class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

327 pipelineConnections=IsrTaskConnections): 

328 """Configuration parameters for IsrTask. 

329 

330 Items are grouped in the order in which they are executed by the task. 

331 """ 

332 datasetType = pexConfig.Field( 

333 dtype=str, 

334 doc="Dataset type for input data; users will typically leave this alone, " 

335 "but camera-specific ISR tasks will override it", 

336 default="raw", 

337 ) 

338 

339 fallbackFilterName = pexConfig.Field( 

340 dtype=str, 

341 doc="Fallback default filter name for calibrations.", 

342 optional=True 

343 ) 

344 useFallbackDate = pexConfig.Field( 

345 dtype=bool, 

346 doc="Pass observation date when using fallback filter.", 

347 default=False, 

348 ) 

349 expectWcs = pexConfig.Field( 

350 dtype=bool, 

351 default=True, 

352 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

353 ) 

354 fwhm = pexConfig.Field( 

355 dtype=float, 

356 doc="FWHM of PSF in arcseconds.", 

357 default=1.0, 

358 ) 

359 qa = pexConfig.ConfigField( 

360 dtype=isrQa.IsrQaConfig, 

361 doc="QA related configuration options.", 

362 ) 

363 

364 # Image conversion configuration 

365 doConvertIntToFloat = pexConfig.Field( 

366 dtype=bool, 

367 doc="Convert integer raw images to floating point values?", 

368 default=True, 

369 ) 

370 

371 # Saturated pixel handling. 

372 doSaturation = pexConfig.Field( 

373 dtype=bool, 

374 doc="Mask saturated pixels? NB: this is totally independent of the" 

375 " interpolation option - this is ONLY setting the bits in the mask." 

376 " To have them interpolated make sure doSaturationInterpolation=True", 

377 default=True, 

378 ) 

379 saturatedMaskName = pexConfig.Field( 

380 dtype=str, 

381 doc="Name of mask plane to use in saturation detection and interpolation", 

382 default="SAT", 

383 ) 

384 saturation = pexConfig.Field( 

385 dtype=float, 

386 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

387 default=float("NaN"), 

388 ) 

389 growSaturationFootprintSize = pexConfig.Field( 

390 dtype=int, 

391 doc="Number of pixels by which to grow the saturation footprints", 

392 default=1, 

393 ) 

394 

395 # Suspect pixel handling. 

396 doSuspect = pexConfig.Field( 

397 dtype=bool, 

398 doc="Mask suspect pixels?", 

399 default=False, 

400 ) 

401 suspectMaskName = pexConfig.Field( 

402 dtype=str, 

403 doc="Name of mask plane to use for suspect pixels", 

404 default="SUSPECT", 

405 ) 

406 numEdgeSuspect = pexConfig.Field( 

407 dtype=int, 

408 doc="Number of edge pixels to be flagged as untrustworthy.", 

409 default=0, 

410 ) 

411 edgeMaskLevel = pexConfig.ChoiceField( 

412 dtype=str, 

413 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

414 default="DETECTOR", 

415 allowed={ 

416 'DETECTOR': 'Mask only the edges of the full detector.', 

417 'AMP': 'Mask edges of each amplifier.', 

418 }, 

419 ) 

420 

421 # Initial masking options. 

422 doSetBadRegions = pexConfig.Field( 

423 dtype=bool, 

424 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

425 default=True, 

426 ) 

427 badStatistic = pexConfig.ChoiceField( 

428 dtype=str, 

429 doc="How to estimate the average value for BAD regions.", 

430 default='MEANCLIP', 

431 allowed={ 

432 "MEANCLIP": "Correct using the (clipped) mean of good data", 

433 "MEDIAN": "Correct using the median of the good data", 

434 }, 

435 ) 

436 

437 # Overscan subtraction configuration. 

438 doOverscan = pexConfig.Field( 

439 dtype=bool, 

440 doc="Do overscan subtraction?", 

441 default=True, 

442 ) 

443 overscan = pexConfig.ConfigurableField( 

444 target=OverscanCorrectionTask, 

445 doc="Overscan subtraction task for image segments.", 

446 ) 

447 

448 overscanFitType = pexConfig.ChoiceField( 

449 dtype=str, 

450 doc="The method for fitting the overscan bias level.", 

451 default='MEDIAN', 

452 allowed={ 

453 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

454 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

455 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

456 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

457 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

458 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

459 "MEAN": "Correct using the mean of the overscan region", 

460 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

461 "MEDIAN": "Correct using the median of the overscan region", 

462 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

463 }, 

464 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

465 " This option will no longer be used, and will be removed after v20.") 

466 ) 

467 overscanOrder = pexConfig.Field( 

468 dtype=int, 

469 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

470 "or number of spline knots if overscan fit type is a spline."), 

471 default=1, 

472 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

473 " This option will no longer be used, and will be removed after v20.") 

474 ) 

475 overscanNumSigmaClip = pexConfig.Field( 

476 dtype=float, 

477 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

478 default=3.0, 

479 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

480 " This option will no longer be used, and will be removed after v20.") 

481 ) 

482 overscanIsInt = pexConfig.Field( 

483 dtype=bool, 

484 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

485 " and overscan.FitType=MEDIAN_PER_ROW.", 

486 default=True, 

487 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

488 " This option will no longer be used, and will be removed after v20.") 

489 ) 

490 # These options do not get deprecated, as they define how we slice up the image data. 

491 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

492 dtype=int, 

493 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

494 default=0, 

495 ) 

496 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

497 dtype=int, 

498 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

499 default=0, 

500 ) 

501 overscanMaxDev = pexConfig.Field( 501 ↛ exitline 501 didn't jump to the function exit

502 dtype=float, 

503 doc="Maximum deviation from the median for overscan", 

504 default=1000.0, check=lambda x: x > 0 

505 ) 

506 overscanBiasJump = pexConfig.Field( 

507 dtype=bool, 

508 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

509 default=False, 

510 ) 

511 overscanBiasJumpKeyword = pexConfig.Field( 

512 dtype=str, 

513 doc="Header keyword containing information about devices.", 

514 default="NO_SUCH_KEY", 

515 ) 

516 overscanBiasJumpDevices = pexConfig.ListField( 

517 dtype=str, 

518 doc="List of devices that need piecewise overscan correction.", 

519 default=(), 

520 ) 

521 overscanBiasJumpLocation = pexConfig.Field( 

522 dtype=int, 

523 doc="Location of bias jump along y-axis.", 

524 default=0, 

525 ) 

526 

527 # Amplifier to CCD assembly configuration 

528 doAssembleCcd = pexConfig.Field( 

529 dtype=bool, 

530 default=True, 

531 doc="Assemble amp-level exposures into a ccd-level exposure?" 

532 ) 

533 assembleCcd = pexConfig.ConfigurableField( 

534 target=AssembleCcdTask, 

535 doc="CCD assembly task", 

536 ) 

537 

538 # General calibration configuration. 

539 doAssembleIsrExposures = pexConfig.Field( 

540 dtype=bool, 

541 default=False, 

542 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

543 ) 

544 doTrimToMatchCalib = pexConfig.Field( 

545 dtype=bool, 

546 default=False, 

547 doc="Trim raw data to match calibration bounding boxes?" 

548 ) 

549 

550 # Bias subtraction. 

551 doBias = pexConfig.Field( 

552 dtype=bool, 

553 doc="Apply bias frame correction?", 

554 default=True, 

555 ) 

556 biasDataProductName = pexConfig.Field( 

557 dtype=str, 

558 doc="Name of the bias data product", 

559 default="bias", 

560 ) 

561 doBiasBeforeOverscan = pexConfig.Field( 

562 dtype=bool, 

563 doc="Reverse order of overscan and bias correction.", 

564 default=False 

565 ) 

566 

567 # Variance construction 

568 doVariance = pexConfig.Field( 

569 dtype=bool, 

570 doc="Calculate variance?", 

571 default=True 

572 ) 

573 gain = pexConfig.Field( 

574 dtype=float, 

575 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

576 default=float("NaN"), 

577 ) 

578 readNoise = pexConfig.Field( 

579 dtype=float, 

580 doc="The read noise to use if no Detector is present in the Exposure", 

581 default=0.0, 

582 ) 

583 doEmpiricalReadNoise = pexConfig.Field( 

584 dtype=bool, 

585 default=False, 

586 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

587 ) 

588 usePtcReadNoise = pexConfig.Field( 

589 dtype=bool, 

590 default=False, 

591 doc="Use readnoise values from the Photon Transfer Curve?" 

592 ) 

593 # Linearization. 

594 doLinearize = pexConfig.Field( 

595 dtype=bool, 

596 doc="Correct for nonlinearity of the detector's response?", 

597 default=True, 

598 ) 

599 

600 # Crosstalk. 

601 doCrosstalk = pexConfig.Field( 

602 dtype=bool, 

603 doc="Apply intra-CCD crosstalk correction?", 

604 default=False, 

605 ) 

606 doCrosstalkBeforeAssemble = pexConfig.Field( 

607 dtype=bool, 

608 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

609 default=False, 

610 ) 

611 crosstalk = pexConfig.ConfigurableField( 

612 target=CrosstalkTask, 

613 doc="Intra-CCD crosstalk correction", 

614 ) 

615 

616 # Masking options. 

617 doDefect = pexConfig.Field( 

618 dtype=bool, 

619 doc="Apply correction for CCD defects, e.g. hot pixels?", 

620 default=True, 

621 ) 

622 doNanMasking = pexConfig.Field( 

623 dtype=bool, 

624 doc="Mask non-finite (NAN, inf) pixels?", 

625 default=True, 

626 ) 

627 doWidenSaturationTrails = pexConfig.Field( 

628 dtype=bool, 

629 doc="Widen bleed trails based on their width?", 

630 default=True 

631 ) 

632 

633 # Brighter-Fatter correction. 

634 doBrighterFatter = pexConfig.Field( 

635 dtype=bool, 

636 default=False, 

637 doc="Apply the brighter-fatter correction?" 

638 ) 

639 brighterFatterLevel = pexConfig.ChoiceField( 

640 dtype=str, 

641 default="DETECTOR", 

642 doc="The level at which to correct for brighter-fatter.", 

643 allowed={ 

644 "AMP": "Every amplifier treated separately.", 

645 "DETECTOR": "One kernel per detector", 

646 } 

647 ) 

648 brighterFatterMaxIter = pexConfig.Field( 

649 dtype=int, 

650 default=10, 

651 doc="Maximum number of iterations for the brighter-fatter correction" 

652 ) 

653 brighterFatterThreshold = pexConfig.Field( 

654 dtype=float, 

655 default=1000, 

656 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

657 "absolute value of the difference between the current corrected image and the one " 

658 "from the previous iteration summed over all the pixels." 

659 ) 

660 brighterFatterApplyGain = pexConfig.Field( 

661 dtype=bool, 

662 default=True, 

663 doc="Should the gain be applied when applying the brighter-fatter correction?" 

664 ) 

665 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

666 dtype=str, 

667 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

668 "correction.", 

669 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

670 ) 

671 brighterFatterMaskGrowSize = pexConfig.Field( 

672 dtype=int, 

673 default=0, 

674 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

675 "when brighter-fatter correction is applied." 

676 ) 

677 

678 # Dark subtraction. 

679 doDark = pexConfig.Field( 

680 dtype=bool, 

681 doc="Apply dark frame correction?", 

682 default=True, 

683 ) 

684 darkDataProductName = pexConfig.Field( 

685 dtype=str, 

686 doc="Name of the dark data product", 

687 default="dark", 

688 ) 

689 

690 # Camera-specific stray light removal. 

691 doStrayLight = pexConfig.Field( 

692 dtype=bool, 

693 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

694 default=False, 

695 ) 

696 strayLight = pexConfig.ConfigurableField( 

697 target=StrayLightTask, 

698 doc="y-band stray light correction" 

699 ) 

700 

701 # Flat correction. 

702 doFlat = pexConfig.Field( 

703 dtype=bool, 

704 doc="Apply flat field correction?", 

705 default=True, 

706 ) 

707 flatDataProductName = pexConfig.Field( 

708 dtype=str, 

709 doc="Name of the flat data product", 

710 default="flat", 

711 ) 

712 flatScalingType = pexConfig.ChoiceField( 

713 dtype=str, 

714 doc="The method for scaling the flat on the fly.", 

715 default='USER', 

716 allowed={ 

717 "USER": "Scale by flatUserScale", 

718 "MEAN": "Scale by the inverse of the mean", 

719 "MEDIAN": "Scale by the inverse of the median", 

720 }, 

721 ) 

722 flatUserScale = pexConfig.Field( 

723 dtype=float, 

724 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

725 default=1.0, 

726 ) 

727 doTweakFlat = pexConfig.Field( 

728 dtype=bool, 

729 doc="Tweak flats to match observed amplifier ratios?", 

730 default=False 

731 ) 

732 

733 # Amplifier normalization based on gains instead of using flats configuration. 

734 doApplyGains = pexConfig.Field( 

735 dtype=bool, 

736 doc="Correct the amplifiers for their gains instead of applying flat correction", 

737 default=False, 

738 ) 

739 usePtcGains = pexConfig.Field( 

740 dtype=bool, 

741 doc="Use the gain values from the Photon Transfer Curve?", 

742 default=False, 

743 ) 

744 normalizeGains = pexConfig.Field( 

745 dtype=bool, 

746 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

747 default=False, 

748 ) 

749 

750 # Fringe correction. 

751 doFringe = pexConfig.Field( 

752 dtype=bool, 

753 doc="Apply fringe correction?", 

754 default=True, 

755 ) 

756 fringe = pexConfig.ConfigurableField( 

757 target=FringeTask, 

758 doc="Fringe subtraction task", 

759 ) 

760 fringeAfterFlat = pexConfig.Field( 

761 dtype=bool, 

762 doc="Do fringe subtraction after flat-fielding?", 

763 default=True, 

764 ) 

765 

766 # Initial CCD-level background statistics options. 

767 doMeasureBackground = pexConfig.Field( 

768 dtype=bool, 

769 doc="Measure the background level on the reduced image?", 

770 default=False, 

771 ) 

772 

773 # Camera-specific masking configuration. 

774 doCameraSpecificMasking = pexConfig.Field( 

775 dtype=bool, 

776 doc="Mask camera-specific bad regions?", 

777 default=False, 

778 ) 

779 masking = pexConfig.ConfigurableField( 

780 target=MaskingTask, 

781 doc="Masking task." 

782 ) 

783 

784 # Interpolation options. 

785 

786 doInterpolate = pexConfig.Field( 

787 dtype=bool, 

788 doc="Interpolate masked pixels?", 

789 default=True, 

790 ) 

791 doSaturationInterpolation = pexConfig.Field( 

792 dtype=bool, 

793 doc="Perform interpolation over pixels masked as saturated?" 

794 " NB: This is independent of doSaturation; if that is False this plane" 

795 " will likely be blank, resulting in a no-op here.", 

796 default=True, 

797 ) 

798 doNanInterpolation = pexConfig.Field( 

799 dtype=bool, 

800 doc="Perform interpolation over pixels masked as NaN?" 

801 " NB: This is independent of doNanMasking; if that is False this plane" 

802 " will likely be blank, resulting in a no-op here.", 

803 default=True, 

804 ) 

805 doNanInterpAfterFlat = pexConfig.Field( 

806 dtype=bool, 

807 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

808 "also have to interpolate them before flat-fielding."), 

809 default=False, 

810 ) 

811 maskListToInterpolate = pexConfig.ListField( 

812 dtype=str, 

813 doc="List of mask planes that should be interpolated.", 

814 default=['SAT', 'BAD'], 

815 ) 

816 doSaveInterpPixels = pexConfig.Field( 

817 dtype=bool, 

818 doc="Save a copy of the pre-interpolated pixel values?", 

819 default=False, 

820 ) 

821 

822 # Default photometric calibration options. 

823 fluxMag0T1 = pexConfig.DictField( 

824 keytype=str, 

825 itemtype=float, 

826 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

827 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

828 )) 

829 ) 

830 defaultFluxMag0T1 = pexConfig.Field( 

831 dtype=float, 

832 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

833 default=pow(10.0, 0.4*28.0) 

834 ) 

835 

836 # Vignette correction configuration. 

837 doVignette = pexConfig.Field( 

838 dtype=bool, 

839 doc="Apply vignetting parameters?", 

840 default=False, 

841 ) 

842 vignette = pexConfig.ConfigurableField( 

843 target=VignetteTask, 

844 doc="Vignetting task.", 

845 ) 

846 

847 # Transmission curve configuration. 

848 doAttachTransmissionCurve = pexConfig.Field( 

849 dtype=bool, 

850 default=False, 

851 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

852 ) 

853 doUseOpticsTransmission = pexConfig.Field( 

854 dtype=bool, 

855 default=True, 

856 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

857 ) 

858 doUseFilterTransmission = pexConfig.Field( 

859 dtype=bool, 

860 default=True, 

861 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

862 ) 

863 doUseSensorTransmission = pexConfig.Field( 

864 dtype=bool, 

865 default=True, 

866 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

867 ) 

868 doUseAtmosphereTransmission = pexConfig.Field( 

869 dtype=bool, 

870 default=True, 

871 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

872 ) 

873 

874 # Illumination correction. 

875 doIlluminationCorrection = pexConfig.Field( 

876 dtype=bool, 

877 default=False, 

878 doc="Perform illumination correction?" 

879 ) 

880 illuminationCorrectionDataProductName = pexConfig.Field( 

881 dtype=str, 

882 doc="Name of the illumination correction data product.", 

883 default="illumcor", 

884 ) 

885 illumScale = pexConfig.Field( 

886 dtype=float, 

887 doc="Scale factor for the illumination correction.", 

888 default=1.0, 

889 ) 

890 illumFilters = pexConfig.ListField( 

891 dtype=str, 

892 default=[], 

893 doc="Only perform illumination correction for these filters." 

894 ) 

895 

896 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

897 doWrite = pexConfig.Field( 

898 dtype=bool, 

899 doc="Persist postISRCCD?", 

900 default=True, 

901 ) 

902 

903 def validate(self): 

904 super().validate() 

905 if self.doFlat and self.doApplyGains: 

906 raise ValueError("You may not specify both doFlat and doApplyGains") 

907 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

908 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

909 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

910 self.maskListToInterpolate.append(self.saturatedMaskName) 

911 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

912 self.maskListToInterpolate.remove(self.saturatedMaskName) 

913 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

914 self.maskListToInterpolate.append("UNMASKEDNAN") 

915 

916 

917class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

918 """Apply common instrument signature correction algorithms to a raw frame. 

919 

920 The process for correcting imaging data is very similar from 

921 camera to camera. This task provides a vanilla implementation of 

922 doing these corrections, including the ability to turn certain 

923 corrections off if they are not needed. The inputs to the primary 

924 method, `run()`, are a raw exposure to be corrected and the 

925 calibration data products. The raw input is a single chip sized 

926 mosaic of all amps including overscans and other non-science 

927 pixels. The method `runDataRef()` identifies and defines the 

928 calibration data products, and is intended for use by a 

929 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

930 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

931 subclassed for different camera, although the most camera specific 

932 methods have been split into subtasks that can be redirected 

933 appropriately. 

934 

935 The __init__ method sets up the subtasks for ISR processing, using 

936 the defaults from `lsst.ip.isr`. 

937 

938 Parameters 

939 ---------- 

940 args : `list` 

941 Positional arguments passed to the Task constructor. None used at this time. 

942 kwargs : `dict`, optional 

943 Keyword arguments passed on to the Task constructor. None used at this time. 

944 """ 

945 ConfigClass = IsrTaskConfig 

946 _DefaultName = "isr" 

947 

948 def __init__(self, **kwargs): 

949 super().__init__(**kwargs) 

950 self.makeSubtask("assembleCcd") 

951 self.makeSubtask("crosstalk") 

952 self.makeSubtask("strayLight") 

953 self.makeSubtask("fringe") 

954 self.makeSubtask("masking") 

955 self.makeSubtask("overscan") 

956 self.makeSubtask("vignette") 

957 

958 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

959 inputs = butlerQC.get(inputRefs) 

960 

961 try: 

962 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

963 except Exception as e: 

964 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

965 (inputRefs, e)) 

966 

967 inputs['isGen3'] = True 

968 

969 detector = inputs['ccdExposure'].getDetector() 

970 

971 if self.config.doCrosstalk is True: 

972 # Crosstalk sources need to be defined by the pipeline 

973 # yaml if they exist. 

974 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

975 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

976 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

977 else: 

978 coeffVector = (self.config.crosstalk.crosstalkValues 

979 if self.config.crosstalk.useConfigCoefficients else None) 

980 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

981 inputs['crosstalk'] = crosstalkCalib 

982 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

983 if 'crosstalkSources' not in inputs: 

984 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

985 

986 if self.doLinearize(detector) is True: 

987 if 'linearizer' in inputs: 

988 if isinstance(inputs['linearizer'], dict): 

989 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

990 linearizer.fromYaml(inputs['linearizer']) 

991 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

992 elif isinstance(inputs['linearizer'], numpy.ndarray): 

993 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

994 detector=detector, 

995 log=self.log) 

996 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

997 else: 

998 linearizer = inputs['linearizer'] 

999 linearizer.log = self.log 

1000 inputs['linearizer'] = linearizer 

1001 else: 

1002 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1003 self.log.warning("Constructing linearizer from cameraGeom information.") 

1004 

1005 if self.config.doDefect is True: 

1006 if "defects" in inputs and inputs['defects'] is not None: 

1007 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

1008 # masking expects a list of defects defined by their bounding box 

1009 if not isinstance(inputs["defects"], Defects): 

1010 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1011 

1012 # Load the correct style of brighter-fatter kernel, and repack 

1013 # the information as a numpy array. 

1014 if self.config.doBrighterFatter: 

1015 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1016 if brighterFatterKernel is None: 

1017 brighterFatterKernel = inputs.get('bfKernel', None) 

1018 

1019 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1020 # This is a ISR calib kernel 

1021 detName = detector.getName() 

1022 level = brighterFatterKernel.level 

1023 

1024 # This is expected to be a dictionary of amp-wise gains. 

1025 inputs['bfGains'] = brighterFatterKernel.gain 

1026 if self.config.brighterFatterLevel == 'DETECTOR': 

1027 if level == 'DETECTOR': 

1028 if detName in brighterFatterKernel.detKernels: 

1029 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1030 else: 

1031 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1032 elif level == 'AMP': 

1033 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1034 "fatter kernels.") 

1035 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1036 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1037 elif self.config.brighterFatterLevel == 'AMP': 

1038 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1039 

1040 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1041 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

1042 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1043 expId=expId, 

1044 assembler=self.assembleCcd 

1045 if self.config.doAssembleIsrExposures else None) 

1046 else: 

1047 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1048 

1049 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1050 if 'strayLightData' not in inputs: 

1051 inputs['strayLightData'] = None 

1052 

1053 outputs = self.run(**inputs) 

1054 butlerQC.put(outputs, outputRefs) 

1055 

1056 def readIsrData(self, dataRef, rawExposure): 

1057 """Retrieve necessary frames for instrument signature removal. 

1058 

1059 Pre-fetching all required ISR data products limits the IO 

1060 required by the ISR. Any conflict between the calibration data 

1061 available and that needed for ISR is also detected prior to 

1062 doing processing, allowing it to fail quickly. 

1063 

1064 Parameters 

1065 ---------- 

1066 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1067 Butler reference of the detector data to be processed 

1068 rawExposure : `afw.image.Exposure` 

1069 The raw exposure that will later be corrected with the 

1070 retrieved calibration data; should not be modified in this 

1071 method. 

1072 

1073 Returns 

1074 ------- 

1075 result : `lsst.pipe.base.Struct` 

1076 Result struct with components (which may be `None`): 

1077 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1078 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1079 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1080 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1081 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1082 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1083 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1084 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1085 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1086 - ``seed``: random seed derived from the ccdExposureId for random 

1087 number generator (`uint32`). 

1088 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1089 A ``TransmissionCurve`` that represents the throughput of the optics, 

1090 to be evaluated in focal-plane coordinates. 

1091 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1092 A ``TransmissionCurve`` that represents the throughput of the filter 

1093 itself, to be evaluated in focal-plane coordinates. 

1094 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1095 A ``TransmissionCurve`` that represents the throughput of the sensor 

1096 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1097 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1098 A ``TransmissionCurve`` that represents the throughput of the 

1099 atmosphere, assumed to be spatially constant. 

1100 - ``strayLightData`` : `object` 

1101 An opaque object containing calibration information for 

1102 stray-light correction. If `None`, no correction will be 

1103 performed. 

1104 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1105 

1106 Raises 

1107 ------ 

1108 NotImplementedError : 

1109 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1110 """ 

1111 try: 

1112 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1113 dateObs = dateObs.toPython().isoformat() 

1114 except RuntimeError: 

1115 self.log.warning("Unable to identify dateObs for rawExposure.") 

1116 dateObs = None 

1117 

1118 ccd = rawExposure.getDetector() 

1119 filterLabel = rawExposure.getFilterLabel() 

1120 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1121 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1122 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1123 if self.config.doBias else None) 

1124 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1125 linearizer = (dataRef.get("linearizer", immediate=True) 

1126 if self.doLinearize(ccd) else None) 

1127 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1128 linearizer.log = self.log 

1129 if isinstance(linearizer, numpy.ndarray): 

1130 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1131 

1132 crosstalkCalib = None 

1133 if self.config.doCrosstalk: 

1134 try: 

1135 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1136 except NoResults: 

1137 coeffVector = (self.config.crosstalk.crosstalkValues 

1138 if self.config.crosstalk.useConfigCoefficients else None) 

1139 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1140 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1141 if self.config.doCrosstalk else None) 

1142 

1143 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1144 if self.config.doDark else None) 

1145 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1146 dateObs=dateObs) 

1147 if self.config.doFlat else None) 

1148 

1149 brighterFatterKernel = None 

1150 brighterFatterGains = None 

1151 if self.config.doBrighterFatter is True: 

1152 try: 

1153 # Use the new-style cp_pipe version of the kernel if it exists 

1154 # If using a new-style kernel, always use the self-consistent 

1155 # gains, i.e. the ones inside the kernel object itself 

1156 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1157 brighterFatterGains = brighterFatterKernel.gain 

1158 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded") 

1159 except NoResults: 

1160 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1161 brighterFatterKernel = dataRef.get("bfKernel") 

1162 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded") 

1163 except NoResults: 

1164 brighterFatterKernel = None 

1165 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1166 # If the kernel is not an ndarray, it's the cp_pipe version 

1167 # so extract the kernel for this detector, or raise an error 

1168 if self.config.brighterFatterLevel == 'DETECTOR': 

1169 if brighterFatterKernel.detKernels: 

1170 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()] 

1171 else: 

1172 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1173 else: 

1174 # TODO DM-15631 for implementing this 

1175 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1176 

1177 defectList = (dataRef.get("defects") 

1178 if self.config.doDefect else None) 

1179 expId = rawExposure.getInfo().getVisitInfo().getExposureId() 

1180 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd 

1181 if self.config.doAssembleIsrExposures else None) 

1182 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1183 else pipeBase.Struct(fringes=None)) 

1184 

1185 if self.config.doAttachTransmissionCurve: 

1186 opticsTransmission = (dataRef.get("transmission_optics") 

1187 if self.config.doUseOpticsTransmission else None) 

1188 filterTransmission = (dataRef.get("transmission_filter") 

1189 if self.config.doUseFilterTransmission else None) 

1190 sensorTransmission = (dataRef.get("transmission_sensor") 

1191 if self.config.doUseSensorTransmission else None) 

1192 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1193 if self.config.doUseAtmosphereTransmission else None) 

1194 else: 

1195 opticsTransmission = None 

1196 filterTransmission = None 

1197 sensorTransmission = None 

1198 atmosphereTransmission = None 

1199 

1200 if self.config.doStrayLight: 

1201 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1202 else: 

1203 strayLightData = None 

1204 

1205 illumMaskedImage = (self.getIsrExposure(dataRef, 

1206 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1207 if (self.config.doIlluminationCorrection 

1208 and physicalFilter in self.config.illumFilters) 

1209 else None) 

1210 

1211 # Struct should include only kwargs to run() 

1212 return pipeBase.Struct(bias=biasExposure, 

1213 linearizer=linearizer, 

1214 crosstalk=crosstalkCalib, 

1215 crosstalkSources=crosstalkSources, 

1216 dark=darkExposure, 

1217 flat=flatExposure, 

1218 bfKernel=brighterFatterKernel, 

1219 bfGains=brighterFatterGains, 

1220 defects=defectList, 

1221 fringes=fringeStruct, 

1222 opticsTransmission=opticsTransmission, 

1223 filterTransmission=filterTransmission, 

1224 sensorTransmission=sensorTransmission, 

1225 atmosphereTransmission=atmosphereTransmission, 

1226 strayLightData=strayLightData, 

1227 illumMaskedImage=illumMaskedImage 

1228 ) 

1229 

1230 @pipeBase.timeMethod 

1231 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1232 crosstalk=None, crosstalkSources=None, 

1233 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1234 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1235 sensorTransmission=None, atmosphereTransmission=None, 

1236 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1237 isGen3=False, 

1238 ): 

1239 """Perform instrument signature removal on an exposure. 

1240 

1241 Steps included in the ISR processing, in order performed, are: 

1242 - saturation and suspect pixel masking 

1243 - overscan subtraction 

1244 - CCD assembly of individual amplifiers 

1245 - bias subtraction 

1246 - variance image construction 

1247 - linearization of non-linear response 

1248 - crosstalk masking 

1249 - brighter-fatter correction 

1250 - dark subtraction 

1251 - fringe correction 

1252 - stray light subtraction 

1253 - flat correction 

1254 - masking of known defects and camera specific features 

1255 - vignette calculation 

1256 - appending transmission curve and distortion model 

1257 

1258 Parameters 

1259 ---------- 

1260 ccdExposure : `lsst.afw.image.Exposure` 

1261 The raw exposure that is to be run through ISR. The 

1262 exposure is modified by this method. 

1263 camera : `lsst.afw.cameraGeom.Camera`, optional 

1264 The camera geometry for this exposure. Required if ``isGen3`` is 

1265 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1266 ``flat`` does not have an associated detector. 

1267 bias : `lsst.afw.image.Exposure`, optional 

1268 Bias calibration frame. 

1269 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1270 Functor for linearization. 

1271 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1272 Calibration for crosstalk. 

1273 crosstalkSources : `list`, optional 

1274 List of possible crosstalk sources. 

1275 dark : `lsst.afw.image.Exposure`, optional 

1276 Dark calibration frame. 

1277 flat : `lsst.afw.image.Exposure`, optional 

1278 Flat calibration frame. 

1279 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1280 Photon transfer curve dataset, with, e.g., gains 

1281 and read noise. 

1282 bfKernel : `numpy.ndarray`, optional 

1283 Brighter-fatter kernel. 

1284 bfGains : `dict` of `float`, optional 

1285 Gains used to override the detector's nominal gains for the 

1286 brighter-fatter correction. A dict keyed by amplifier name for 

1287 the detector in question. 

1288 defects : `lsst.ip.isr.Defects`, optional 

1289 List of defects. 

1290 fringes : `lsst.pipe.base.Struct`, optional 

1291 Struct containing the fringe correction data, with 

1292 elements: 

1293 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1294 - ``seed``: random seed derived from the ccdExposureId for random 

1295 number generator (`uint32`) 

1296 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1297 A ``TransmissionCurve`` that represents the throughput of the optics, 

1298 to be evaluated in focal-plane coordinates. 

1299 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1300 A ``TransmissionCurve`` that represents the throughput of the filter 

1301 itself, to be evaluated in focal-plane coordinates. 

1302 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1303 A ``TransmissionCurve`` that represents the throughput of the sensor 

1304 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1305 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1306 A ``TransmissionCurve`` that represents the throughput of the 

1307 atmosphere, assumed to be spatially constant. 

1308 detectorNum : `int`, optional 

1309 The integer number for the detector to process. 

1310 isGen3 : bool, optional 

1311 Flag this call to run() as using the Gen3 butler environment. 

1312 strayLightData : `object`, optional 

1313 Opaque object containing calibration information for stray-light 

1314 correction. If `None`, no correction will be performed. 

1315 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1316 Illumination correction image. 

1317 

1318 Returns 

1319 ------- 

1320 result : `lsst.pipe.base.Struct` 

1321 Result struct with component: 

1322 - ``exposure`` : `afw.image.Exposure` 

1323 The fully ISR corrected exposure. 

1324 - ``outputExposure`` : `afw.image.Exposure` 

1325 An alias for `exposure` 

1326 - ``ossThumb`` : `numpy.ndarray` 

1327 Thumbnail image of the exposure after overscan subtraction. 

1328 - ``flattenedThumb`` : `numpy.ndarray` 

1329 Thumbnail image of the exposure after flat-field correction. 

1330 

1331 Raises 

1332 ------ 

1333 RuntimeError 

1334 Raised if a configuration option is set to True, but the 

1335 required calibration data has not been specified. 

1336 

1337 Notes 

1338 ----- 

1339 The current processed exposure can be viewed by setting the 

1340 appropriate lsstDebug entries in the `debug.display` 

1341 dictionary. The names of these entries correspond to some of 

1342 the IsrTaskConfig Boolean options, with the value denoting the 

1343 frame to use. The exposure is shown inside the matching 

1344 option check and after the processing of that step has 

1345 finished. The steps with debug points are: 

1346 

1347 doAssembleCcd 

1348 doBias 

1349 doCrosstalk 

1350 doBrighterFatter 

1351 doDark 

1352 doFringe 

1353 doStrayLight 

1354 doFlat 

1355 

1356 In addition, setting the "postISRCCD" entry displays the 

1357 exposure after all ISR processing has finished. 

1358 

1359 """ 

1360 

1361 if isGen3 is True: 

1362 # Gen3 currently cannot automatically do configuration overrides. 

1363 # DM-15257 looks to discuss this issue. 

1364 # Configure input exposures; 

1365 if detectorNum is None: 

1366 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1367 

1368 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1369 bias = self.ensureExposure(bias, camera, detectorNum) 

1370 dark = self.ensureExposure(dark, camera, detectorNum) 

1371 flat = self.ensureExposure(flat, camera, detectorNum) 

1372 else: 

1373 if isinstance(ccdExposure, ButlerDataRef): 

1374 return self.runDataRef(ccdExposure) 

1375 

1376 ccd = ccdExposure.getDetector() 

1377 filterLabel = ccdExposure.getFilterLabel() 

1378 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1379 

1380 if not ccd: 

1381 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1382 ccd = [FakeAmp(ccdExposure, self.config)] 

1383 

1384 # Validate Input 

1385 if self.config.doBias and bias is None: 

1386 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1387 if self.doLinearize(ccd) and linearizer is None: 

1388 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1389 if self.config.doBrighterFatter and bfKernel is None: 

1390 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1391 if self.config.doDark and dark is None: 

1392 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1393 if self.config.doFlat and flat is None: 

1394 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1395 if self.config.doDefect and defects is None: 

1396 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1397 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1398 and fringes.fringes is None): 

1399 # The `fringes` object needs to be a pipeBase.Struct, as 

1400 # we use it as a `dict` for the parameters of 

1401 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1402 # not be `None` if `doFringe=True`. Otherwise, raise. 

1403 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1404 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1405 and illumMaskedImage is None): 

1406 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1407 

1408 # Begin ISR processing. 

1409 if self.config.doConvertIntToFloat: 

1410 self.log.info("Converting exposure to floating point values.") 

1411 ccdExposure = self.convertIntToFloat(ccdExposure) 

1412 

1413 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1414 self.log.info("Applying bias correction.") 

1415 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1416 trimToFit=self.config.doTrimToMatchCalib) 

1417 self.debugView(ccdExposure, "doBias") 

1418 

1419 # Amplifier level processing. 

1420 overscans = [] 

1421 for amp in ccd: 

1422 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1423 if ccdExposure.getBBox().contains(amp.getBBox()): 

1424 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1425 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1426 

1427 if self.config.doOverscan and not badAmp: 

1428 # Overscan correction on amp-by-amp basis. 

1429 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1430 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1431 if overscanResults is not None and \ 

1432 self.config.qa is not None and self.config.qa.saveStats is True: 

1433 if isinstance(overscanResults.overscanFit, float): 

1434 qaMedian = overscanResults.overscanFit 

1435 qaStdev = float("NaN") 

1436 else: 

1437 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1438 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1439 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1440 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1441 

1442 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian) 

1443 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev) 

1444 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1445 amp.getName(), qaMedian, qaStdev) 

1446 

1447 # Residuals after overscan correction 

1448 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1449 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1450 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1451 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1452 

1453 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter) 

1454 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter) 

1455 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1456 amp.getName(), qaMedianAfter, qaStdevAfter) 

1457 

1458 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1459 else: 

1460 if badAmp: 

1461 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1462 overscanResults = None 

1463 

1464 overscans.append(overscanResults if overscanResults is not None else None) 

1465 else: 

1466 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1467 

1468 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1469 self.log.info("Applying crosstalk correction.") 

1470 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1471 crosstalkSources=crosstalkSources, camera=camera) 

1472 self.debugView(ccdExposure, "doCrosstalk") 

1473 

1474 if self.config.doAssembleCcd: 

1475 self.log.info("Assembling CCD from amplifiers.") 

1476 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1477 

1478 if self.config.expectWcs and not ccdExposure.getWcs(): 

1479 self.log.warning("No WCS found in input exposure.") 

1480 self.debugView(ccdExposure, "doAssembleCcd") 

1481 

1482 ossThumb = None 

1483 if self.config.qa.doThumbnailOss: 

1484 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1485 

1486 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1487 self.log.info("Applying bias correction.") 

1488 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1489 trimToFit=self.config.doTrimToMatchCalib) 

1490 self.debugView(ccdExposure, "doBias") 

1491 

1492 if self.config.doVariance: 

1493 for amp, overscanResults in zip(ccd, overscans): 

1494 if ccdExposure.getBBox().contains(amp.getBBox()): 

1495 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1496 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1497 if overscanResults is not None: 

1498 self.updateVariance(ampExposure, amp, 

1499 overscanImage=overscanResults.overscanImage, 

1500 ptcDataset=ptc) 

1501 else: 

1502 self.updateVariance(ampExposure, amp, 

1503 overscanImage=None, 

1504 ptcDataset=ptc) 

1505 if self.config.qa is not None and self.config.qa.saveStats is True: 

1506 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1507 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1508 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1509 qaStats.getValue(afwMath.MEDIAN)) 

1510 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1511 qaStats.getValue(afwMath.STDEVCLIP)) 

1512 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1513 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1514 qaStats.getValue(afwMath.STDEVCLIP)) 

1515 

1516 if self.doLinearize(ccd): 

1517 self.log.info("Applying linearizer.") 

1518 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1519 detector=ccd, log=self.log) 

1520 

1521 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1522 self.log.info("Applying crosstalk correction.") 

1523 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1524 crosstalkSources=crosstalkSources, isTrimmed=True) 

1525 self.debugView(ccdExposure, "doCrosstalk") 

1526 

1527 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do 

1528 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1529 if self.config.doDefect: 

1530 self.log.info("Masking defects.") 

1531 self.maskDefect(ccdExposure, defects) 

1532 

1533 if self.config.numEdgeSuspect > 0: 

1534 self.log.info("Masking edges as SUSPECT.") 

1535 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1536 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1537 

1538 if self.config.doNanMasking: 

1539 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1540 self.maskNan(ccdExposure) 

1541 

1542 if self.config.doWidenSaturationTrails: 

1543 self.log.info("Widening saturation trails.") 

1544 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1545 

1546 if self.config.doCameraSpecificMasking: 

1547 self.log.info("Masking regions for camera specific reasons.") 

1548 self.masking.run(ccdExposure) 

1549 

1550 if self.config.doBrighterFatter: 

1551 # We need to apply flats and darks before we can interpolate, and we 

1552 # need to interpolate before we do B-F, but we do B-F without the 

1553 # flats and darks applied so we can work in units of electrons or holes. 

1554 # This context manager applies and then removes the darks and flats. 

1555 # 

1556 # We also do not want to interpolate values here, so operate on temporary 

1557 # images so we can apply only the BF-correction and roll back the 

1558 # interpolation. 

1559 interpExp = ccdExposure.clone() 

1560 with self.flatContext(interpExp, flat, dark): 

1561 isrFunctions.interpolateFromMask( 

1562 maskedImage=interpExp.getMaskedImage(), 

1563 fwhm=self.config.fwhm, 

1564 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1565 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1566 ) 

1567 bfExp = interpExp.clone() 

1568 

1569 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1570 type(bfKernel), type(bfGains)) 

1571 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1572 self.config.brighterFatterMaxIter, 

1573 self.config.brighterFatterThreshold, 

1574 self.config.brighterFatterApplyGain, 

1575 bfGains) 

1576 if bfResults[1] == self.config.brighterFatterMaxIter: 

1577 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1578 bfResults[0]) 

1579 else: 

1580 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1581 bfResults[1]) 

1582 image = ccdExposure.getMaskedImage().getImage() 

1583 bfCorr = bfExp.getMaskedImage().getImage() 

1584 bfCorr -= interpExp.getMaskedImage().getImage() 

1585 image += bfCorr 

1586 

1587 # Applying the brighter-fatter correction applies a 

1588 # convolution to the science image. At the edges this 

1589 # convolution may not have sufficient valid pixels to 

1590 # produce a valid correction. Mark pixels within the size 

1591 # of the brighter-fatter kernel as EDGE to warn of this 

1592 # fact. 

1593 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1594 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1595 maskPlane="EDGE") 

1596 

1597 if self.config.brighterFatterMaskGrowSize > 0: 

1598 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1599 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1600 isrFunctions.growMasks(ccdExposure.getMask(), 

1601 radius=self.config.brighterFatterMaskGrowSize, 

1602 maskNameList=maskPlane, 

1603 maskValue=maskPlane) 

1604 

1605 self.debugView(ccdExposure, "doBrighterFatter") 

1606 

1607 if self.config.doDark: 

1608 self.log.info("Applying dark correction.") 

1609 self.darkCorrection(ccdExposure, dark) 

1610 self.debugView(ccdExposure, "doDark") 

1611 

1612 if self.config.doFringe and not self.config.fringeAfterFlat: 

1613 self.log.info("Applying fringe correction before flat.") 

1614 self.fringe.run(ccdExposure, **fringes.getDict()) 

1615 self.debugView(ccdExposure, "doFringe") 

1616 

1617 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1618 self.log.info("Checking strayLight correction.") 

1619 self.strayLight.run(ccdExposure, strayLightData) 

1620 self.debugView(ccdExposure, "doStrayLight") 

1621 

1622 if self.config.doFlat: 

1623 self.log.info("Applying flat correction.") 

1624 self.flatCorrection(ccdExposure, flat) 

1625 self.debugView(ccdExposure, "doFlat") 

1626 

1627 if self.config.doApplyGains: 

1628 self.log.info("Applying gain correction instead of flat.") 

1629 if self.config.usePtcGains: 

1630 self.log.info("Using gains from the Photon Transfer Curve.") 

1631 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1632 ptcGains=ptc.gain) 

1633 else: 

1634 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1635 

1636 if self.config.doFringe and self.config.fringeAfterFlat: 

1637 self.log.info("Applying fringe correction after flat.") 

1638 self.fringe.run(ccdExposure, **fringes.getDict()) 

1639 

1640 if self.config.doVignette: 

1641 self.log.info("Constructing Vignette polygon.") 

1642 self.vignettePolygon = self.vignette.run(ccdExposure) 

1643 

1644 if self.config.vignette.doWriteVignettePolygon: 

1645 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1646 

1647 if self.config.doAttachTransmissionCurve: 

1648 self.log.info("Adding transmission curves.") 

1649 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1650 filterTransmission=filterTransmission, 

1651 sensorTransmission=sensorTransmission, 

1652 atmosphereTransmission=atmosphereTransmission) 

1653 

1654 flattenedThumb = None 

1655 if self.config.qa.doThumbnailFlattened: 

1656 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1657 

1658 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1659 self.log.info("Performing illumination correction.") 

1660 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1661 illumMaskedImage, illumScale=self.config.illumScale, 

1662 trimToFit=self.config.doTrimToMatchCalib) 

1663 

1664 preInterpExp = None 

1665 if self.config.doSaveInterpPixels: 

1666 preInterpExp = ccdExposure.clone() 

1667 

1668 # Reset and interpolate bad pixels. 

1669 # 

1670 # Large contiguous bad regions (which should have the BAD mask 

1671 # bit set) should have their values set to the image median. 

1672 # This group should include defects and bad amplifiers. As the 

1673 # area covered by these defects are large, there's little 

1674 # reason to expect that interpolation would provide a more 

1675 # useful value. 

1676 # 

1677 # Smaller defects can be safely interpolated after the larger 

1678 # regions have had their pixel values reset. This ensures 

1679 # that the remaining defects adjacent to bad amplifiers (as an 

1680 # example) do not attempt to interpolate extreme values. 

1681 if self.config.doSetBadRegions: 

1682 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1683 if badPixelCount > 0: 

1684 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1685 

1686 if self.config.doInterpolate: 

1687 self.log.info("Interpolating masked pixels.") 

1688 isrFunctions.interpolateFromMask( 

1689 maskedImage=ccdExposure.getMaskedImage(), 

1690 fwhm=self.config.fwhm, 

1691 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1692 maskNameList=list(self.config.maskListToInterpolate) 

1693 ) 

1694 

1695 self.roughZeroPoint(ccdExposure) 

1696 

1697 if self.config.doMeasureBackground: 

1698 self.log.info("Measuring background level.") 

1699 self.measureBackground(ccdExposure, self.config.qa) 

1700 

1701 if self.config.qa is not None and self.config.qa.saveStats is True: 

1702 for amp in ccd: 

1703 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1704 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1705 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1706 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1707 qaStats.getValue(afwMath.MEDIAN)) 

1708 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1709 qaStats.getValue(afwMath.STDEVCLIP)) 

1710 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1711 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1712 qaStats.getValue(afwMath.STDEVCLIP)) 

1713 

1714 self.debugView(ccdExposure, "postISRCCD") 

1715 

1716 return pipeBase.Struct( 

1717 exposure=ccdExposure, 

1718 ossThumb=ossThumb, 

1719 flattenedThumb=flattenedThumb, 

1720 

1721 preInterpExposure=preInterpExp, 

1722 outputExposure=ccdExposure, 

1723 outputOssThumbnail=ossThumb, 

1724 outputFlattenedThumbnail=flattenedThumb, 

1725 ) 

1726 

1727 @pipeBase.timeMethod 

1728 def runDataRef(self, sensorRef): 

1729 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1730 

1731 This method contains the `CmdLineTask` interface to the ISR 

1732 processing. All IO is handled here, freeing the `run()` method 

1733 to manage only pixel-level calculations. The steps performed 

1734 are: 

1735 - Read in necessary detrending/isr/calibration data. 

1736 - Process raw exposure in `run()`. 

1737 - Persist the ISR-corrected exposure as "postISRCCD" if 

1738 config.doWrite=True. 

1739 

1740 Parameters 

1741 ---------- 

1742 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1743 DataRef of the detector data to be processed 

1744 

1745 Returns 

1746 ------- 

1747 result : `lsst.pipe.base.Struct` 

1748 Result struct with component: 

1749 - ``exposure`` : `afw.image.Exposure` 

1750 The fully ISR corrected exposure. 

1751 

1752 Raises 

1753 ------ 

1754 RuntimeError 

1755 Raised if a configuration option is set to True, but the 

1756 required calibration data does not exist. 

1757 

1758 """ 

1759 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1760 

1761 ccdExposure = sensorRef.get(self.config.datasetType) 

1762 

1763 camera = sensorRef.get("camera") 

1764 isrData = self.readIsrData(sensorRef, ccdExposure) 

1765 

1766 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1767 

1768 if self.config.doWrite: 

1769 sensorRef.put(result.exposure, "postISRCCD") 

1770 if result.preInterpExposure is not None: 

1771 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated") 

1772 if result.ossThumb is not None: 

1773 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1774 if result.flattenedThumb is not None: 

1775 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1776 

1777 return result 

1778 

1779 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1780 """Retrieve a calibration dataset for removing instrument signature. 

1781 

1782 Parameters 

1783 ---------- 

1784 

1785 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1786 DataRef of the detector data to find calibration datasets 

1787 for. 

1788 datasetType : `str` 

1789 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1790 dateObs : `str`, optional 

1791 Date of the observation. Used to correct butler failures 

1792 when using fallback filters. 

1793 immediate : `Bool` 

1794 If True, disable butler proxies to enable error handling 

1795 within this routine. 

1796 

1797 Returns 

1798 ------- 

1799 exposure : `lsst.afw.image.Exposure` 

1800 Requested calibration frame. 

1801 

1802 Raises 

1803 ------ 

1804 RuntimeError 

1805 Raised if no matching calibration frame can be found. 

1806 """ 

1807 try: 

1808 exp = dataRef.get(datasetType, immediate=immediate) 

1809 except Exception as exc1: 

1810 if not self.config.fallbackFilterName: 

1811 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1812 try: 

1813 if self.config.useFallbackDate and dateObs: 

1814 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1815 dateObs=dateObs, immediate=immediate) 

1816 else: 

1817 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1818 except Exception as exc2: 

1819 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1820 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1821 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1822 

1823 if self.config.doAssembleIsrExposures: 

1824 exp = self.assembleCcd.assembleCcd(exp) 

1825 return exp 

1826 

1827 def ensureExposure(self, inputExp, camera, detectorNum): 

1828 """Ensure that the data returned by Butler is a fully constructed exposure. 

1829 

1830 ISR requires exposure-level image data for historical reasons, so if we did 

1831 not recieve that from Butler, construct it from what we have, modifying the 

1832 input in place. 

1833 

1834 Parameters 

1835 ---------- 

1836 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1837 `lsst.afw.image.ImageF` 

1838 The input data structure obtained from Butler. 

1839 camera : `lsst.afw.cameraGeom.camera` 

1840 The camera associated with the image. Used to find the appropriate 

1841 detector. 

1842 detectorNum : `int` 

1843 The detector this exposure should match. 

1844 

1845 Returns 

1846 ------- 

1847 inputExp : `lsst.afw.image.Exposure` 

1848 The re-constructed exposure, with appropriate detector parameters. 

1849 

1850 Raises 

1851 ------ 

1852 TypeError 

1853 Raised if the input data cannot be used to construct an exposure. 

1854 """ 

1855 if isinstance(inputExp, afwImage.DecoratedImageU): 

1856 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1857 elif isinstance(inputExp, afwImage.ImageF): 

1858 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1859 elif isinstance(inputExp, afwImage.MaskedImageF): 

1860 inputExp = afwImage.makeExposure(inputExp) 

1861 elif isinstance(inputExp, afwImage.Exposure): 

1862 pass 

1863 elif inputExp is None: 

1864 # Assume this will be caught by the setup if it is a problem. 

1865 return inputExp 

1866 else: 

1867 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1868 (type(inputExp), )) 

1869 

1870 if inputExp.getDetector() is None: 

1871 inputExp.setDetector(camera[detectorNum]) 

1872 

1873 return inputExp 

1874 

1875 def convertIntToFloat(self, exposure): 

1876 """Convert exposure image from uint16 to float. 

1877 

1878 If the exposure does not need to be converted, the input is 

1879 immediately returned. For exposures that are converted to use 

1880 floating point pixels, the variance is set to unity and the 

1881 mask to zero. 

1882 

1883 Parameters 

1884 ---------- 

1885 exposure : `lsst.afw.image.Exposure` 

1886 The raw exposure to be converted. 

1887 

1888 Returns 

1889 ------- 

1890 newexposure : `lsst.afw.image.Exposure` 

1891 The input ``exposure``, converted to floating point pixels. 

1892 

1893 Raises 

1894 ------ 

1895 RuntimeError 

1896 Raised if the exposure type cannot be converted to float. 

1897 

1898 """ 

1899 if isinstance(exposure, afwImage.ExposureF): 

1900 # Nothing to be done 

1901 self.log.debug("Exposure already of type float.") 

1902 return exposure 

1903 if not hasattr(exposure, "convertF"): 

1904 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1905 

1906 newexposure = exposure.convertF() 

1907 newexposure.variance[:] = 1 

1908 newexposure.mask[:] = 0x0 

1909 

1910 return newexposure 

1911 

1912 def maskAmplifier(self, ccdExposure, amp, defects): 

1913 """Identify bad amplifiers, saturated and suspect pixels. 

1914 

1915 Parameters 

1916 ---------- 

1917 ccdExposure : `lsst.afw.image.Exposure` 

1918 Input exposure to be masked. 

1919 amp : `lsst.afw.table.AmpInfoCatalog` 

1920 Catalog of parameters defining the amplifier on this 

1921 exposure to mask. 

1922 defects : `lsst.ip.isr.Defects` 

1923 List of defects. Used to determine if the entire 

1924 amplifier is bad. 

1925 

1926 Returns 

1927 ------- 

1928 badAmp : `Bool` 

1929 If this is true, the entire amplifier area is covered by 

1930 defects and unusable. 

1931 

1932 """ 

1933 maskedImage = ccdExposure.getMaskedImage() 

1934 

1935 badAmp = False 

1936 

1937 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1938 # comparison with current defects definition. 

1939 if defects is not None: 

1940 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1941 

1942 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1943 # association with pixels in current ccdExposure). 

1944 if badAmp: 

1945 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1946 afwImage.PARENT) 

1947 maskView = dataView.getMask() 

1948 maskView |= maskView.getPlaneBitMask("BAD") 

1949 del maskView 

1950 return badAmp 

1951 

1952 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1953 # Saturation and suspect pixels can be masked now, though. 

1954 limits = dict() 

1955 if self.config.doSaturation and not badAmp: 

1956 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1957 if self.config.doSuspect and not badAmp: 

1958 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1959 if math.isfinite(self.config.saturation): 

1960 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1961 

1962 for maskName, maskThreshold in limits.items(): 

1963 if not math.isnan(maskThreshold): 

1964 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1965 isrFunctions.makeThresholdMask( 

1966 maskedImage=dataView, 

1967 threshold=maskThreshold, 

1968 growFootprints=0, 

1969 maskName=maskName 

1970 ) 

1971 

1972 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1973 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1974 afwImage.PARENT) 

1975 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1976 self.config.suspectMaskName]) 

1977 if numpy.all(maskView.getArray() & maskVal > 0): 

1978 badAmp = True 

1979 maskView |= maskView.getPlaneBitMask("BAD") 

1980 

1981 return badAmp 

1982 

1983 def overscanCorrection(self, ccdExposure, amp): 

1984 """Apply overscan correction in place. 

1985 

1986 This method does initial pixel rejection of the overscan 

1987 region. The overscan can also be optionally segmented to 

1988 allow for discontinuous overscan responses to be fit 

1989 separately. The actual overscan subtraction is performed by 

1990 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1991 which is called here after the amplifier is preprocessed. 

1992 

1993 Parameters 

1994 ---------- 

1995 ccdExposure : `lsst.afw.image.Exposure` 

1996 Exposure to have overscan correction performed. 

1997 amp : `lsst.afw.cameraGeom.Amplifer` 

1998 The amplifier to consider while correcting the overscan. 

1999 

2000 Returns 

2001 ------- 

2002 overscanResults : `lsst.pipe.base.Struct` 

2003 Result struct with components: 

2004 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

2005 Value or fit subtracted from the amplifier image data. 

2006 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

2007 Value or fit subtracted from the overscan image data. 

2008 - ``overscanImage`` : `lsst.afw.image.Image` 

2009 Image of the overscan region with the overscan 

2010 correction applied. This quantity is used to estimate 

2011 the amplifier read noise empirically. 

2012 

2013 Raises 

2014 ------ 

2015 RuntimeError 

2016 Raised if the ``amp`` does not contain raw pixel information. 

2017 

2018 See Also 

2019 -------- 

2020 lsst.ip.isr.isrFunctions.overscanCorrection 

2021 """ 

2022 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2023 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2024 return None 

2025 

2026 statControl = afwMath.StatisticsControl() 

2027 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2028 

2029 # Determine the bounding boxes 

2030 dataBBox = amp.getRawDataBBox() 

2031 oscanBBox = amp.getRawHorizontalOverscanBBox() 

2032 dx0 = 0 

2033 dx1 = 0 

2034 

2035 prescanBBox = amp.getRawPrescanBBox() 

2036 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

2037 dx0 += self.config.overscanNumLeadingColumnsToSkip 

2038 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

2039 else: 

2040 dx0 += self.config.overscanNumTrailingColumnsToSkip 

2041 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

2042 

2043 # Determine if we need to work on subregions of the amplifier and overscan. 

2044 imageBBoxes = [] 

2045 overscanBBoxes = [] 

2046 

2047 if ((self.config.overscanBiasJump 

2048 and self.config.overscanBiasJumpLocation) 

2049 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

2050 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

2051 self.config.overscanBiasJumpDevices)): 

2052 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

2053 yLower = self.config.overscanBiasJumpLocation 

2054 yUpper = dataBBox.getHeight() - yLower 

2055 else: 

2056 yUpper = self.config.overscanBiasJumpLocation 

2057 yLower = dataBBox.getHeight() - yUpper 

2058 

2059 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2060 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

2061 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2062 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2063 yLower))) 

2064 

2065 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

2066 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

2067 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

2068 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2069 yUpper))) 

2070 else: 

2071 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2072 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

2073 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2074 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2075 oscanBBox.getHeight()))) 

2076 

2077 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

2078 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

2079 ampImage = ccdExposure.maskedImage[imageBBox] 

2080 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2081 

2082 overscanArray = overscanImage.image.array 

2083 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2084 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2085 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2086 

2087 statControl = afwMath.StatisticsControl() 

2088 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2089 

2090 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2091 

2092 # Measure average overscan levels and record them in the metadata. 

2093 levelStat = afwMath.MEDIAN 

2094 sigmaStat = afwMath.STDEVCLIP 

2095 

2096 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2097 self.config.qa.flatness.nIter) 

2098 metadata = ccdExposure.getMetadata() 

2099 ampNum = amp.getName() 

2100 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2101 if isinstance(overscanResults.overscanFit, float): 

2102 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2103 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2104 else: 

2105 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2106 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2107 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2108 

2109 return overscanResults 

2110 

2111 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2112 """Set the variance plane using the gain and read noise 

2113 

2114 The read noise is calculated from the ``overscanImage`` if the 

2115 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2116 the value from the amplifier data is used. 

2117 

2118 Parameters 

2119 ---------- 

2120 ampExposure : `lsst.afw.image.Exposure` 

2121 Exposure to process. 

2122 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2123 Amplifier detector data. 

2124 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2125 Image of overscan, required only for empirical read noise. 

2126 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2127 PTC dataset containing the gains and read noise. 

2128 

2129 

2130 Raises 

2131 ------ 

2132 RuntimeError 

2133 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2134 are ``True``, but ptcDataset is not provided. 

2135 

2136 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2137 ``overscanImage`` is ``None``. 

2138 

2139 See also 

2140 -------- 

2141 lsst.ip.isr.isrFunctions.updateVariance 

2142 """ 

2143 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2144 if self.config.usePtcGains: 

2145 if ptcDataset is None: 

2146 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2147 else: 

2148 gain = ptcDataset.gain[amp.getName()] 

2149 self.log.info("Using gain from Photon Transfer Curve.") 

2150 else: 

2151 gain = amp.getGain() 

2152 

2153 if math.isnan(gain): 

2154 gain = 1.0 

2155 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2156 elif gain <= 0: 

2157 patchedGain = 1.0 

2158 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2159 amp.getName(), gain, patchedGain) 

2160 gain = patchedGain 

2161 

2162 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2163 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2164 

2165 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2166 stats = afwMath.StatisticsControl() 

2167 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2168 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2169 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2170 amp.getName(), readNoise) 

2171 elif self.config.usePtcReadNoise: 

2172 if ptcDataset is None: 

2173 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2174 else: 

2175 readNoise = ptcDataset.noise[amp.getName()] 

2176 self.log.info("Using read noise from Photon Transfer Curve.") 

2177 else: 

2178 readNoise = amp.getReadNoise() 

2179 

2180 isrFunctions.updateVariance( 

2181 maskedImage=ampExposure.getMaskedImage(), 

2182 gain=gain, 

2183 readNoise=readNoise, 

2184 ) 

2185 

2186 def darkCorrection(self, exposure, darkExposure, invert=False): 

2187 """Apply dark correction in place. 

2188 

2189 Parameters 

2190 ---------- 

2191 exposure : `lsst.afw.image.Exposure` 

2192 Exposure to process. 

2193 darkExposure : `lsst.afw.image.Exposure` 

2194 Dark exposure of the same size as ``exposure``. 

2195 invert : `Bool`, optional 

2196 If True, re-add the dark to an already corrected image. 

2197 

2198 Raises 

2199 ------ 

2200 RuntimeError 

2201 Raised if either ``exposure`` or ``darkExposure`` do not 

2202 have their dark time defined. 

2203 

2204 See Also 

2205 -------- 

2206 lsst.ip.isr.isrFunctions.darkCorrection 

2207 """ 

2208 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2209 if math.isnan(expScale): 

2210 raise RuntimeError("Exposure darktime is NAN.") 

2211 if darkExposure.getInfo().getVisitInfo() is not None \ 

2212 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2213 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2214 else: 

2215 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2216 # so getDarkTime() does not exist. 

2217 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2218 darkScale = 1.0 

2219 

2220 isrFunctions.darkCorrection( 

2221 maskedImage=exposure.getMaskedImage(), 

2222 darkMaskedImage=darkExposure.getMaskedImage(), 

2223 expScale=expScale, 

2224 darkScale=darkScale, 

2225 invert=invert, 

2226 trimToFit=self.config.doTrimToMatchCalib 

2227 ) 

2228 

2229 def doLinearize(self, detector): 

2230 """Check if linearization is needed for the detector cameraGeom. 

2231 

2232 Checks config.doLinearize and the linearity type of the first 

2233 amplifier. 

2234 

2235 Parameters 

2236 ---------- 

2237 detector : `lsst.afw.cameraGeom.Detector` 

2238 Detector to get linearity type from. 

2239 

2240 Returns 

2241 ------- 

2242 doLinearize : `Bool` 

2243 If True, linearization should be performed. 

2244 """ 

2245 return self.config.doLinearize and \ 

2246 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2247 

2248 def flatCorrection(self, exposure, flatExposure, invert=False): 

2249 """Apply flat correction in place. 

2250 

2251 Parameters 

2252 ---------- 

2253 exposure : `lsst.afw.image.Exposure` 

2254 Exposure to process. 

2255 flatExposure : `lsst.afw.image.Exposure` 

2256 Flat exposure of the same size as ``exposure``. 

2257 invert : `Bool`, optional 

2258 If True, unflatten an already flattened image. 

2259 

2260 See Also 

2261 -------- 

2262 lsst.ip.isr.isrFunctions.flatCorrection 

2263 """ 

2264 isrFunctions.flatCorrection( 

2265 maskedImage=exposure.getMaskedImage(), 

2266 flatMaskedImage=flatExposure.getMaskedImage(), 

2267 scalingType=self.config.flatScalingType, 

2268 userScale=self.config.flatUserScale, 

2269 invert=invert, 

2270 trimToFit=self.config.doTrimToMatchCalib 

2271 ) 

2272 

2273 def saturationDetection(self, exposure, amp): 

2274 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2275 

2276 Parameters 

2277 ---------- 

2278 exposure : `lsst.afw.image.Exposure` 

2279 Exposure to process. Only the amplifier DataSec is processed. 

2280 amp : `lsst.afw.table.AmpInfoCatalog` 

2281 Amplifier detector data. 

2282 

2283 See Also 

2284 -------- 

2285 lsst.ip.isr.isrFunctions.makeThresholdMask 

2286 """ 

2287 if not math.isnan(amp.getSaturation()): 

2288 maskedImage = exposure.getMaskedImage() 

2289 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2290 isrFunctions.makeThresholdMask( 

2291 maskedImage=dataView, 

2292 threshold=amp.getSaturation(), 

2293 growFootprints=0, 

2294 maskName=self.config.saturatedMaskName, 

2295 ) 

2296 

2297 def saturationInterpolation(self, exposure): 

2298 """Interpolate over saturated pixels, in place. 

2299 

2300 This method should be called after `saturationDetection`, to 

2301 ensure that the saturated pixels have been identified in the 

2302 SAT mask. It should also be called after `assembleCcd`, since 

2303 saturated regions may cross amplifier boundaries. 

2304 

2305 Parameters 

2306 ---------- 

2307 exposure : `lsst.afw.image.Exposure` 

2308 Exposure to process. 

2309 

2310 See Also 

2311 -------- 

2312 lsst.ip.isr.isrTask.saturationDetection 

2313 lsst.ip.isr.isrFunctions.interpolateFromMask 

2314 """ 

2315 isrFunctions.interpolateFromMask( 

2316 maskedImage=exposure.getMaskedImage(), 

2317 fwhm=self.config.fwhm, 

2318 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2319 maskNameList=list(self.config.saturatedMaskName), 

2320 ) 

2321 

2322 def suspectDetection(self, exposure, amp): 

2323 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2324 

2325 Parameters 

2326 ---------- 

2327 exposure : `lsst.afw.image.Exposure` 

2328 Exposure to process. Only the amplifier DataSec is processed. 

2329 amp : `lsst.afw.table.AmpInfoCatalog` 

2330 Amplifier detector data. 

2331 

2332 See Also 

2333 -------- 

2334 lsst.ip.isr.isrFunctions.makeThresholdMask 

2335 

2336 Notes 

2337 ----- 

2338 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2339 This is intended to indicate pixels that may be affected by unknown systematics; 

2340 for example if non-linearity corrections above a certain level are unstable 

2341 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2342 that no such level exists and no pixels are to be masked as suspicious. 

2343 """ 

2344 suspectLevel = amp.getSuspectLevel() 

2345 if math.isnan(suspectLevel): 

2346 return 

2347 

2348 maskedImage = exposure.getMaskedImage() 

2349 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2350 isrFunctions.makeThresholdMask( 

2351 maskedImage=dataView, 

2352 threshold=suspectLevel, 

2353 growFootprints=0, 

2354 maskName=self.config.suspectMaskName, 

2355 ) 

2356 

2357 def maskDefect(self, exposure, defectBaseList): 

2358 """Mask defects using mask plane "BAD", in place. 

2359 

2360 Parameters 

2361 ---------- 

2362 exposure : `lsst.afw.image.Exposure` 

2363 Exposure to process. 

2364 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2365 `lsst.afw.image.DefectBase`. 

2366 List of defects to mask. 

2367 

2368 Notes 

2369 ----- 

2370 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2371 """ 

2372 maskedImage = exposure.getMaskedImage() 

2373 if not isinstance(defectBaseList, Defects): 

2374 # Promotes DefectBase to Defect 

2375 defectList = Defects(defectBaseList) 

2376 else: 

2377 defectList = defectBaseList 

2378 defectList.maskPixels(maskedImage, maskName="BAD") 

2379 

2380 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2381 """Mask edge pixels with applicable mask plane. 

2382 

2383 Parameters 

2384 ---------- 

2385 exposure : `lsst.afw.image.Exposure` 

2386 Exposure to process. 

2387 numEdgePixels : `int`, optional 

2388 Number of edge pixels to mask. 

2389 maskPlane : `str`, optional 

2390 Mask plane name to use. 

2391 level : `str`, optional 

2392 Level at which to mask edges. 

2393 """ 

2394 maskedImage = exposure.getMaskedImage() 

2395 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2396 

2397 if numEdgePixels > 0: 

2398 if level == 'DETECTOR': 

2399 boxes = [maskedImage.getBBox()] 

2400 elif level == 'AMP': 

2401 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2402 

2403 for box in boxes: 

2404 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2405 subImage = maskedImage[box] 

2406 box.grow(-numEdgePixels) 

2407 # Mask pixels outside box 

2408 SourceDetectionTask.setEdgeBits( 

2409 subImage, 

2410 box, 

2411 maskBitMask) 

2412 

2413 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2414 """Mask and interpolate defects using mask plane "BAD", in place. 

2415 

2416 Parameters 

2417 ---------- 

2418 exposure : `lsst.afw.image.Exposure` 

2419 Exposure to process. 

2420 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2421 `lsst.afw.image.DefectBase`. 

2422 List of defects to mask and interpolate. 

2423 

2424 See Also 

2425 -------- 

2426 lsst.ip.isr.isrTask.maskDefect 

2427 """ 

2428 self.maskDefect(exposure, defectBaseList) 

2429 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2430 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2431 isrFunctions.interpolateFromMask( 

2432 maskedImage=exposure.getMaskedImage(), 

2433 fwhm=self.config.fwhm, 

2434 growSaturatedFootprints=0, 

2435 maskNameList=["BAD"], 

2436 ) 

2437 

2438 def maskNan(self, exposure): 

2439 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2440 

2441 Parameters 

2442 ---------- 

2443 exposure : `lsst.afw.image.Exposure` 

2444 Exposure to process. 

2445 

2446 Notes 

2447 ----- 

2448 We mask over all non-finite values (NaN, inf), including those 

2449 that are masked with other bits (because those may or may not be 

2450 interpolated over later, and we want to remove all NaN/infs). 

2451 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2452 preserve the historical name. 

2453 """ 

2454 maskedImage = exposure.getMaskedImage() 

2455 

2456 # Find and mask NaNs 

2457 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2458 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2459 numNans = maskNans(maskedImage, maskVal) 

2460 self.metadata.set("NUMNANS", numNans) 

2461 if numNans > 0: 

2462 self.log.warning("There were %d unmasked NaNs.", numNans) 

2463 

2464 def maskAndInterpolateNan(self, exposure): 

2465 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2466 in place. 

2467 

2468 Parameters 

2469 ---------- 

2470 exposure : `lsst.afw.image.Exposure` 

2471 Exposure to process. 

2472 

2473 See Also 

2474 -------- 

2475 lsst.ip.isr.isrTask.maskNan 

2476 """ 

2477 self.maskNan(exposure) 

2478 isrFunctions.interpolateFromMask( 

2479 maskedImage=exposure.getMaskedImage(), 

2480 fwhm=self.config.fwhm, 

2481 growSaturatedFootprints=0, 

2482 maskNameList=["UNMASKEDNAN"], 

2483 ) 

2484 

2485 def measureBackground(self, exposure, IsrQaConfig=None): 

2486 """Measure the image background in subgrids, for quality control purposes. 

2487 

2488 Parameters 

2489 ---------- 

2490 exposure : `lsst.afw.image.Exposure` 

2491 Exposure to process. 

2492 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2493 Configuration object containing parameters on which background 

2494 statistics and subgrids to use. 

2495 """ 

2496 if IsrQaConfig is not None: 

2497 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2498 IsrQaConfig.flatness.nIter) 

2499 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2500 statsControl.setAndMask(maskVal) 

2501 maskedImage = exposure.getMaskedImage() 

2502 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2503 skyLevel = stats.getValue(afwMath.MEDIAN) 

2504 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2505 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2506 metadata = exposure.getMetadata() 

2507 metadata.set('SKYLEVEL', skyLevel) 

2508 metadata.set('SKYSIGMA', skySigma) 

2509 

2510 # calcluating flatlevel over the subgrids 

2511 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2512 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2513 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2514 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2515 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2516 skyLevels = numpy.zeros((nX, nY)) 

2517 

2518 for j in range(nY): 

2519 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2520 for i in range(nX): 

2521 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2522 

2523 xLLC = xc - meshXHalf 

2524 yLLC = yc - meshYHalf 

2525 xURC = xc + meshXHalf - 1 

2526 yURC = yc + meshYHalf - 1 

2527 

2528 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2529 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2530 

2531 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2532 

2533 good = numpy.where(numpy.isfinite(skyLevels)) 

2534 skyMedian = numpy.median(skyLevels[good]) 

2535 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2536 flatness_rms = numpy.std(flatness) 

2537 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2538 

2539 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2540 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2541 nX, nY, flatness_pp, flatness_rms) 

2542 

2543 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2544 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2545 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2546 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2547 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2548 

2549 def roughZeroPoint(self, exposure): 

2550 """Set an approximate magnitude zero point for the exposure. 

2551 

2552 Parameters 

2553 ---------- 

2554 exposure : `lsst.afw.image.Exposure` 

2555 Exposure to process. 

2556 """ 

2557 filterLabel = exposure.getFilterLabel() 

2558 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2559 

2560 if physicalFilter in self.config.fluxMag0T1: 

2561 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2562 else: 

2563 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2564 fluxMag0 = self.config.defaultFluxMag0T1 

2565 

2566 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2567 if not expTime > 0: # handle NaN as well as <= 0 

2568 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2569 return 

2570 

2571 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2572 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2573 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2574 

2575 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2576 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2577 

2578 Parameters 

2579 ---------- 

2580 ccdExposure : `lsst.afw.image.Exposure` 

2581 Exposure to process. 

2582 fpPolygon : `lsst.afw.geom.Polygon` 

2583 Polygon in focal plane coordinates. 

2584 """ 

2585 # Get ccd corners in focal plane coordinates 

2586 ccd = ccdExposure.getDetector() 

2587 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2588 ccdPolygon = Polygon(fpCorners) 

2589 

2590 # Get intersection of ccd corners with fpPolygon 

2591 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2592 

2593 # Transform back to pixel positions and build new polygon 

2594 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2595 validPolygon = Polygon(ccdPoints) 

2596 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2597 

2598 @contextmanager 

2599 def flatContext(self, exp, flat, dark=None): 

2600 """Context manager that applies and removes flats and darks, 

2601 if the task is configured to apply them. 

2602 

2603 Parameters 

2604 ---------- 

2605 exp : `lsst.afw.image.Exposure` 

2606 Exposure to process. 

2607 flat : `lsst.afw.image.Exposure` 

2608 Flat exposure the same size as ``exp``. 

2609 dark : `lsst.afw.image.Exposure`, optional 

2610 Dark exposure the same size as ``exp``. 

2611 

2612 Yields 

2613 ------ 

2614 exp : `lsst.afw.image.Exposure` 

2615 The flat and dark corrected exposure. 

2616 """ 

2617 if self.config.doDark and dark is not None: 

2618 self.darkCorrection(exp, dark) 

2619 if self.config.doFlat: 

2620 self.flatCorrection(exp, flat) 

2621 try: 

2622 yield exp 

2623 finally: 

2624 if self.config.doFlat: 

2625 self.flatCorrection(exp, flat, invert=True) 

2626 if self.config.doDark and dark is not None: 

2627 self.darkCorrection(exp, dark, invert=True) 

2628 

2629 def debugView(self, exposure, stepname): 

2630 """Utility function to examine ISR exposure at different stages. 

2631 

2632 Parameters 

2633 ---------- 

2634 exposure : `lsst.afw.image.Exposure` 

2635 Exposure to view. 

2636 stepname : `str` 

2637 State of processing to view. 

2638 """ 

2639 frame = getDebugFrame(self._display, stepname) 

2640 if frame: 

2641 display = getDisplay(frame) 

2642 display.scale('asinh', 'zscale') 

2643 display.mtv(exposure) 

2644 prompt = "Press Enter to continue [c]... " 

2645 while True: 

2646 ans = input(prompt).lower() 

2647 if ans in ("", "c",): 

2648 break 

2649 

2650 

2651class FakeAmp(object): 

2652 """A Detector-like object that supports returning gain and saturation level 

2653 

2654 This is used when the input exposure does not have a detector. 

2655 

2656 Parameters 

2657 ---------- 

2658 exposure : `lsst.afw.image.Exposure` 

2659 Exposure to generate a fake amplifier for. 

2660 config : `lsst.ip.isr.isrTaskConfig` 

2661 Configuration to apply to the fake amplifier. 

2662 """ 

2663 

2664 def __init__(self, exposure, config): 

2665 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2666 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2667 self._gain = config.gain 

2668 self._readNoise = config.readNoise 

2669 self._saturation = config.saturation 

2670 

2671 def getBBox(self): 

2672 return self._bbox 

2673 

2674 def getRawBBox(self): 

2675 return self._bbox 

2676 

2677 def getRawHorizontalOverscanBBox(self): 

2678 return self._RawHorizontalOverscanBBox 

2679 

2680 def getGain(self): 

2681 return self._gain 

2682 

2683 def getReadNoise(self): 

2684 return self._readNoise 

2685 

2686 def getSaturation(self): 

2687 return self._saturation 

2688 

2689 def getSuspectLevel(self): 

2690 return float("NaN") 

2691 

2692 

2693class RunIsrConfig(pexConfig.Config): 

2694 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2695 

2696 

2697class RunIsrTask(pipeBase.CmdLineTask): 

2698 """Task to wrap the default IsrTask to allow it to be retargeted. 

2699 

2700 The standard IsrTask can be called directly from a command line 

2701 program, but doing so removes the ability of the task to be 

2702 retargeted. As most cameras override some set of the IsrTask 

2703 methods, this would remove those data-specific methods in the 

2704 output post-ISR images. This wrapping class fixes the issue, 

2705 allowing identical post-ISR images to be generated by both the 

2706 processCcd and isrTask code. 

2707 """ 

2708 ConfigClass = RunIsrConfig 

2709 _DefaultName = "runIsr" 

2710 

2711 def __init__(self, *args, **kwargs): 

2712 super().__init__(*args, **kwargs) 

2713 self.makeSubtask("isr") 

2714 

2715 def runDataRef(self, dataRef): 

2716 """ 

2717 Parameters 

2718 ---------- 

2719 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2720 data reference of the detector data to be processed 

2721 

2722 Returns 

2723 ------- 

2724 result : `pipeBase.Struct` 

2725 Result struct with component: 

2726 

2727 - exposure : `lsst.afw.image.Exposure` 

2728 Post-ISR processed exposure. 

2729 """ 

2730 return self.isr.runDataRef(dataRef)