Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42 

43from . import isrFunctions 

44from . import isrQa 

45from . import linearize 

46from .defects import Defects 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 ) 

123 # TODO: DM-25348. This does not work yet to correctly load 

124 # possible crosstalk sources. 

125 crosstalkSources = cT.PrerequisiteInput( 

126 name="isrOverscanCorrected", 

127 doc="Overscan corrected input images.", 

128 storageClass="Exposure", 

129 dimensions=["instrument", "exposure", "detector"], 

130 deferLoad=True, 

131 multiple=True, 

132 lookupFunction=crosstalkSourceLookup, 

133 ) 

134 bias = cT.PrerequisiteInput( 

135 name="bias", 

136 doc="Input bias calibration.", 

137 storageClass="ExposureF", 

138 dimensions=["instrument", "detector"], 

139 isCalibration=True, 

140 ) 

141 dark = cT.PrerequisiteInput( 

142 name='dark', 

143 doc="Input dark calibration.", 

144 storageClass="ExposureF", 

145 dimensions=["instrument", "detector"], 

146 isCalibration=True, 

147 ) 

148 flat = cT.PrerequisiteInput( 

149 name="flat", 

150 doc="Input flat calibration.", 

151 storageClass="ExposureF", 

152 dimensions=["instrument", "physical_filter", "detector"], 

153 isCalibration=True, 

154 ) 

155 ptc = cT.PrerequisiteInput( 

156 name="ptc", 

157 doc="Input Photon Transfer Curve dataset", 

158 storageClass="PhotonTransferCurveDataset", 

159 dimensions=["instrument", "detector"], 

160 isCalibration=True, 

161 ) 

162 fringes = cT.PrerequisiteInput( 

163 name="fringe", 

164 doc="Input fringe calibration.", 

165 storageClass="ExposureF", 

166 dimensions=["instrument", "physical_filter", "detector"], 

167 isCalibration=True, 

168 ) 

169 strayLightData = cT.PrerequisiteInput( 

170 name='yBackground', 

171 doc="Input stray light calibration.", 

172 storageClass="StrayLightData", 

173 dimensions=["instrument", "physical_filter", "detector"], 

174 isCalibration=True, 

175 ) 

176 bfKernel = cT.PrerequisiteInput( 

177 name='bfKernel', 

178 doc="Input brighter-fatter kernel.", 

179 storageClass="NumpyArray", 

180 dimensions=["instrument"], 

181 isCalibration=True, 

182 ) 

183 newBFKernel = cT.PrerequisiteInput( 

184 name='brighterFatterKernel', 

185 doc="Newer complete kernel + gain solutions.", 

186 storageClass="BrighterFatterKernel", 

187 dimensions=["instrument", "detector"], 

188 isCalibration=True, 

189 ) 

190 defects = cT.PrerequisiteInput( 

191 name='defects', 

192 doc="Input defect tables.", 

193 storageClass="Defects", 

194 dimensions=["instrument", "detector"], 

195 isCalibration=True, 

196 ) 

197 linearizer = cT.PrerequisiteInput( 

198 name='linearizer', 

199 storageClass="Linearizer", 

200 doc="Linearity correction calibration.", 

201 dimensions=["instrument", "detector"], 

202 isCalibration=True, 

203 ) 

204 opticsTransmission = cT.PrerequisiteInput( 

205 name="transmission_optics", 

206 storageClass="TransmissionCurve", 

207 doc="Transmission curve due to the optics.", 

208 dimensions=["instrument"], 

209 isCalibration=True, 

210 ) 

211 filterTransmission = cT.PrerequisiteInput( 

212 name="transmission_filter", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the filter.", 

215 dimensions=["instrument", "physical_filter"], 

216 isCalibration=True, 

217 ) 

218 sensorTransmission = cT.PrerequisiteInput( 

219 name="transmission_sensor", 

220 storageClass="TransmissionCurve", 

221 doc="Transmission curve due to the sensor.", 

222 dimensions=["instrument", "detector"], 

223 isCalibration=True, 

224 ) 

225 atmosphereTransmission = cT.PrerequisiteInput( 

226 name="transmission_atmosphere", 

227 storageClass="TransmissionCurve", 

228 doc="Transmission curve due to the atmosphere.", 

229 dimensions=["instrument"], 

230 isCalibration=True, 

231 ) 

232 illumMaskedImage = cT.PrerequisiteInput( 

233 name="illum", 

234 doc="Input illumination correction.", 

235 storageClass="MaskedImageF", 

236 dimensions=["instrument", "physical_filter", "detector"], 

237 isCalibration=True, 

238 ) 

239 

240 outputExposure = cT.Output( 

241 name='postISRCCD', 

242 doc="Output ISR processed exposure.", 

243 storageClass="Exposure", 

244 dimensions=["instrument", "exposure", "detector"], 

245 ) 

246 preInterpExposure = cT.Output( 

247 name='preInterpISRCCD', 

248 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

249 storageClass="ExposureF", 

250 dimensions=["instrument", "exposure", "detector"], 

251 ) 

252 outputOssThumbnail = cT.Output( 

253 name="OssThumb", 

254 doc="Output Overscan-subtracted thumbnail image.", 

255 storageClass="Thumbnail", 

256 dimensions=["instrument", "exposure", "detector"], 

257 ) 

258 outputFlattenedThumbnail = cT.Output( 

259 name="FlattenedThumb", 

260 doc="Output flat-corrected thumbnail image.", 

261 storageClass="Thumbnail", 

262 dimensions=["instrument", "exposure", "detector"], 

263 ) 

264 

265 def __init__(self, *, config=None): 

266 super().__init__(config=config) 

267 

268 if config.doBias is not True: 

269 self.prerequisiteInputs.discard("bias") 

270 if config.doLinearize is not True: 

271 self.prerequisiteInputs.discard("linearizer") 

272 if config.doCrosstalk is not True: 

273 self.inputs.discard("crosstalkSources") 

274 self.prerequisiteInputs.discard("crosstalk") 

275 if config.doBrighterFatter is not True: 

276 self.prerequisiteInputs.discard("bfKernel") 

277 self.prerequisiteInputs.discard("newBFKernel") 

278 if config.doDefect is not True: 

279 self.prerequisiteInputs.discard("defects") 

280 if config.doDark is not True: 

281 self.prerequisiteInputs.discard("dark") 

282 if config.doFlat is not True: 

283 self.prerequisiteInputs.discard("flat") 

284 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

285 self.prerequisiteInputs.discard("ptc") 

286 if config.doAttachTransmissionCurve is not True: 

287 self.prerequisiteInputs.discard("opticsTransmission") 

288 self.prerequisiteInputs.discard("filterTransmission") 

289 self.prerequisiteInputs.discard("sensorTransmission") 

290 self.prerequisiteInputs.discard("atmosphereTransmission") 

291 if config.doUseOpticsTransmission is not True: 

292 self.prerequisiteInputs.discard("opticsTransmission") 

293 if config.doUseFilterTransmission is not True: 

294 self.prerequisiteInputs.discard("filterTransmission") 

295 if config.doUseSensorTransmission is not True: 

296 self.prerequisiteInputs.discard("sensorTransmission") 

297 if config.doUseAtmosphereTransmission is not True: 

298 self.prerequisiteInputs.discard("atmosphereTransmission") 

299 if config.doIlluminationCorrection is not True: 

300 self.prerequisiteInputs.discard("illumMaskedImage") 

301 

302 if config.doWrite is not True: 

303 self.outputs.discard("outputExposure") 

304 self.outputs.discard("preInterpExposure") 

305 self.outputs.discard("outputFlattenedThumbnail") 

306 self.outputs.discard("outputOssThumbnail") 

307 if config.doSaveInterpPixels is not True: 

308 self.outputs.discard("preInterpExposure") 

309 if config.qa.doThumbnailOss is not True: 

310 self.outputs.discard("outputOssThumbnail") 

311 if config.qa.doThumbnailFlattened is not True: 

312 self.outputs.discard("outputFlattenedThumbnail") 

313 

314 

315class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

316 pipelineConnections=IsrTaskConnections): 

317 """Configuration parameters for IsrTask. 

318 

319 Items are grouped in the order in which they are executed by the task. 

320 """ 

321 datasetType = pexConfig.Field( 

322 dtype=str, 

323 doc="Dataset type for input data; users will typically leave this alone, " 

324 "but camera-specific ISR tasks will override it", 

325 default="raw", 

326 ) 

327 

328 fallbackFilterName = pexConfig.Field( 

329 dtype=str, 

330 doc="Fallback default filter name for calibrations.", 

331 optional=True 

332 ) 

333 useFallbackDate = pexConfig.Field( 

334 dtype=bool, 

335 doc="Pass observation date when using fallback filter.", 

336 default=False, 

337 ) 

338 expectWcs = pexConfig.Field( 

339 dtype=bool, 

340 default=True, 

341 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

342 ) 

343 fwhm = pexConfig.Field( 

344 dtype=float, 

345 doc="FWHM of PSF in arcseconds.", 

346 default=1.0, 

347 ) 

348 qa = pexConfig.ConfigField( 

349 dtype=isrQa.IsrQaConfig, 

350 doc="QA related configuration options.", 

351 ) 

352 

353 # Image conversion configuration 

354 doConvertIntToFloat = pexConfig.Field( 

355 dtype=bool, 

356 doc="Convert integer raw images to floating point values?", 

357 default=True, 

358 ) 

359 

360 # Saturated pixel handling. 

361 doSaturation = pexConfig.Field( 

362 dtype=bool, 

363 doc="Mask saturated pixels? NB: this is totally independent of the" 

364 " interpolation option - this is ONLY setting the bits in the mask." 

365 " To have them interpolated make sure doSaturationInterpolation=True", 

366 default=True, 

367 ) 

368 saturatedMaskName = pexConfig.Field( 

369 dtype=str, 

370 doc="Name of mask plane to use in saturation detection and interpolation", 

371 default="SAT", 

372 ) 

373 saturation = pexConfig.Field( 

374 dtype=float, 

375 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

376 default=float("NaN"), 

377 ) 

378 growSaturationFootprintSize = pexConfig.Field( 

379 dtype=int, 

380 doc="Number of pixels by which to grow the saturation footprints", 

381 default=1, 

382 ) 

383 

384 # Suspect pixel handling. 

385 doSuspect = pexConfig.Field( 

386 dtype=bool, 

387 doc="Mask suspect pixels?", 

388 default=False, 

389 ) 

390 suspectMaskName = pexConfig.Field( 

391 dtype=str, 

392 doc="Name of mask plane to use for suspect pixels", 

393 default="SUSPECT", 

394 ) 

395 numEdgeSuspect = pexConfig.Field( 

396 dtype=int, 

397 doc="Number of edge pixels to be flagged as untrustworthy.", 

398 default=0, 

399 ) 

400 edgeMaskLevel = pexConfig.ChoiceField( 

401 dtype=str, 

402 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

403 default="DETECTOR", 

404 allowed={ 

405 'DETECTOR': 'Mask only the edges of the full detector.', 

406 'AMP': 'Mask edges of each amplifier.', 

407 }, 

408 ) 

409 

410 # Initial masking options. 

411 doSetBadRegions = pexConfig.Field( 

412 dtype=bool, 

413 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

414 default=True, 

415 ) 

416 badStatistic = pexConfig.ChoiceField( 

417 dtype=str, 

418 doc="How to estimate the average value for BAD regions.", 

419 default='MEANCLIP', 

420 allowed={ 

421 "MEANCLIP": "Correct using the (clipped) mean of good data", 

422 "MEDIAN": "Correct using the median of the good data", 

423 }, 

424 ) 

425 

426 # Overscan subtraction configuration. 

427 doOverscan = pexConfig.Field( 

428 dtype=bool, 

429 doc="Do overscan subtraction?", 

430 default=True, 

431 ) 

432 overscan = pexConfig.ConfigurableField( 

433 target=OverscanCorrectionTask, 

434 doc="Overscan subtraction task for image segments.", 

435 ) 

436 

437 overscanFitType = pexConfig.ChoiceField( 

438 dtype=str, 

439 doc="The method for fitting the overscan bias level.", 

440 default='MEDIAN', 

441 allowed={ 

442 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

443 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

444 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

445 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

446 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

447 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

448 "MEAN": "Correct using the mean of the overscan region", 

449 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

450 "MEDIAN": "Correct using the median of the overscan region", 

451 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

452 }, 

453 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

454 " This option will no longer be used, and will be removed after v20.") 

455 ) 

456 overscanOrder = pexConfig.Field( 

457 dtype=int, 

458 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

459 "or number of spline knots if overscan fit type is a spline."), 

460 default=1, 

461 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

462 " This option will no longer be used, and will be removed after v20.") 

463 ) 

464 overscanNumSigmaClip = pexConfig.Field( 

465 dtype=float, 

466 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

467 default=3.0, 

468 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

469 " This option will no longer be used, and will be removed after v20.") 

470 ) 

471 overscanIsInt = pexConfig.Field( 

472 dtype=bool, 

473 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

474 " and overscan.FitType=MEDIAN_PER_ROW.", 

475 default=True, 

476 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

477 " This option will no longer be used, and will be removed after v20.") 

478 ) 

479 # These options do not get deprecated, as they define how we slice up the image data. 

480 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

481 dtype=int, 

482 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

483 default=0, 

484 ) 

485 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

486 dtype=int, 

487 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

488 default=0, 

489 ) 

490 overscanMaxDev = pexConfig.Field( 490 ↛ exitline 490 didn't jump to the function exit

491 dtype=float, 

492 doc="Maximum deviation from the median for overscan", 

493 default=1000.0, check=lambda x: x > 0 

494 ) 

495 overscanBiasJump = pexConfig.Field( 

496 dtype=bool, 

497 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

498 default=False, 

499 ) 

500 overscanBiasJumpKeyword = pexConfig.Field( 

501 dtype=str, 

502 doc="Header keyword containing information about devices.", 

503 default="NO_SUCH_KEY", 

504 ) 

505 overscanBiasJumpDevices = pexConfig.ListField( 

506 dtype=str, 

507 doc="List of devices that need piecewise overscan correction.", 

508 default=(), 

509 ) 

510 overscanBiasJumpLocation = pexConfig.Field( 

511 dtype=int, 

512 doc="Location of bias jump along y-axis.", 

513 default=0, 

514 ) 

515 

516 # Amplifier to CCD assembly configuration 

517 doAssembleCcd = pexConfig.Field( 

518 dtype=bool, 

519 default=True, 

520 doc="Assemble amp-level exposures into a ccd-level exposure?" 

521 ) 

522 assembleCcd = pexConfig.ConfigurableField( 

523 target=AssembleCcdTask, 

524 doc="CCD assembly task", 

525 ) 

526 

527 # General calibration configuration. 

528 doAssembleIsrExposures = pexConfig.Field( 

529 dtype=bool, 

530 default=False, 

531 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

532 ) 

533 doTrimToMatchCalib = pexConfig.Field( 

534 dtype=bool, 

535 default=False, 

536 doc="Trim raw data to match calibration bounding boxes?" 

537 ) 

538 

539 # Bias subtraction. 

540 doBias = pexConfig.Field( 

541 dtype=bool, 

542 doc="Apply bias frame correction?", 

543 default=True, 

544 ) 

545 biasDataProductName = pexConfig.Field( 

546 dtype=str, 

547 doc="Name of the bias data product", 

548 default="bias", 

549 ) 

550 doBiasBeforeOverscan = pexConfig.Field( 

551 dtype=bool, 

552 doc="Reverse order of overscan and bias correction.", 

553 default=False 

554 ) 

555 

556 # Variance construction 

557 doVariance = pexConfig.Field( 

558 dtype=bool, 

559 doc="Calculate variance?", 

560 default=True 

561 ) 

562 gain = pexConfig.Field( 

563 dtype=float, 

564 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

565 default=float("NaN"), 

566 ) 

567 readNoise = pexConfig.Field( 

568 dtype=float, 

569 doc="The read noise to use if no Detector is present in the Exposure", 

570 default=0.0, 

571 ) 

572 doEmpiricalReadNoise = pexConfig.Field( 

573 dtype=bool, 

574 default=False, 

575 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

576 ) 

577 usePtcReadNoise = pexConfig.Field( 

578 dtype=bool, 

579 default=False, 

580 doc="Use readnoise values from the Photon Transfer Curve?" 

581 ) 

582 # Linearization. 

583 doLinearize = pexConfig.Field( 

584 dtype=bool, 

585 doc="Correct for nonlinearity of the detector's response?", 

586 default=True, 

587 ) 

588 

589 # Crosstalk. 

590 doCrosstalk = pexConfig.Field( 

591 dtype=bool, 

592 doc="Apply intra-CCD crosstalk correction?", 

593 default=False, 

594 ) 

595 doCrosstalkBeforeAssemble = pexConfig.Field( 

596 dtype=bool, 

597 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

598 default=False, 

599 ) 

600 crosstalk = pexConfig.ConfigurableField( 

601 target=CrosstalkTask, 

602 doc="Intra-CCD crosstalk correction", 

603 ) 

604 

605 # Masking options. 

606 doDefect = pexConfig.Field( 

607 dtype=bool, 

608 doc="Apply correction for CCD defects, e.g. hot pixels?", 

609 default=True, 

610 ) 

611 doNanMasking = pexConfig.Field( 

612 dtype=bool, 

613 doc="Mask non-finite (NAN, inf) pixels?", 

614 default=True, 

615 ) 

616 doWidenSaturationTrails = pexConfig.Field( 

617 dtype=bool, 

618 doc="Widen bleed trails based on their width?", 

619 default=True 

620 ) 

621 

622 # Brighter-Fatter correction. 

623 doBrighterFatter = pexConfig.Field( 

624 dtype=bool, 

625 default=False, 

626 doc="Apply the brighter-fatter correction?" 

627 ) 

628 brighterFatterLevel = pexConfig.ChoiceField( 

629 dtype=str, 

630 default="DETECTOR", 

631 doc="The level at which to correct for brighter-fatter.", 

632 allowed={ 

633 "AMP": "Every amplifier treated separately.", 

634 "DETECTOR": "One kernel per detector", 

635 } 

636 ) 

637 brighterFatterMaxIter = pexConfig.Field( 

638 dtype=int, 

639 default=10, 

640 doc="Maximum number of iterations for the brighter-fatter correction" 

641 ) 

642 brighterFatterThreshold = pexConfig.Field( 

643 dtype=float, 

644 default=1000, 

645 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

646 "absolute value of the difference between the current corrected image and the one " 

647 "from the previous iteration summed over all the pixels." 

648 ) 

649 brighterFatterApplyGain = pexConfig.Field( 

650 dtype=bool, 

651 default=True, 

652 doc="Should the gain be applied when applying the brighter-fatter correction?" 

653 ) 

654 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

655 dtype=str, 

656 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

657 "correction.", 

658 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

659 ) 

660 brighterFatterMaskGrowSize = pexConfig.Field( 

661 dtype=int, 

662 default=0, 

663 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

664 "when brighter-fatter correction is applied." 

665 ) 

666 

667 # Dark subtraction. 

668 doDark = pexConfig.Field( 

669 dtype=bool, 

670 doc="Apply dark frame correction?", 

671 default=True, 

672 ) 

673 darkDataProductName = pexConfig.Field( 

674 dtype=str, 

675 doc="Name of the dark data product", 

676 default="dark", 

677 ) 

678 

679 # Camera-specific stray light removal. 

680 doStrayLight = pexConfig.Field( 

681 dtype=bool, 

682 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

683 default=False, 

684 ) 

685 strayLight = pexConfig.ConfigurableField( 

686 target=StrayLightTask, 

687 doc="y-band stray light correction" 

688 ) 

689 

690 # Flat correction. 

691 doFlat = pexConfig.Field( 

692 dtype=bool, 

693 doc="Apply flat field correction?", 

694 default=True, 

695 ) 

696 flatDataProductName = pexConfig.Field( 

697 dtype=str, 

698 doc="Name of the flat data product", 

699 default="flat", 

700 ) 

701 flatScalingType = pexConfig.ChoiceField( 

702 dtype=str, 

703 doc="The method for scaling the flat on the fly.", 

704 default='USER', 

705 allowed={ 

706 "USER": "Scale by flatUserScale", 

707 "MEAN": "Scale by the inverse of the mean", 

708 "MEDIAN": "Scale by the inverse of the median", 

709 }, 

710 ) 

711 flatUserScale = pexConfig.Field( 

712 dtype=float, 

713 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

714 default=1.0, 

715 ) 

716 doTweakFlat = pexConfig.Field( 

717 dtype=bool, 

718 doc="Tweak flats to match observed amplifier ratios?", 

719 default=False 

720 ) 

721 

722 # Amplifier normalization based on gains instead of using flats configuration. 

723 doApplyGains = pexConfig.Field( 

724 dtype=bool, 

725 doc="Correct the amplifiers for their gains instead of applying flat correction", 

726 default=False, 

727 ) 

728 usePtcGains = pexConfig.Field( 

729 dtype=bool, 

730 doc="Use the gain values from the Photon Transfer Curve?", 

731 default=False, 

732 ) 

733 normalizeGains = pexConfig.Field( 

734 dtype=bool, 

735 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

736 default=False, 

737 ) 

738 

739 # Fringe correction. 

740 doFringe = pexConfig.Field( 

741 dtype=bool, 

742 doc="Apply fringe correction?", 

743 default=True, 

744 ) 

745 fringe = pexConfig.ConfigurableField( 

746 target=FringeTask, 

747 doc="Fringe subtraction task", 

748 ) 

749 fringeAfterFlat = pexConfig.Field( 

750 dtype=bool, 

751 doc="Do fringe subtraction after flat-fielding?", 

752 default=True, 

753 ) 

754 

755 # Initial CCD-level background statistics options. 

756 doMeasureBackground = pexConfig.Field( 

757 dtype=bool, 

758 doc="Measure the background level on the reduced image?", 

759 default=False, 

760 ) 

761 

762 # Camera-specific masking configuration. 

763 doCameraSpecificMasking = pexConfig.Field( 

764 dtype=bool, 

765 doc="Mask camera-specific bad regions?", 

766 default=False, 

767 ) 

768 masking = pexConfig.ConfigurableField( 

769 target=MaskingTask, 

770 doc="Masking task." 

771 ) 

772 

773 # Interpolation options. 

774 

775 doInterpolate = pexConfig.Field( 

776 dtype=bool, 

777 doc="Interpolate masked pixels?", 

778 default=True, 

779 ) 

780 doSaturationInterpolation = pexConfig.Field( 

781 dtype=bool, 

782 doc="Perform interpolation over pixels masked as saturated?" 

783 " NB: This is independent of doSaturation; if that is False this plane" 

784 " will likely be blank, resulting in a no-op here.", 

785 default=True, 

786 ) 

787 doNanInterpolation = pexConfig.Field( 

788 dtype=bool, 

789 doc="Perform interpolation over pixels masked as NaN?" 

790 " NB: This is independent of doNanMasking; if that is False this plane" 

791 " will likely be blank, resulting in a no-op here.", 

792 default=True, 

793 ) 

794 doNanInterpAfterFlat = pexConfig.Field( 

795 dtype=bool, 

796 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

797 "also have to interpolate them before flat-fielding."), 

798 default=False, 

799 ) 

800 maskListToInterpolate = pexConfig.ListField( 

801 dtype=str, 

802 doc="List of mask planes that should be interpolated.", 

803 default=['SAT', 'BAD'], 

804 ) 

805 doSaveInterpPixels = pexConfig.Field( 

806 dtype=bool, 

807 doc="Save a copy of the pre-interpolated pixel values?", 

808 default=False, 

809 ) 

810 

811 # Default photometric calibration options. 

812 fluxMag0T1 = pexConfig.DictField( 

813 keytype=str, 

814 itemtype=float, 

815 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

816 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

817 )) 

818 ) 

819 defaultFluxMag0T1 = pexConfig.Field( 

820 dtype=float, 

821 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

822 default=pow(10.0, 0.4*28.0) 

823 ) 

824 

825 # Vignette correction configuration. 

826 doVignette = pexConfig.Field( 

827 dtype=bool, 

828 doc="Apply vignetting parameters?", 

829 default=False, 

830 ) 

831 vignette = pexConfig.ConfigurableField( 

832 target=VignetteTask, 

833 doc="Vignetting task.", 

834 ) 

835 

836 # Transmission curve configuration. 

837 doAttachTransmissionCurve = pexConfig.Field( 

838 dtype=bool, 

839 default=False, 

840 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

841 ) 

842 doUseOpticsTransmission = pexConfig.Field( 

843 dtype=bool, 

844 default=True, 

845 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

846 ) 

847 doUseFilterTransmission = pexConfig.Field( 

848 dtype=bool, 

849 default=True, 

850 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

851 ) 

852 doUseSensorTransmission = pexConfig.Field( 

853 dtype=bool, 

854 default=True, 

855 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

856 ) 

857 doUseAtmosphereTransmission = pexConfig.Field( 

858 dtype=bool, 

859 default=True, 

860 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

861 ) 

862 

863 # Illumination correction. 

864 doIlluminationCorrection = pexConfig.Field( 

865 dtype=bool, 

866 default=False, 

867 doc="Perform illumination correction?" 

868 ) 

869 illuminationCorrectionDataProductName = pexConfig.Field( 

870 dtype=str, 

871 doc="Name of the illumination correction data product.", 

872 default="illumcor", 

873 ) 

874 illumScale = pexConfig.Field( 

875 dtype=float, 

876 doc="Scale factor for the illumination correction.", 

877 default=1.0, 

878 ) 

879 illumFilters = pexConfig.ListField( 

880 dtype=str, 

881 default=[], 

882 doc="Only perform illumination correction for these filters." 

883 ) 

884 

885 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

886 doWrite = pexConfig.Field( 

887 dtype=bool, 

888 doc="Persist postISRCCD?", 

889 default=True, 

890 ) 

891 

892 def validate(self): 

893 super().validate() 

894 if self.doFlat and self.doApplyGains: 

895 raise ValueError("You may not specify both doFlat and doApplyGains") 

896 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

897 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

898 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

899 self.maskListToInterpolate.append(self.saturatedMaskName) 

900 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

901 self.maskListToInterpolate.remove(self.saturatedMaskName) 

902 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

903 self.maskListToInterpolate.append("UNMASKEDNAN") 

904 

905 

906class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

907 """Apply common instrument signature correction algorithms to a raw frame. 

908 

909 The process for correcting imaging data is very similar from 

910 camera to camera. This task provides a vanilla implementation of 

911 doing these corrections, including the ability to turn certain 

912 corrections off if they are not needed. The inputs to the primary 

913 method, `run()`, are a raw exposure to be corrected and the 

914 calibration data products. The raw input is a single chip sized 

915 mosaic of all amps including overscans and other non-science 

916 pixels. The method `runDataRef()` identifies and defines the 

917 calibration data products, and is intended for use by a 

918 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

919 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

920 subclassed for different camera, although the most camera specific 

921 methods have been split into subtasks that can be redirected 

922 appropriately. 

923 

924 The __init__ method sets up the subtasks for ISR processing, using 

925 the defaults from `lsst.ip.isr`. 

926 

927 Parameters 

928 ---------- 

929 args : `list` 

930 Positional arguments passed to the Task constructor. None used at this time. 

931 kwargs : `dict`, optional 

932 Keyword arguments passed on to the Task constructor. None used at this time. 

933 """ 

934 ConfigClass = IsrTaskConfig 

935 _DefaultName = "isr" 

936 

937 def __init__(self, **kwargs): 

938 super().__init__(**kwargs) 

939 self.makeSubtask("assembleCcd") 

940 self.makeSubtask("crosstalk") 

941 self.makeSubtask("strayLight") 

942 self.makeSubtask("fringe") 

943 self.makeSubtask("masking") 

944 self.makeSubtask("overscan") 

945 self.makeSubtask("vignette") 

946 

947 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

948 inputs = butlerQC.get(inputRefs) 

949 

950 try: 

951 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

952 except Exception as e: 

953 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

954 (inputRefs, e)) 

955 

956 inputs['isGen3'] = True 

957 

958 detector = inputs['ccdExposure'].getDetector() 

959 

960 if self.config.doCrosstalk is True: 

961 # Crosstalk sources need to be defined by the pipeline 

962 # yaml if they exist. 

963 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

964 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

965 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

966 else: 

967 coeffVector = (self.config.crosstalk.crosstalkValues 

968 if self.config.crosstalk.useConfigCoefficients else None) 

969 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

970 inputs['crosstalk'] = crosstalkCalib 

971 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

972 if 'crosstalkSources' not in inputs: 

973 self.log.warn("No crosstalkSources found for chip with interChip terms!") 

974 

975 if self.doLinearize(detector) is True: 

976 if 'linearizer' in inputs: 

977 if isinstance(inputs['linearizer'], dict): 

978 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

979 linearizer.fromYaml(inputs['linearizer']) 

980 self.log.warn("Dictionary linearizers will be deprecated in DM-28741.") 

981 elif isinstance(inputs['linearizer'], numpy.ndarray): 

982 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

983 detector=detector, 

984 log=self.log) 

985 self.log.warn("Bare lookup table linearizers will be deprecated in DM-28741.") 

986 else: 

987 linearizer = inputs['linearizer'] 

988 linearizer.log = self.log 

989 inputs['linearizer'] = linearizer 

990 else: 

991 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

992 self.log.warn("Constructing linearizer from cameraGeom information.") 

993 

994 if self.config.doDefect is True: 

995 if "defects" in inputs and inputs['defects'] is not None: 

996 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

997 # masking expects a list of defects defined by their bounding box 

998 if not isinstance(inputs["defects"], Defects): 

999 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1000 

1001 # Load the correct style of brighter-fatter kernel, and repack 

1002 # the information as a numpy array. 

1003 if self.config.doBrighterFatter: 

1004 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1005 if brighterFatterKernel is None: 

1006 brighterFatterKernel = inputs.get('bfKernel', None) 

1007 

1008 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1009 # This is a ISR calib kernel 

1010 detName = detector.getName() 

1011 level = brighterFatterKernel.level 

1012 

1013 # This is expected to be a dictionary of amp-wise gains. 

1014 inputs['bfGains'] = brighterFatterKernel.gain 

1015 if self.config.brighterFatterLevel == 'DETECTOR': 

1016 if level == 'DETECTOR': 

1017 if detName in brighterFatterKernel.detKernels: 

1018 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1019 else: 

1020 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1021 elif level == 'AMP': 

1022 self.log.warn("Making DETECTOR level kernel from AMP based brighter fatter kernels.") 

1023 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1024 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1025 elif self.config.brighterFatterLevel == 'AMP': 

1026 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1027 

1028 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1029 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

1030 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1031 expId=expId, 

1032 assembler=self.assembleCcd 

1033 if self.config.doAssembleIsrExposures else None) 

1034 else: 

1035 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1036 

1037 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1038 if 'strayLightData' not in inputs: 

1039 inputs['strayLightData'] = None 

1040 

1041 outputs = self.run(**inputs) 

1042 butlerQC.put(outputs, outputRefs) 

1043 

1044 def readIsrData(self, dataRef, rawExposure): 

1045 """Retrieve necessary frames for instrument signature removal. 

1046 

1047 Pre-fetching all required ISR data products limits the IO 

1048 required by the ISR. Any conflict between the calibration data 

1049 available and that needed for ISR is also detected prior to 

1050 doing processing, allowing it to fail quickly. 

1051 

1052 Parameters 

1053 ---------- 

1054 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1055 Butler reference of the detector data to be processed 

1056 rawExposure : `afw.image.Exposure` 

1057 The raw exposure that will later be corrected with the 

1058 retrieved calibration data; should not be modified in this 

1059 method. 

1060 

1061 Returns 

1062 ------- 

1063 result : `lsst.pipe.base.Struct` 

1064 Result struct with components (which may be `None`): 

1065 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1066 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1067 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1068 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1069 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1070 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1071 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1072 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1073 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1074 - ``seed``: random seed derived from the ccdExposureId for random 

1075 number generator (`uint32`). 

1076 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1077 A ``TransmissionCurve`` that represents the throughput of the optics, 

1078 to be evaluated in focal-plane coordinates. 

1079 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1080 A ``TransmissionCurve`` that represents the throughput of the filter 

1081 itself, to be evaluated in focal-plane coordinates. 

1082 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1083 A ``TransmissionCurve`` that represents the throughput of the sensor 

1084 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1085 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1086 A ``TransmissionCurve`` that represents the throughput of the 

1087 atmosphere, assumed to be spatially constant. 

1088 - ``strayLightData`` : `object` 

1089 An opaque object containing calibration information for 

1090 stray-light correction. If `None`, no correction will be 

1091 performed. 

1092 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1093 

1094 Raises 

1095 ------ 

1096 NotImplementedError : 

1097 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1098 """ 

1099 try: 

1100 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1101 dateObs = dateObs.toPython().isoformat() 

1102 except RuntimeError: 

1103 self.log.warn("Unable to identify dateObs for rawExposure.") 

1104 dateObs = None 

1105 

1106 ccd = rawExposure.getDetector() 

1107 filterLabel = rawExposure.getFilterLabel() 

1108 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1109 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1110 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1111 if self.config.doBias else None) 

1112 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1113 linearizer = (dataRef.get("linearizer", immediate=True) 

1114 if self.doLinearize(ccd) else None) 

1115 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1116 linearizer.log = self.log 

1117 if isinstance(linearizer, numpy.ndarray): 

1118 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1119 

1120 crosstalkCalib = None 

1121 if self.config.doCrosstalk: 

1122 try: 

1123 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1124 except NoResults: 

1125 coeffVector = (self.config.crosstalk.crosstalkValues 

1126 if self.config.crosstalk.useConfigCoefficients else None) 

1127 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1128 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1129 if self.config.doCrosstalk else None) 

1130 

1131 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1132 if self.config.doDark else None) 

1133 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1134 dateObs=dateObs) 

1135 if self.config.doFlat else None) 

1136 

1137 brighterFatterKernel = None 

1138 brighterFatterGains = None 

1139 if self.config.doBrighterFatter is True: 

1140 try: 

1141 # Use the new-style cp_pipe version of the kernel if it exists 

1142 # If using a new-style kernel, always use the self-consistent 

1143 # gains, i.e. the ones inside the kernel object itself 

1144 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1145 brighterFatterGains = brighterFatterKernel.gain 

1146 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded") 

1147 except NoResults: 

1148 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1149 brighterFatterKernel = dataRef.get("bfKernel") 

1150 self.log.info("Old style brighter-fatter kernel (np.array) loaded") 

1151 except NoResults: 

1152 brighterFatterKernel = None 

1153 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1154 # If the kernel is not an ndarray, it's the cp_pipe version 

1155 # so extract the kernel for this detector, or raise an error 

1156 if self.config.brighterFatterLevel == 'DETECTOR': 

1157 if brighterFatterKernel.detectorKernel: 

1158 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

1159 else: 

1160 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1161 else: 

1162 # TODO DM-15631 for implementing this 

1163 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1164 

1165 defectList = (dataRef.get("defects") 

1166 if self.config.doDefect else None) 

1167 expId = rawExposure.getInfo().getVisitInfo().getExposureId() 

1168 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd 

1169 if self.config.doAssembleIsrExposures else None) 

1170 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1171 else pipeBase.Struct(fringes=None)) 

1172 

1173 if self.config.doAttachTransmissionCurve: 

1174 opticsTransmission = (dataRef.get("transmission_optics") 

1175 if self.config.doUseOpticsTransmission else None) 

1176 filterTransmission = (dataRef.get("transmission_filter") 

1177 if self.config.doUseFilterTransmission else None) 

1178 sensorTransmission = (dataRef.get("transmission_sensor") 

1179 if self.config.doUseSensorTransmission else None) 

1180 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1181 if self.config.doUseAtmosphereTransmission else None) 

1182 else: 

1183 opticsTransmission = None 

1184 filterTransmission = None 

1185 sensorTransmission = None 

1186 atmosphereTransmission = None 

1187 

1188 if self.config.doStrayLight: 

1189 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1190 else: 

1191 strayLightData = None 

1192 

1193 illumMaskedImage = (self.getIsrExposure(dataRef, 

1194 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1195 if (self.config.doIlluminationCorrection 

1196 and physicalFilter in self.config.illumFilters) 

1197 else None) 

1198 

1199 # Struct should include only kwargs to run() 

1200 return pipeBase.Struct(bias=biasExposure, 

1201 linearizer=linearizer, 

1202 crosstalk=crosstalkCalib, 

1203 crosstalkSources=crosstalkSources, 

1204 dark=darkExposure, 

1205 flat=flatExposure, 

1206 bfKernel=brighterFatterKernel, 

1207 bfGains=brighterFatterGains, 

1208 defects=defectList, 

1209 fringes=fringeStruct, 

1210 opticsTransmission=opticsTransmission, 

1211 filterTransmission=filterTransmission, 

1212 sensorTransmission=sensorTransmission, 

1213 atmosphereTransmission=atmosphereTransmission, 

1214 strayLightData=strayLightData, 

1215 illumMaskedImage=illumMaskedImage 

1216 ) 

1217 

1218 @pipeBase.timeMethod 

1219 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1220 crosstalk=None, crosstalkSources=None, 

1221 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1222 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1223 sensorTransmission=None, atmosphereTransmission=None, 

1224 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1225 isGen3=False, 

1226 ): 

1227 """Perform instrument signature removal on an exposure. 

1228 

1229 Steps included in the ISR processing, in order performed, are: 

1230 - saturation and suspect pixel masking 

1231 - overscan subtraction 

1232 - CCD assembly of individual amplifiers 

1233 - bias subtraction 

1234 - variance image construction 

1235 - linearization of non-linear response 

1236 - crosstalk masking 

1237 - brighter-fatter correction 

1238 - dark subtraction 

1239 - fringe correction 

1240 - stray light subtraction 

1241 - flat correction 

1242 - masking of known defects and camera specific features 

1243 - vignette calculation 

1244 - appending transmission curve and distortion model 

1245 

1246 Parameters 

1247 ---------- 

1248 ccdExposure : `lsst.afw.image.Exposure` 

1249 The raw exposure that is to be run through ISR. The 

1250 exposure is modified by this method. 

1251 camera : `lsst.afw.cameraGeom.Camera`, optional 

1252 The camera geometry for this exposure. Required if ``isGen3`` is 

1253 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1254 ``flat`` does not have an associated detector. 

1255 bias : `lsst.afw.image.Exposure`, optional 

1256 Bias calibration frame. 

1257 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1258 Functor for linearization. 

1259 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1260 Calibration for crosstalk. 

1261 crosstalkSources : `list`, optional 

1262 List of possible crosstalk sources. 

1263 dark : `lsst.afw.image.Exposure`, optional 

1264 Dark calibration frame. 

1265 flat : `lsst.afw.image.Exposure`, optional 

1266 Flat calibration frame. 

1267 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1268 Photon transfer curve dataset, with, e.g., gains 

1269 and read noise. 

1270 bfKernel : `numpy.ndarray`, optional 

1271 Brighter-fatter kernel. 

1272 bfGains : `dict` of `float`, optional 

1273 Gains used to override the detector's nominal gains for the 

1274 brighter-fatter correction. A dict keyed by amplifier name for 

1275 the detector in question. 

1276 defects : `lsst.ip.isr.Defects`, optional 

1277 List of defects. 

1278 fringes : `lsst.pipe.base.Struct`, optional 

1279 Struct containing the fringe correction data, with 

1280 elements: 

1281 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1282 - ``seed``: random seed derived from the ccdExposureId for random 

1283 number generator (`uint32`) 

1284 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1285 A ``TransmissionCurve`` that represents the throughput of the optics, 

1286 to be evaluated in focal-plane coordinates. 

1287 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1288 A ``TransmissionCurve`` that represents the throughput of the filter 

1289 itself, to be evaluated in focal-plane coordinates. 

1290 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1291 A ``TransmissionCurve`` that represents the throughput of the sensor 

1292 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1293 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1294 A ``TransmissionCurve`` that represents the throughput of the 

1295 atmosphere, assumed to be spatially constant. 

1296 detectorNum : `int`, optional 

1297 The integer number for the detector to process. 

1298 isGen3 : bool, optional 

1299 Flag this call to run() as using the Gen3 butler environment. 

1300 strayLightData : `object`, optional 

1301 Opaque object containing calibration information for stray-light 

1302 correction. If `None`, no correction will be performed. 

1303 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1304 Illumination correction image. 

1305 

1306 Returns 

1307 ------- 

1308 result : `lsst.pipe.base.Struct` 

1309 Result struct with component: 

1310 - ``exposure`` : `afw.image.Exposure` 

1311 The fully ISR corrected exposure. 

1312 - ``outputExposure`` : `afw.image.Exposure` 

1313 An alias for `exposure` 

1314 - ``ossThumb`` : `numpy.ndarray` 

1315 Thumbnail image of the exposure after overscan subtraction. 

1316 - ``flattenedThumb`` : `numpy.ndarray` 

1317 Thumbnail image of the exposure after flat-field correction. 

1318 

1319 Raises 

1320 ------ 

1321 RuntimeError 

1322 Raised if a configuration option is set to True, but the 

1323 required calibration data has not been specified. 

1324 

1325 Notes 

1326 ----- 

1327 The current processed exposure can be viewed by setting the 

1328 appropriate lsstDebug entries in the `debug.display` 

1329 dictionary. The names of these entries correspond to some of 

1330 the IsrTaskConfig Boolean options, with the value denoting the 

1331 frame to use. The exposure is shown inside the matching 

1332 option check and after the processing of that step has 

1333 finished. The steps with debug points are: 

1334 

1335 doAssembleCcd 

1336 doBias 

1337 doCrosstalk 

1338 doBrighterFatter 

1339 doDark 

1340 doFringe 

1341 doStrayLight 

1342 doFlat 

1343 

1344 In addition, setting the "postISRCCD" entry displays the 

1345 exposure after all ISR processing has finished. 

1346 

1347 """ 

1348 

1349 if isGen3 is True: 

1350 # Gen3 currently cannot automatically do configuration overrides. 

1351 # DM-15257 looks to discuss this issue. 

1352 # Configure input exposures; 

1353 if detectorNum is None: 

1354 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1355 

1356 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1357 bias = self.ensureExposure(bias, camera, detectorNum) 

1358 dark = self.ensureExposure(dark, camera, detectorNum) 

1359 flat = self.ensureExposure(flat, camera, detectorNum) 

1360 else: 

1361 if isinstance(ccdExposure, ButlerDataRef): 

1362 return self.runDataRef(ccdExposure) 

1363 

1364 ccd = ccdExposure.getDetector() 

1365 filterLabel = ccdExposure.getFilterLabel() 

1366 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1367 

1368 if not ccd: 

1369 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1370 ccd = [FakeAmp(ccdExposure, self.config)] 

1371 

1372 # Validate Input 

1373 if self.config.doBias and bias is None: 

1374 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1375 if self.doLinearize(ccd) and linearizer is None: 

1376 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1377 if self.config.doBrighterFatter and bfKernel is None: 

1378 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1379 if self.config.doDark and dark is None: 

1380 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1381 if self.config.doFlat and flat is None: 

1382 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1383 if self.config.doDefect and defects is None: 

1384 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1385 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1386 and fringes.fringes is None): 

1387 # The `fringes` object needs to be a pipeBase.Struct, as 

1388 # we use it as a `dict` for the parameters of 

1389 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1390 # not be `None` if `doFringe=True`. Otherwise, raise. 

1391 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1392 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1393 and illumMaskedImage is None): 

1394 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1395 

1396 # Begin ISR processing. 

1397 if self.config.doConvertIntToFloat: 

1398 self.log.info("Converting exposure to floating point values.") 

1399 ccdExposure = self.convertIntToFloat(ccdExposure) 

1400 

1401 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1402 self.log.info("Applying bias correction.") 

1403 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1404 trimToFit=self.config.doTrimToMatchCalib) 

1405 self.debugView(ccdExposure, "doBias") 

1406 

1407 # Amplifier level processing. 

1408 overscans = [] 

1409 for amp in ccd: 

1410 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1411 if ccdExposure.getBBox().contains(amp.getBBox()): 

1412 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1413 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1414 

1415 if self.config.doOverscan and not badAmp: 

1416 # Overscan correction on amp-by-amp basis. 

1417 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1418 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1419 if overscanResults is not None and \ 

1420 self.config.qa is not None and self.config.qa.saveStats is True: 

1421 if isinstance(overscanResults.overscanFit, float): 

1422 qaMedian = overscanResults.overscanFit 

1423 qaStdev = float("NaN") 

1424 else: 

1425 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1426 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1427 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1428 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1429 

1430 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian) 

1431 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev) 

1432 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1433 amp.getName(), qaMedian, qaStdev) 

1434 

1435 # Residuals after overscan correction 

1436 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1437 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1438 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1439 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1440 

1441 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter) 

1442 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter) 

1443 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1444 amp.getName(), qaMedianAfter, qaStdevAfter) 

1445 

1446 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1447 else: 

1448 if badAmp: 

1449 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1450 overscanResults = None 

1451 

1452 overscans.append(overscanResults if overscanResults is not None else None) 

1453 else: 

1454 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1455 

1456 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1457 self.log.info("Applying crosstalk correction.") 

1458 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1459 crosstalkSources=crosstalkSources, camera=camera) 

1460 self.debugView(ccdExposure, "doCrosstalk") 

1461 

1462 if self.config.doAssembleCcd: 

1463 self.log.info("Assembling CCD from amplifiers.") 

1464 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1465 

1466 if self.config.expectWcs and not ccdExposure.getWcs(): 

1467 self.log.warn("No WCS found in input exposure.") 

1468 self.debugView(ccdExposure, "doAssembleCcd") 

1469 

1470 ossThumb = None 

1471 if self.config.qa.doThumbnailOss: 

1472 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1473 

1474 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1475 self.log.info("Applying bias correction.") 

1476 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1477 trimToFit=self.config.doTrimToMatchCalib) 

1478 self.debugView(ccdExposure, "doBias") 

1479 

1480 if self.config.doVariance: 

1481 for amp, overscanResults in zip(ccd, overscans): 

1482 if ccdExposure.getBBox().contains(amp.getBBox()): 

1483 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1484 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1485 if overscanResults is not None: 

1486 self.updateVariance(ampExposure, amp, 

1487 overscanImage=overscanResults.overscanImage, 

1488 ptcDataset=ptc) 

1489 else: 

1490 self.updateVariance(ampExposure, amp, 

1491 overscanImage=None, 

1492 ptcDataset=ptc) 

1493 if self.config.qa is not None and self.config.qa.saveStats is True: 

1494 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1495 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1496 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1497 qaStats.getValue(afwMath.MEDIAN)) 

1498 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1499 qaStats.getValue(afwMath.STDEVCLIP)) 

1500 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1501 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1502 qaStats.getValue(afwMath.STDEVCLIP)) 

1503 

1504 if self.doLinearize(ccd): 

1505 self.log.info("Applying linearizer.") 

1506 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1507 detector=ccd, log=self.log) 

1508 

1509 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1510 self.log.info("Applying crosstalk correction.") 

1511 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1512 crosstalkSources=crosstalkSources, isTrimmed=True) 

1513 self.debugView(ccdExposure, "doCrosstalk") 

1514 

1515 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do 

1516 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1517 if self.config.doDefect: 

1518 self.log.info("Masking defects.") 

1519 self.maskDefect(ccdExposure, defects) 

1520 

1521 if self.config.numEdgeSuspect > 0: 

1522 self.log.info("Masking edges as SUSPECT.") 

1523 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1524 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1525 

1526 if self.config.doNanMasking: 

1527 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1528 self.maskNan(ccdExposure) 

1529 

1530 if self.config.doWidenSaturationTrails: 

1531 self.log.info("Widening saturation trails.") 

1532 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1533 

1534 if self.config.doCameraSpecificMasking: 

1535 self.log.info("Masking regions for camera specific reasons.") 

1536 self.masking.run(ccdExposure) 

1537 

1538 if self.config.doBrighterFatter: 

1539 # We need to apply flats and darks before we can interpolate, and we 

1540 # need to interpolate before we do B-F, but we do B-F without the 

1541 # flats and darks applied so we can work in units of electrons or holes. 

1542 # This context manager applies and then removes the darks and flats. 

1543 # 

1544 # We also do not want to interpolate values here, so operate on temporary 

1545 # images so we can apply only the BF-correction and roll back the 

1546 # interpolation. 

1547 interpExp = ccdExposure.clone() 

1548 with self.flatContext(interpExp, flat, dark): 

1549 isrFunctions.interpolateFromMask( 

1550 maskedImage=interpExp.getMaskedImage(), 

1551 fwhm=self.config.fwhm, 

1552 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1553 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1554 ) 

1555 bfExp = interpExp.clone() 

1556 

1557 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1558 type(bfKernel), type(bfGains)) 

1559 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1560 self.config.brighterFatterMaxIter, 

1561 self.config.brighterFatterThreshold, 

1562 self.config.brighterFatterApplyGain, 

1563 bfGains) 

1564 if bfResults[1] == self.config.brighterFatterMaxIter: 

1565 self.log.warn("Brighter-fatter correction did not converge, final difference %f.", 

1566 bfResults[0]) 

1567 else: 

1568 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1569 bfResults[1]) 

1570 image = ccdExposure.getMaskedImage().getImage() 

1571 bfCorr = bfExp.getMaskedImage().getImage() 

1572 bfCorr -= interpExp.getMaskedImage().getImage() 

1573 image += bfCorr 

1574 

1575 # Applying the brighter-fatter correction applies a 

1576 # convolution to the science image. At the edges this 

1577 # convolution may not have sufficient valid pixels to 

1578 # produce a valid correction. Mark pixels within the size 

1579 # of the brighter-fatter kernel as EDGE to warn of this 

1580 # fact. 

1581 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1582 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1583 maskPlane="EDGE") 

1584 

1585 if self.config.brighterFatterMaskGrowSize > 0: 

1586 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1587 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1588 isrFunctions.growMasks(ccdExposure.getMask(), 

1589 radius=self.config.brighterFatterMaskGrowSize, 

1590 maskNameList=maskPlane, 

1591 maskValue=maskPlane) 

1592 

1593 self.debugView(ccdExposure, "doBrighterFatter") 

1594 

1595 if self.config.doDark: 

1596 self.log.info("Applying dark correction.") 

1597 self.darkCorrection(ccdExposure, dark) 

1598 self.debugView(ccdExposure, "doDark") 

1599 

1600 if self.config.doFringe and not self.config.fringeAfterFlat: 

1601 self.log.info("Applying fringe correction before flat.") 

1602 self.fringe.run(ccdExposure, **fringes.getDict()) 

1603 self.debugView(ccdExposure, "doFringe") 

1604 

1605 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1606 self.log.info("Checking strayLight correction.") 

1607 self.strayLight.run(ccdExposure, strayLightData) 

1608 self.debugView(ccdExposure, "doStrayLight") 

1609 

1610 if self.config.doFlat: 

1611 self.log.info("Applying flat correction.") 

1612 self.flatCorrection(ccdExposure, flat) 

1613 self.debugView(ccdExposure, "doFlat") 

1614 

1615 if self.config.doApplyGains: 

1616 self.log.info("Applying gain correction instead of flat.") 

1617 if self.config.usePtcGains: 

1618 self.log.info("Using gains from the Photon Transfer Curve.") 

1619 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1620 ptcGains=ptc.gain) 

1621 else: 

1622 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1623 

1624 if self.config.doFringe and self.config.fringeAfterFlat: 

1625 self.log.info("Applying fringe correction after flat.") 

1626 self.fringe.run(ccdExposure, **fringes.getDict()) 

1627 

1628 if self.config.doVignette: 

1629 self.log.info("Constructing Vignette polygon.") 

1630 self.vignettePolygon = self.vignette.run(ccdExposure) 

1631 

1632 if self.config.vignette.doWriteVignettePolygon: 

1633 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1634 

1635 if self.config.doAttachTransmissionCurve: 

1636 self.log.info("Adding transmission curves.") 

1637 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1638 filterTransmission=filterTransmission, 

1639 sensorTransmission=sensorTransmission, 

1640 atmosphereTransmission=atmosphereTransmission) 

1641 

1642 flattenedThumb = None 

1643 if self.config.qa.doThumbnailFlattened: 

1644 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1645 

1646 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1647 self.log.info("Performing illumination correction.") 

1648 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1649 illumMaskedImage, illumScale=self.config.illumScale, 

1650 trimToFit=self.config.doTrimToMatchCalib) 

1651 

1652 preInterpExp = None 

1653 if self.config.doSaveInterpPixels: 

1654 preInterpExp = ccdExposure.clone() 

1655 

1656 # Reset and interpolate bad pixels. 

1657 # 

1658 # Large contiguous bad regions (which should have the BAD mask 

1659 # bit set) should have their values set to the image median. 

1660 # This group should include defects and bad amplifiers. As the 

1661 # area covered by these defects are large, there's little 

1662 # reason to expect that interpolation would provide a more 

1663 # useful value. 

1664 # 

1665 # Smaller defects can be safely interpolated after the larger 

1666 # regions have had their pixel values reset. This ensures 

1667 # that the remaining defects adjacent to bad amplifiers (as an 

1668 # example) do not attempt to interpolate extreme values. 

1669 if self.config.doSetBadRegions: 

1670 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1671 if badPixelCount > 0: 

1672 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1673 

1674 if self.config.doInterpolate: 

1675 self.log.info("Interpolating masked pixels.") 

1676 isrFunctions.interpolateFromMask( 

1677 maskedImage=ccdExposure.getMaskedImage(), 

1678 fwhm=self.config.fwhm, 

1679 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1680 maskNameList=list(self.config.maskListToInterpolate) 

1681 ) 

1682 

1683 self.roughZeroPoint(ccdExposure) 

1684 

1685 if self.config.doMeasureBackground: 

1686 self.log.info("Measuring background level.") 

1687 self.measureBackground(ccdExposure, self.config.qa) 

1688 

1689 if self.config.qa is not None and self.config.qa.saveStats is True: 

1690 for amp in ccd: 

1691 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1692 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1693 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1694 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1695 qaStats.getValue(afwMath.MEDIAN)) 

1696 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1697 qaStats.getValue(afwMath.STDEVCLIP)) 

1698 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1699 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1700 qaStats.getValue(afwMath.STDEVCLIP)) 

1701 

1702 self.debugView(ccdExposure, "postISRCCD") 

1703 

1704 return pipeBase.Struct( 

1705 exposure=ccdExposure, 

1706 ossThumb=ossThumb, 

1707 flattenedThumb=flattenedThumb, 

1708 

1709 preInterpolatedExposure=preInterpExp, 

1710 outputExposure=ccdExposure, 

1711 outputOssThumbnail=ossThumb, 

1712 outputFlattenedThumbnail=flattenedThumb, 

1713 ) 

1714 

1715 @pipeBase.timeMethod 

1716 def runDataRef(self, sensorRef): 

1717 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1718 

1719 This method contains the `CmdLineTask` interface to the ISR 

1720 processing. All IO is handled here, freeing the `run()` method 

1721 to manage only pixel-level calculations. The steps performed 

1722 are: 

1723 - Read in necessary detrending/isr/calibration data. 

1724 - Process raw exposure in `run()`. 

1725 - Persist the ISR-corrected exposure as "postISRCCD" if 

1726 config.doWrite=True. 

1727 

1728 Parameters 

1729 ---------- 

1730 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1731 DataRef of the detector data to be processed 

1732 

1733 Returns 

1734 ------- 

1735 result : `lsst.pipe.base.Struct` 

1736 Result struct with component: 

1737 - ``exposure`` : `afw.image.Exposure` 

1738 The fully ISR corrected exposure. 

1739 

1740 Raises 

1741 ------ 

1742 RuntimeError 

1743 Raised if a configuration option is set to True, but the 

1744 required calibration data does not exist. 

1745 

1746 """ 

1747 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1748 

1749 ccdExposure = sensorRef.get(self.config.datasetType) 

1750 

1751 camera = sensorRef.get("camera") 

1752 isrData = self.readIsrData(sensorRef, ccdExposure) 

1753 

1754 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1755 

1756 if self.config.doWrite: 

1757 sensorRef.put(result.exposure, "postISRCCD") 

1758 if result.preInterpolatedExposure is not None: 

1759 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1760 if result.ossThumb is not None: 

1761 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1762 if result.flattenedThumb is not None: 

1763 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1764 

1765 return result 

1766 

1767 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1768 """Retrieve a calibration dataset for removing instrument signature. 

1769 

1770 Parameters 

1771 ---------- 

1772 

1773 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1774 DataRef of the detector data to find calibration datasets 

1775 for. 

1776 datasetType : `str` 

1777 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1778 dateObs : `str`, optional 

1779 Date of the observation. Used to correct butler failures 

1780 when using fallback filters. 

1781 immediate : `Bool` 

1782 If True, disable butler proxies to enable error handling 

1783 within this routine. 

1784 

1785 Returns 

1786 ------- 

1787 exposure : `lsst.afw.image.Exposure` 

1788 Requested calibration frame. 

1789 

1790 Raises 

1791 ------ 

1792 RuntimeError 

1793 Raised if no matching calibration frame can be found. 

1794 """ 

1795 try: 

1796 exp = dataRef.get(datasetType, immediate=immediate) 

1797 except Exception as exc1: 

1798 if not self.config.fallbackFilterName: 

1799 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1800 try: 

1801 if self.config.useFallbackDate and dateObs: 

1802 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1803 dateObs=dateObs, immediate=immediate) 

1804 else: 

1805 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1806 except Exception as exc2: 

1807 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1808 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1809 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1810 

1811 if self.config.doAssembleIsrExposures: 

1812 exp = self.assembleCcd.assembleCcd(exp) 

1813 return exp 

1814 

1815 def ensureExposure(self, inputExp, camera, detectorNum): 

1816 """Ensure that the data returned by Butler is a fully constructed exposure. 

1817 

1818 ISR requires exposure-level image data for historical reasons, so if we did 

1819 not recieve that from Butler, construct it from what we have, modifying the 

1820 input in place. 

1821 

1822 Parameters 

1823 ---------- 

1824 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1825 `lsst.afw.image.ImageF` 

1826 The input data structure obtained from Butler. 

1827 camera : `lsst.afw.cameraGeom.camera` 

1828 The camera associated with the image. Used to find the appropriate 

1829 detector. 

1830 detectorNum : `int` 

1831 The detector this exposure should match. 

1832 

1833 Returns 

1834 ------- 

1835 inputExp : `lsst.afw.image.Exposure` 

1836 The re-constructed exposure, with appropriate detector parameters. 

1837 

1838 Raises 

1839 ------ 

1840 TypeError 

1841 Raised if the input data cannot be used to construct an exposure. 

1842 """ 

1843 if isinstance(inputExp, afwImage.DecoratedImageU): 

1844 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1845 elif isinstance(inputExp, afwImage.ImageF): 

1846 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1847 elif isinstance(inputExp, afwImage.MaskedImageF): 

1848 inputExp = afwImage.makeExposure(inputExp) 

1849 elif isinstance(inputExp, afwImage.Exposure): 

1850 pass 

1851 elif inputExp is None: 

1852 # Assume this will be caught by the setup if it is a problem. 

1853 return inputExp 

1854 else: 

1855 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1856 (type(inputExp), )) 

1857 

1858 if inputExp.getDetector() is None: 

1859 inputExp.setDetector(camera[detectorNum]) 

1860 

1861 return inputExp 

1862 

1863 def convertIntToFloat(self, exposure): 

1864 """Convert exposure image from uint16 to float. 

1865 

1866 If the exposure does not need to be converted, the input is 

1867 immediately returned. For exposures that are converted to use 

1868 floating point pixels, the variance is set to unity and the 

1869 mask to zero. 

1870 

1871 Parameters 

1872 ---------- 

1873 exposure : `lsst.afw.image.Exposure` 

1874 The raw exposure to be converted. 

1875 

1876 Returns 

1877 ------- 

1878 newexposure : `lsst.afw.image.Exposure` 

1879 The input ``exposure``, converted to floating point pixels. 

1880 

1881 Raises 

1882 ------ 

1883 RuntimeError 

1884 Raised if the exposure type cannot be converted to float. 

1885 

1886 """ 

1887 if isinstance(exposure, afwImage.ExposureF): 

1888 # Nothing to be done 

1889 self.log.debug("Exposure already of type float.") 

1890 return exposure 

1891 if not hasattr(exposure, "convertF"): 

1892 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1893 

1894 newexposure = exposure.convertF() 

1895 newexposure.variance[:] = 1 

1896 newexposure.mask[:] = 0x0 

1897 

1898 return newexposure 

1899 

1900 def maskAmplifier(self, ccdExposure, amp, defects): 

1901 """Identify bad amplifiers, saturated and suspect pixels. 

1902 

1903 Parameters 

1904 ---------- 

1905 ccdExposure : `lsst.afw.image.Exposure` 

1906 Input exposure to be masked. 

1907 amp : `lsst.afw.table.AmpInfoCatalog` 

1908 Catalog of parameters defining the amplifier on this 

1909 exposure to mask. 

1910 defects : `lsst.ip.isr.Defects` 

1911 List of defects. Used to determine if the entire 

1912 amplifier is bad. 

1913 

1914 Returns 

1915 ------- 

1916 badAmp : `Bool` 

1917 If this is true, the entire amplifier area is covered by 

1918 defects and unusable. 

1919 

1920 """ 

1921 maskedImage = ccdExposure.getMaskedImage() 

1922 

1923 badAmp = False 

1924 

1925 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1926 # comparison with current defects definition. 

1927 if defects is not None: 

1928 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1929 

1930 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1931 # association with pixels in current ccdExposure). 

1932 if badAmp: 

1933 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1934 afwImage.PARENT) 

1935 maskView = dataView.getMask() 

1936 maskView |= maskView.getPlaneBitMask("BAD") 

1937 del maskView 

1938 return badAmp 

1939 

1940 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1941 # Saturation and suspect pixels can be masked now, though. 

1942 limits = dict() 

1943 if self.config.doSaturation and not badAmp: 

1944 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1945 if self.config.doSuspect and not badAmp: 

1946 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1947 if math.isfinite(self.config.saturation): 

1948 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1949 

1950 for maskName, maskThreshold in limits.items(): 

1951 if not math.isnan(maskThreshold): 

1952 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1953 isrFunctions.makeThresholdMask( 

1954 maskedImage=dataView, 

1955 threshold=maskThreshold, 

1956 growFootprints=0, 

1957 maskName=maskName 

1958 ) 

1959 

1960 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1961 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1962 afwImage.PARENT) 

1963 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1964 self.config.suspectMaskName]) 

1965 if numpy.all(maskView.getArray() & maskVal > 0): 

1966 badAmp = True 

1967 maskView |= maskView.getPlaneBitMask("BAD") 

1968 

1969 return badAmp 

1970 

1971 def overscanCorrection(self, ccdExposure, amp): 

1972 """Apply overscan correction in place. 

1973 

1974 This method does initial pixel rejection of the overscan 

1975 region. The overscan can also be optionally segmented to 

1976 allow for discontinuous overscan responses to be fit 

1977 separately. The actual overscan subtraction is performed by 

1978 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1979 which is called here after the amplifier is preprocessed. 

1980 

1981 Parameters 

1982 ---------- 

1983 ccdExposure : `lsst.afw.image.Exposure` 

1984 Exposure to have overscan correction performed. 

1985 amp : `lsst.afw.cameraGeom.Amplifer` 

1986 The amplifier to consider while correcting the overscan. 

1987 

1988 Returns 

1989 ------- 

1990 overscanResults : `lsst.pipe.base.Struct` 

1991 Result struct with components: 

1992 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1993 Value or fit subtracted from the amplifier image data. 

1994 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1995 Value or fit subtracted from the overscan image data. 

1996 - ``overscanImage`` : `lsst.afw.image.Image` 

1997 Image of the overscan region with the overscan 

1998 correction applied. This quantity is used to estimate 

1999 the amplifier read noise empirically. 

2000 

2001 Raises 

2002 ------ 

2003 RuntimeError 

2004 Raised if the ``amp`` does not contain raw pixel information. 

2005 

2006 See Also 

2007 -------- 

2008 lsst.ip.isr.isrFunctions.overscanCorrection 

2009 """ 

2010 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2011 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2012 return None 

2013 

2014 statControl = afwMath.StatisticsControl() 

2015 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2016 

2017 # Determine the bounding boxes 

2018 dataBBox = amp.getRawDataBBox() 

2019 oscanBBox = amp.getRawHorizontalOverscanBBox() 

2020 dx0 = 0 

2021 dx1 = 0 

2022 

2023 prescanBBox = amp.getRawPrescanBBox() 

2024 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

2025 dx0 += self.config.overscanNumLeadingColumnsToSkip 

2026 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

2027 else: 

2028 dx0 += self.config.overscanNumTrailingColumnsToSkip 

2029 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

2030 

2031 # Determine if we need to work on subregions of the amplifier and overscan. 

2032 imageBBoxes = [] 

2033 overscanBBoxes = [] 

2034 

2035 if ((self.config.overscanBiasJump 

2036 and self.config.overscanBiasJumpLocation) 

2037 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

2038 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

2039 self.config.overscanBiasJumpDevices)): 

2040 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

2041 yLower = self.config.overscanBiasJumpLocation 

2042 yUpper = dataBBox.getHeight() - yLower 

2043 else: 

2044 yUpper = self.config.overscanBiasJumpLocation 

2045 yLower = dataBBox.getHeight() - yUpper 

2046 

2047 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2048 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

2049 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2050 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2051 yLower))) 

2052 

2053 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

2054 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

2055 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

2056 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2057 yUpper))) 

2058 else: 

2059 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2060 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

2061 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2062 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2063 oscanBBox.getHeight()))) 

2064 

2065 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

2066 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

2067 ampImage = ccdExposure.maskedImage[imageBBox] 

2068 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2069 

2070 overscanArray = overscanImage.image.array 

2071 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2072 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2073 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2074 

2075 statControl = afwMath.StatisticsControl() 

2076 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2077 

2078 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2079 

2080 # Measure average overscan levels and record them in the metadata. 

2081 levelStat = afwMath.MEDIAN 

2082 sigmaStat = afwMath.STDEVCLIP 

2083 

2084 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2085 self.config.qa.flatness.nIter) 

2086 metadata = ccdExposure.getMetadata() 

2087 ampNum = amp.getName() 

2088 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2089 if isinstance(overscanResults.overscanFit, float): 

2090 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2091 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2092 else: 

2093 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2094 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2095 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2096 

2097 return overscanResults 

2098 

2099 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2100 """Set the variance plane using the gain and read noise 

2101 

2102 The read noise is calculated from the ``overscanImage`` if the 

2103 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2104 the value from the amplifier data is used. 

2105 

2106 Parameters 

2107 ---------- 

2108 ampExposure : `lsst.afw.image.Exposure` 

2109 Exposure to process. 

2110 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2111 Amplifier detector data. 

2112 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2113 Image of overscan, required only for empirical read noise. 

2114 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2115 PTC dataset containing the gains and read noise. 

2116 

2117 

2118 Raises 

2119 ------ 

2120 RuntimeError 

2121 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2122 are ``True``, but ptcDataset is not provided. 

2123 

2124 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2125 ``overscanImage`` is ``None``. 

2126 

2127 See also 

2128 -------- 

2129 lsst.ip.isr.isrFunctions.updateVariance 

2130 """ 

2131 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2132 if self.config.usePtcGains: 

2133 if ptcDataset is None: 

2134 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2135 else: 

2136 gain = ptcDataset.gain[amp.getName()] 

2137 self.log.info("Using gain from Photon Transfer Curve.") 

2138 else: 

2139 gain = amp.getGain() 

2140 

2141 if math.isnan(gain): 

2142 gain = 1.0 

2143 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2144 elif gain <= 0: 

2145 patchedGain = 1.0 

2146 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

2147 amp.getName(), gain, patchedGain) 

2148 gain = patchedGain 

2149 

2150 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2151 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2152 

2153 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2154 stats = afwMath.StatisticsControl() 

2155 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2156 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2157 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2158 amp.getName(), readNoise) 

2159 elif self.config.usePtcReadNoise: 

2160 if ptcDataset is None: 

2161 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2162 else: 

2163 readNoise = ptcDataset.noise[amp.getName()] 

2164 self.log.info("Using read noise from Photon Transfer Curve.") 

2165 else: 

2166 readNoise = amp.getReadNoise() 

2167 

2168 isrFunctions.updateVariance( 

2169 maskedImage=ampExposure.getMaskedImage(), 

2170 gain=gain, 

2171 readNoise=readNoise, 

2172 ) 

2173 

2174 def darkCorrection(self, exposure, darkExposure, invert=False): 

2175 """Apply dark correction in place. 

2176 

2177 Parameters 

2178 ---------- 

2179 exposure : `lsst.afw.image.Exposure` 

2180 Exposure to process. 

2181 darkExposure : `lsst.afw.image.Exposure` 

2182 Dark exposure of the same size as ``exposure``. 

2183 invert : `Bool`, optional 

2184 If True, re-add the dark to an already corrected image. 

2185 

2186 Raises 

2187 ------ 

2188 RuntimeError 

2189 Raised if either ``exposure`` or ``darkExposure`` do not 

2190 have their dark time defined. 

2191 

2192 See Also 

2193 -------- 

2194 lsst.ip.isr.isrFunctions.darkCorrection 

2195 """ 

2196 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2197 if math.isnan(expScale): 

2198 raise RuntimeError("Exposure darktime is NAN.") 

2199 if darkExposure.getInfo().getVisitInfo() is not None \ 

2200 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2201 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2202 else: 

2203 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2204 # so getDarkTime() does not exist. 

2205 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2206 darkScale = 1.0 

2207 

2208 isrFunctions.darkCorrection( 

2209 maskedImage=exposure.getMaskedImage(), 

2210 darkMaskedImage=darkExposure.getMaskedImage(), 

2211 expScale=expScale, 

2212 darkScale=darkScale, 

2213 invert=invert, 

2214 trimToFit=self.config.doTrimToMatchCalib 

2215 ) 

2216 

2217 def doLinearize(self, detector): 

2218 """Check if linearization is needed for the detector cameraGeom. 

2219 

2220 Checks config.doLinearize and the linearity type of the first 

2221 amplifier. 

2222 

2223 Parameters 

2224 ---------- 

2225 detector : `lsst.afw.cameraGeom.Detector` 

2226 Detector to get linearity type from. 

2227 

2228 Returns 

2229 ------- 

2230 doLinearize : `Bool` 

2231 If True, linearization should be performed. 

2232 """ 

2233 return self.config.doLinearize and \ 

2234 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2235 

2236 def flatCorrection(self, exposure, flatExposure, invert=False): 

2237 """Apply flat correction in place. 

2238 

2239 Parameters 

2240 ---------- 

2241 exposure : `lsst.afw.image.Exposure` 

2242 Exposure to process. 

2243 flatExposure : `lsst.afw.image.Exposure` 

2244 Flat exposure of the same size as ``exposure``. 

2245 invert : `Bool`, optional 

2246 If True, unflatten an already flattened image. 

2247 

2248 See Also 

2249 -------- 

2250 lsst.ip.isr.isrFunctions.flatCorrection 

2251 """ 

2252 isrFunctions.flatCorrection( 

2253 maskedImage=exposure.getMaskedImage(), 

2254 flatMaskedImage=flatExposure.getMaskedImage(), 

2255 scalingType=self.config.flatScalingType, 

2256 userScale=self.config.flatUserScale, 

2257 invert=invert, 

2258 trimToFit=self.config.doTrimToMatchCalib 

2259 ) 

2260 

2261 def saturationDetection(self, exposure, amp): 

2262 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2263 

2264 Parameters 

2265 ---------- 

2266 exposure : `lsst.afw.image.Exposure` 

2267 Exposure to process. Only the amplifier DataSec is processed. 

2268 amp : `lsst.afw.table.AmpInfoCatalog` 

2269 Amplifier detector data. 

2270 

2271 See Also 

2272 -------- 

2273 lsst.ip.isr.isrFunctions.makeThresholdMask 

2274 """ 

2275 if not math.isnan(amp.getSaturation()): 

2276 maskedImage = exposure.getMaskedImage() 

2277 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2278 isrFunctions.makeThresholdMask( 

2279 maskedImage=dataView, 

2280 threshold=amp.getSaturation(), 

2281 growFootprints=0, 

2282 maskName=self.config.saturatedMaskName, 

2283 ) 

2284 

2285 def saturationInterpolation(self, exposure): 

2286 """Interpolate over saturated pixels, in place. 

2287 

2288 This method should be called after `saturationDetection`, to 

2289 ensure that the saturated pixels have been identified in the 

2290 SAT mask. It should also be called after `assembleCcd`, since 

2291 saturated regions may cross amplifier boundaries. 

2292 

2293 Parameters 

2294 ---------- 

2295 exposure : `lsst.afw.image.Exposure` 

2296 Exposure to process. 

2297 

2298 See Also 

2299 -------- 

2300 lsst.ip.isr.isrTask.saturationDetection 

2301 lsst.ip.isr.isrFunctions.interpolateFromMask 

2302 """ 

2303 isrFunctions.interpolateFromMask( 

2304 maskedImage=exposure.getMaskedImage(), 

2305 fwhm=self.config.fwhm, 

2306 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2307 maskNameList=list(self.config.saturatedMaskName), 

2308 ) 

2309 

2310 def suspectDetection(self, exposure, amp): 

2311 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2312 

2313 Parameters 

2314 ---------- 

2315 exposure : `lsst.afw.image.Exposure` 

2316 Exposure to process. Only the amplifier DataSec is processed. 

2317 amp : `lsst.afw.table.AmpInfoCatalog` 

2318 Amplifier detector data. 

2319 

2320 See Also 

2321 -------- 

2322 lsst.ip.isr.isrFunctions.makeThresholdMask 

2323 

2324 Notes 

2325 ----- 

2326 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2327 This is intended to indicate pixels that may be affected by unknown systematics; 

2328 for example if non-linearity corrections above a certain level are unstable 

2329 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2330 that no such level exists and no pixels are to be masked as suspicious. 

2331 """ 

2332 suspectLevel = amp.getSuspectLevel() 

2333 if math.isnan(suspectLevel): 

2334 return 

2335 

2336 maskedImage = exposure.getMaskedImage() 

2337 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2338 isrFunctions.makeThresholdMask( 

2339 maskedImage=dataView, 

2340 threshold=suspectLevel, 

2341 growFootprints=0, 

2342 maskName=self.config.suspectMaskName, 

2343 ) 

2344 

2345 def maskDefect(self, exposure, defectBaseList): 

2346 """Mask defects using mask plane "BAD", in place. 

2347 

2348 Parameters 

2349 ---------- 

2350 exposure : `lsst.afw.image.Exposure` 

2351 Exposure to process. 

2352 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2353 `lsst.afw.image.DefectBase`. 

2354 List of defects to mask. 

2355 

2356 Notes 

2357 ----- 

2358 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2359 """ 

2360 maskedImage = exposure.getMaskedImage() 

2361 if not isinstance(defectBaseList, Defects): 

2362 # Promotes DefectBase to Defect 

2363 defectList = Defects(defectBaseList) 

2364 else: 

2365 defectList = defectBaseList 

2366 defectList.maskPixels(maskedImage, maskName="BAD") 

2367 

2368 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2369 """Mask edge pixels with applicable mask plane. 

2370 

2371 Parameters 

2372 ---------- 

2373 exposure : `lsst.afw.image.Exposure` 

2374 Exposure to process. 

2375 numEdgePixels : `int`, optional 

2376 Number of edge pixels to mask. 

2377 maskPlane : `str`, optional 

2378 Mask plane name to use. 

2379 level : `str`, optional 

2380 Level at which to mask edges. 

2381 """ 

2382 maskedImage = exposure.getMaskedImage() 

2383 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2384 

2385 if numEdgePixels > 0: 

2386 if level == 'DETECTOR': 

2387 boxes = [maskedImage.getBBox()] 

2388 elif level == 'AMP': 

2389 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2390 

2391 for box in boxes: 

2392 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2393 subImage = maskedImage[box] 

2394 box.grow(-numEdgePixels) 

2395 # Mask pixels outside box 

2396 SourceDetectionTask.setEdgeBits( 

2397 subImage, 

2398 box, 

2399 maskBitMask) 

2400 

2401 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2402 """Mask and interpolate defects using mask plane "BAD", in place. 

2403 

2404 Parameters 

2405 ---------- 

2406 exposure : `lsst.afw.image.Exposure` 

2407 Exposure to process. 

2408 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2409 `lsst.afw.image.DefectBase`. 

2410 List of defects to mask and interpolate. 

2411 

2412 See Also 

2413 -------- 

2414 lsst.ip.isr.isrTask.maskDefect 

2415 """ 

2416 self.maskDefect(exposure, defectBaseList) 

2417 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2418 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2419 isrFunctions.interpolateFromMask( 

2420 maskedImage=exposure.getMaskedImage(), 

2421 fwhm=self.config.fwhm, 

2422 growSaturatedFootprints=0, 

2423 maskNameList=["BAD"], 

2424 ) 

2425 

2426 def maskNan(self, exposure): 

2427 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2428 

2429 Parameters 

2430 ---------- 

2431 exposure : `lsst.afw.image.Exposure` 

2432 Exposure to process. 

2433 

2434 Notes 

2435 ----- 

2436 We mask over all non-finite values (NaN, inf), including those 

2437 that are masked with other bits (because those may or may not be 

2438 interpolated over later, and we want to remove all NaN/infs). 

2439 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2440 preserve the historical name. 

2441 """ 

2442 maskedImage = exposure.getMaskedImage() 

2443 

2444 # Find and mask NaNs 

2445 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2446 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2447 numNans = maskNans(maskedImage, maskVal) 

2448 self.metadata.set("NUMNANS", numNans) 

2449 if numNans > 0: 

2450 self.log.warn("There were %d unmasked NaNs.", numNans) 

2451 

2452 def maskAndInterpolateNan(self, exposure): 

2453 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2454 in place. 

2455 

2456 Parameters 

2457 ---------- 

2458 exposure : `lsst.afw.image.Exposure` 

2459 Exposure to process. 

2460 

2461 See Also 

2462 -------- 

2463 lsst.ip.isr.isrTask.maskNan 

2464 """ 

2465 self.maskNan(exposure) 

2466 isrFunctions.interpolateFromMask( 

2467 maskedImage=exposure.getMaskedImage(), 

2468 fwhm=self.config.fwhm, 

2469 growSaturatedFootprints=0, 

2470 maskNameList=["UNMASKEDNAN"], 

2471 ) 

2472 

2473 def measureBackground(self, exposure, IsrQaConfig=None): 

2474 """Measure the image background in subgrids, for quality control purposes. 

2475 

2476 Parameters 

2477 ---------- 

2478 exposure : `lsst.afw.image.Exposure` 

2479 Exposure to process. 

2480 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2481 Configuration object containing parameters on which background 

2482 statistics and subgrids to use. 

2483 """ 

2484 if IsrQaConfig is not None: 

2485 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2486 IsrQaConfig.flatness.nIter) 

2487 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2488 statsControl.setAndMask(maskVal) 

2489 maskedImage = exposure.getMaskedImage() 

2490 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2491 skyLevel = stats.getValue(afwMath.MEDIAN) 

2492 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2493 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2494 metadata = exposure.getMetadata() 

2495 metadata.set('SKYLEVEL', skyLevel) 

2496 metadata.set('SKYSIGMA', skySigma) 

2497 

2498 # calcluating flatlevel over the subgrids 

2499 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2500 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2501 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2502 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2503 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2504 skyLevels = numpy.zeros((nX, nY)) 

2505 

2506 for j in range(nY): 

2507 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2508 for i in range(nX): 

2509 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2510 

2511 xLLC = xc - meshXHalf 

2512 yLLC = yc - meshYHalf 

2513 xURC = xc + meshXHalf - 1 

2514 yURC = yc + meshYHalf - 1 

2515 

2516 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2517 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2518 

2519 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2520 

2521 good = numpy.where(numpy.isfinite(skyLevels)) 

2522 skyMedian = numpy.median(skyLevels[good]) 

2523 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2524 flatness_rms = numpy.std(flatness) 

2525 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2526 

2527 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2528 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2529 nX, nY, flatness_pp, flatness_rms) 

2530 

2531 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2532 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2533 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2534 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2535 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2536 

2537 def roughZeroPoint(self, exposure): 

2538 """Set an approximate magnitude zero point for the exposure. 

2539 

2540 Parameters 

2541 ---------- 

2542 exposure : `lsst.afw.image.Exposure` 

2543 Exposure to process. 

2544 """ 

2545 filterLabel = exposure.getFilterLabel() 

2546 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2547 

2548 if physicalFilter in self.config.fluxMag0T1: 

2549 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2550 else: 

2551 self.log.warn("No rough magnitude zero point defined for filter {}.".format(physicalFilter)) 

2552 fluxMag0 = self.config.defaultFluxMag0T1 

2553 

2554 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2555 if not expTime > 0: # handle NaN as well as <= 0 

2556 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2557 return 

2558 

2559 self.log.info("Setting rough magnitude zero point for filter {}: {}". 

2560 format(physicalFilter, 2.5*math.log10(fluxMag0*expTime))) 

2561 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2562 

2563 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2564 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2565 

2566 Parameters 

2567 ---------- 

2568 ccdExposure : `lsst.afw.image.Exposure` 

2569 Exposure to process. 

2570 fpPolygon : `lsst.afw.geom.Polygon` 

2571 Polygon in focal plane coordinates. 

2572 """ 

2573 # Get ccd corners in focal plane coordinates 

2574 ccd = ccdExposure.getDetector() 

2575 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2576 ccdPolygon = Polygon(fpCorners) 

2577 

2578 # Get intersection of ccd corners with fpPolygon 

2579 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2580 

2581 # Transform back to pixel positions and build new polygon 

2582 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2583 validPolygon = Polygon(ccdPoints) 

2584 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2585 

2586 @contextmanager 

2587 def flatContext(self, exp, flat, dark=None): 

2588 """Context manager that applies and removes flats and darks, 

2589 if the task is configured to apply them. 

2590 

2591 Parameters 

2592 ---------- 

2593 exp : `lsst.afw.image.Exposure` 

2594 Exposure to process. 

2595 flat : `lsst.afw.image.Exposure` 

2596 Flat exposure the same size as ``exp``. 

2597 dark : `lsst.afw.image.Exposure`, optional 

2598 Dark exposure the same size as ``exp``. 

2599 

2600 Yields 

2601 ------ 

2602 exp : `lsst.afw.image.Exposure` 

2603 The flat and dark corrected exposure. 

2604 """ 

2605 if self.config.doDark and dark is not None: 

2606 self.darkCorrection(exp, dark) 

2607 if self.config.doFlat: 

2608 self.flatCorrection(exp, flat) 

2609 try: 

2610 yield exp 

2611 finally: 

2612 if self.config.doFlat: 

2613 self.flatCorrection(exp, flat, invert=True) 

2614 if self.config.doDark and dark is not None: 

2615 self.darkCorrection(exp, dark, invert=True) 

2616 

2617 def debugView(self, exposure, stepname): 

2618 """Utility function to examine ISR exposure at different stages. 

2619 

2620 Parameters 

2621 ---------- 

2622 exposure : `lsst.afw.image.Exposure` 

2623 Exposure to view. 

2624 stepname : `str` 

2625 State of processing to view. 

2626 """ 

2627 frame = getDebugFrame(self._display, stepname) 

2628 if frame: 

2629 display = getDisplay(frame) 

2630 display.scale('asinh', 'zscale') 

2631 display.mtv(exposure) 

2632 prompt = "Press Enter to continue [c]... " 

2633 while True: 

2634 ans = input(prompt).lower() 

2635 if ans in ("", "c",): 

2636 break 

2637 

2638 

2639class FakeAmp(object): 

2640 """A Detector-like object that supports returning gain and saturation level 

2641 

2642 This is used when the input exposure does not have a detector. 

2643 

2644 Parameters 

2645 ---------- 

2646 exposure : `lsst.afw.image.Exposure` 

2647 Exposure to generate a fake amplifier for. 

2648 config : `lsst.ip.isr.isrTaskConfig` 

2649 Configuration to apply to the fake amplifier. 

2650 """ 

2651 

2652 def __init__(self, exposure, config): 

2653 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2654 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2655 self._gain = config.gain 

2656 self._readNoise = config.readNoise 

2657 self._saturation = config.saturation 

2658 

2659 def getBBox(self): 

2660 return self._bbox 

2661 

2662 def getRawBBox(self): 

2663 return self._bbox 

2664 

2665 def getRawHorizontalOverscanBBox(self): 

2666 return self._RawHorizontalOverscanBBox 

2667 

2668 def getGain(self): 

2669 return self._gain 

2670 

2671 def getReadNoise(self): 

2672 return self._readNoise 

2673 

2674 def getSaturation(self): 

2675 return self._saturation 

2676 

2677 def getSuspectLevel(self): 

2678 return float("NaN") 

2679 

2680 

2681class RunIsrConfig(pexConfig.Config): 

2682 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2683 

2684 

2685class RunIsrTask(pipeBase.CmdLineTask): 

2686 """Task to wrap the default IsrTask to allow it to be retargeted. 

2687 

2688 The standard IsrTask can be called directly from a command line 

2689 program, but doing so removes the ability of the task to be 

2690 retargeted. As most cameras override some set of the IsrTask 

2691 methods, this would remove those data-specific methods in the 

2692 output post-ISR images. This wrapping class fixes the issue, 

2693 allowing identical post-ISR images to be generated by both the 

2694 processCcd and isrTask code. 

2695 """ 

2696 ConfigClass = RunIsrConfig 

2697 _DefaultName = "runIsr" 

2698 

2699 def __init__(self, *args, **kwargs): 

2700 super().__init__(*args, **kwargs) 

2701 self.makeSubtask("isr") 

2702 

2703 def runDataRef(self, dataRef): 

2704 """ 

2705 Parameters 

2706 ---------- 

2707 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2708 data reference of the detector data to be processed 

2709 

2710 Returns 

2711 ------- 

2712 result : `pipeBase.Struct` 

2713 Result struct with component: 

2714 

2715 - exposure : `lsst.afw.image.Exposure` 

2716 Post-ISR processed exposure. 

2717 """ 

2718 return self.isr.runDataRef(dataRef)