Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42 

43from . import isrFunctions 

44from . import isrQa 

45from . import linearize 

46from .defects import Defects 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 ) 

123 # TODO: DM-25348. This does not work yet to correctly load 

124 # possible crosstalk sources. 

125 crosstalkSources = cT.PrerequisiteInput( 

126 name="isrOverscanCorrected", 

127 doc="Overscan corrected input images.", 

128 storageClass="Exposure", 

129 dimensions=["instrument", "exposure", "detector"], 

130 deferLoad=True, 

131 multiple=True, 

132 lookupFunction=crosstalkSourceLookup, 

133 ) 

134 bias = cT.PrerequisiteInput( 

135 name="bias", 

136 doc="Input bias calibration.", 

137 storageClass="ExposureF", 

138 dimensions=["instrument", "detector"], 

139 isCalibration=True, 

140 ) 

141 dark = cT.PrerequisiteInput( 

142 name='dark', 

143 doc="Input dark calibration.", 

144 storageClass="ExposureF", 

145 dimensions=["instrument", "detector"], 

146 isCalibration=True, 

147 ) 

148 flat = cT.PrerequisiteInput( 

149 name="flat", 

150 doc="Input flat calibration.", 

151 storageClass="ExposureF", 

152 dimensions=["instrument", "physical_filter", "detector"], 

153 isCalibration=True, 

154 ) 

155 fringes = cT.PrerequisiteInput( 

156 name="fringe", 

157 doc="Input fringe calibration.", 

158 storageClass="ExposureF", 

159 dimensions=["instrument", "physical_filter", "detector"], 

160 isCalibration=True, 

161 ) 

162 strayLightData = cT.PrerequisiteInput( 

163 name='yBackground', 

164 doc="Input stray light calibration.", 

165 storageClass="StrayLightData", 

166 dimensions=["instrument", "physical_filter", "detector"], 

167 isCalibration=True, 

168 ) 

169 bfKernel = cT.PrerequisiteInput( 

170 name='bfKernel', 

171 doc="Input brighter-fatter kernel.", 

172 storageClass="NumpyArray", 

173 dimensions=["instrument"], 

174 isCalibration=True, 

175 ) 

176 newBFKernel = cT.PrerequisiteInput( 

177 name='brighterFatterKernel', 

178 doc="Newer complete kernel + gain solutions.", 

179 storageClass="BrighterFatterKernel", 

180 dimensions=["instrument", "detector"], 

181 isCalibration=True, 

182 ) 

183 defects = cT.PrerequisiteInput( 

184 name='defects', 

185 doc="Input defect tables.", 

186 storageClass="Defects", 

187 dimensions=["instrument", "detector"], 

188 isCalibration=True, 

189 ) 

190 opticsTransmission = cT.PrerequisiteInput( 

191 name="transmission_optics", 

192 storageClass="TransmissionCurve", 

193 doc="Transmission curve due to the optics.", 

194 dimensions=["instrument"], 

195 isCalibration=True, 

196 ) 

197 filterTransmission = cT.PrerequisiteInput( 

198 name="transmission_filter", 

199 storageClass="TransmissionCurve", 

200 doc="Transmission curve due to the filter.", 

201 dimensions=["instrument", "physical_filter"], 

202 isCalibration=True, 

203 ) 

204 sensorTransmission = cT.PrerequisiteInput( 

205 name="transmission_sensor", 

206 storageClass="TransmissionCurve", 

207 doc="Transmission curve due to the sensor.", 

208 dimensions=["instrument", "detector"], 

209 isCalibration=True, 

210 ) 

211 atmosphereTransmission = cT.PrerequisiteInput( 

212 name="transmission_atmosphere", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the atmosphere.", 

215 dimensions=["instrument"], 

216 isCalibration=True, 

217 ) 

218 illumMaskedImage = cT.PrerequisiteInput( 

219 name="illum", 

220 doc="Input illumination correction.", 

221 storageClass="MaskedImageF", 

222 dimensions=["instrument", "physical_filter", "detector"], 

223 isCalibration=True, 

224 ) 

225 

226 outputExposure = cT.Output( 

227 name='postISRCCD', 

228 doc="Output ISR processed exposure.", 

229 storageClass="Exposure", 

230 dimensions=["instrument", "exposure", "detector"], 

231 ) 

232 preInterpExposure = cT.Output( 

233 name='preInterpISRCCD', 

234 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

235 storageClass="ExposureF", 

236 dimensions=["instrument", "exposure", "detector"], 

237 ) 

238 outputOssThumbnail = cT.Output( 

239 name="OssThumb", 

240 doc="Output Overscan-subtracted thumbnail image.", 

241 storageClass="Thumbnail", 

242 dimensions=["instrument", "exposure", "detector"], 

243 ) 

244 outputFlattenedThumbnail = cT.Output( 

245 name="FlattenedThumb", 

246 doc="Output flat-corrected thumbnail image.", 

247 storageClass="Thumbnail", 

248 dimensions=["instrument", "exposure", "detector"], 

249 ) 

250 

251 def __init__(self, *, config=None): 

252 super().__init__(config=config) 

253 

254 if config.doBias is not True: 

255 self.prerequisiteInputs.discard("bias") 

256 if config.doLinearize is not True: 

257 self.prerequisiteInputs.discard("linearizer") 

258 if config.doCrosstalk is not True: 

259 self.inputs.discard("crosstalkSources") 

260 self.prerequisiteInputs.discard("crosstalk") 

261 if config.doBrighterFatter is not True: 

262 self.prerequisiteInputs.discard("bfKernel") 

263 self.prerequisiteInputs.discard("newBFKernel") 

264 if config.doDefect is not True: 

265 self.prerequisiteInputs.discard("defects") 

266 if config.doDark is not True: 

267 self.prerequisiteInputs.discard("dark") 

268 if config.doFlat is not True: 

269 self.prerequisiteInputs.discard("flat") 

270 if config.doAttachTransmissionCurve is not True: 

271 self.prerequisiteInputs.discard("opticsTransmission") 

272 self.prerequisiteInputs.discard("filterTransmission") 

273 self.prerequisiteInputs.discard("sensorTransmission") 

274 self.prerequisiteInputs.discard("atmosphereTransmission") 

275 if config.doUseOpticsTransmission is not True: 

276 self.prerequisiteInputs.discard("opticsTransmission") 

277 if config.doUseFilterTransmission is not True: 

278 self.prerequisiteInputs.discard("filterTransmission") 

279 if config.doUseSensorTransmission is not True: 

280 self.prerequisiteInputs.discard("sensorTransmission") 

281 if config.doUseAtmosphereTransmission is not True: 

282 self.prerequisiteInputs.discard("atmosphereTransmission") 

283 if config.doIlluminationCorrection is not True: 

284 self.prerequisiteInputs.discard("illumMaskedImage") 

285 

286 if config.doWrite is not True: 

287 self.outputs.discard("outputExposure") 

288 self.outputs.discard("preInterpExposure") 

289 self.outputs.discard("outputFlattenedThumbnail") 

290 self.outputs.discard("outputOssThumbnail") 

291 if config.doSaveInterpPixels is not True: 

292 self.outputs.discard("preInterpExposure") 

293 if config.qa.doThumbnailOss is not True: 

294 self.outputs.discard("outputOssThumbnail") 

295 if config.qa.doThumbnailFlattened is not True: 

296 self.outputs.discard("outputFlattenedThumbnail") 

297 

298 

299class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

300 pipelineConnections=IsrTaskConnections): 

301 """Configuration parameters for IsrTask. 

302 

303 Items are grouped in the order in which they are executed by the task. 

304 """ 

305 datasetType = pexConfig.Field( 

306 dtype=str, 

307 doc="Dataset type for input data; users will typically leave this alone, " 

308 "but camera-specific ISR tasks will override it", 

309 default="raw", 

310 ) 

311 

312 fallbackFilterName = pexConfig.Field( 

313 dtype=str, 

314 doc="Fallback default filter name for calibrations.", 

315 optional=True 

316 ) 

317 useFallbackDate = pexConfig.Field( 

318 dtype=bool, 

319 doc="Pass observation date when using fallback filter.", 

320 default=False, 

321 ) 

322 expectWcs = pexConfig.Field( 

323 dtype=bool, 

324 default=True, 

325 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

326 ) 

327 fwhm = pexConfig.Field( 

328 dtype=float, 

329 doc="FWHM of PSF in arcseconds.", 

330 default=1.0, 

331 ) 

332 qa = pexConfig.ConfigField( 

333 dtype=isrQa.IsrQaConfig, 

334 doc="QA related configuration options.", 

335 ) 

336 

337 # Image conversion configuration 

338 doConvertIntToFloat = pexConfig.Field( 

339 dtype=bool, 

340 doc="Convert integer raw images to floating point values?", 

341 default=True, 

342 ) 

343 

344 # Saturated pixel handling. 

345 doSaturation = pexConfig.Field( 

346 dtype=bool, 

347 doc="Mask saturated pixels? NB: this is totally independent of the" 

348 " interpolation option - this is ONLY setting the bits in the mask." 

349 " To have them interpolated make sure doSaturationInterpolation=True", 

350 default=True, 

351 ) 

352 saturatedMaskName = pexConfig.Field( 

353 dtype=str, 

354 doc="Name of mask plane to use in saturation detection and interpolation", 

355 default="SAT", 

356 ) 

357 saturation = pexConfig.Field( 

358 dtype=float, 

359 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

360 default=float("NaN"), 

361 ) 

362 growSaturationFootprintSize = pexConfig.Field( 

363 dtype=int, 

364 doc="Number of pixels by which to grow the saturation footprints", 

365 default=1, 

366 ) 

367 

368 # Suspect pixel handling. 

369 doSuspect = pexConfig.Field( 

370 dtype=bool, 

371 doc="Mask suspect pixels?", 

372 default=False, 

373 ) 

374 suspectMaskName = pexConfig.Field( 

375 dtype=str, 

376 doc="Name of mask plane to use for suspect pixels", 

377 default="SUSPECT", 

378 ) 

379 numEdgeSuspect = pexConfig.Field( 

380 dtype=int, 

381 doc="Number of edge pixels to be flagged as untrustworthy.", 

382 default=0, 

383 ) 

384 edgeMaskLevel = pexConfig.ChoiceField( 

385 dtype=str, 

386 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

387 default="DETECTOR", 

388 allowed={ 

389 'DETECTOR': 'Mask only the edges of the full detector.', 

390 'AMP': 'Mask edges of each amplifier.', 

391 }, 

392 ) 

393 

394 # Initial masking options. 

395 doSetBadRegions = pexConfig.Field( 

396 dtype=bool, 

397 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

398 default=True, 

399 ) 

400 badStatistic = pexConfig.ChoiceField( 

401 dtype=str, 

402 doc="How to estimate the average value for BAD regions.", 

403 default='MEANCLIP', 

404 allowed={ 

405 "MEANCLIP": "Correct using the (clipped) mean of good data", 

406 "MEDIAN": "Correct using the median of the good data", 

407 }, 

408 ) 

409 

410 # Overscan subtraction configuration. 

411 doOverscan = pexConfig.Field( 

412 dtype=bool, 

413 doc="Do overscan subtraction?", 

414 default=True, 

415 ) 

416 overscan = pexConfig.ConfigurableField( 

417 target=OverscanCorrectionTask, 

418 doc="Overscan subtraction task for image segments.", 

419 ) 

420 

421 overscanFitType = pexConfig.ChoiceField( 

422 dtype=str, 

423 doc="The method for fitting the overscan bias level.", 

424 default='MEDIAN', 

425 allowed={ 

426 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

427 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

428 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

429 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

430 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

431 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

432 "MEAN": "Correct using the mean of the overscan region", 

433 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

434 "MEDIAN": "Correct using the median of the overscan region", 

435 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

436 }, 

437 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

438 " This option will no longer be used, and will be removed after v20.") 

439 ) 

440 overscanOrder = pexConfig.Field( 

441 dtype=int, 

442 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

443 "or number of spline knots if overscan fit type is a spline."), 

444 default=1, 

445 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

446 " This option will no longer be used, and will be removed after v20.") 

447 ) 

448 overscanNumSigmaClip = pexConfig.Field( 

449 dtype=float, 

450 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

451 default=3.0, 

452 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

453 " This option will no longer be used, and will be removed after v20.") 

454 ) 

455 overscanIsInt = pexConfig.Field( 

456 dtype=bool, 

457 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

458 " and overscan.FitType=MEDIAN_PER_ROW.", 

459 default=True, 

460 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

461 " This option will no longer be used, and will be removed after v20.") 

462 ) 

463 # These options do not get deprecated, as they define how we slice up the image data. 

464 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

465 dtype=int, 

466 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

467 default=0, 

468 ) 

469 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

470 dtype=int, 

471 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

472 default=0, 

473 ) 

474 overscanMaxDev = pexConfig.Field( 474 ↛ exitline 474 didn't jump to the function exit

475 dtype=float, 

476 doc="Maximum deviation from the median for overscan", 

477 default=1000.0, check=lambda x: x > 0 

478 ) 

479 overscanBiasJump = pexConfig.Field( 

480 dtype=bool, 

481 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

482 default=False, 

483 ) 

484 overscanBiasJumpKeyword = pexConfig.Field( 

485 dtype=str, 

486 doc="Header keyword containing information about devices.", 

487 default="NO_SUCH_KEY", 

488 ) 

489 overscanBiasJumpDevices = pexConfig.ListField( 

490 dtype=str, 

491 doc="List of devices that need piecewise overscan correction.", 

492 default=(), 

493 ) 

494 overscanBiasJumpLocation = pexConfig.Field( 

495 dtype=int, 

496 doc="Location of bias jump along y-axis.", 

497 default=0, 

498 ) 

499 

500 # Amplifier to CCD assembly configuration 

501 doAssembleCcd = pexConfig.Field( 

502 dtype=bool, 

503 default=True, 

504 doc="Assemble amp-level exposures into a ccd-level exposure?" 

505 ) 

506 assembleCcd = pexConfig.ConfigurableField( 

507 target=AssembleCcdTask, 

508 doc="CCD assembly task", 

509 ) 

510 

511 # General calibration configuration. 

512 doAssembleIsrExposures = pexConfig.Field( 

513 dtype=bool, 

514 default=False, 

515 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

516 ) 

517 doTrimToMatchCalib = pexConfig.Field( 

518 dtype=bool, 

519 default=False, 

520 doc="Trim raw data to match calibration bounding boxes?" 

521 ) 

522 

523 # Bias subtraction. 

524 doBias = pexConfig.Field( 

525 dtype=bool, 

526 doc="Apply bias frame correction?", 

527 default=True, 

528 ) 

529 biasDataProductName = pexConfig.Field( 

530 dtype=str, 

531 doc="Name of the bias data product", 

532 default="bias", 

533 ) 

534 doBiasBeforeOverscan = pexConfig.Field( 

535 dtype=bool, 

536 doc="Reverse order of overscan and bias correction.", 

537 default=False 

538 ) 

539 

540 # Variance construction 

541 doVariance = pexConfig.Field( 

542 dtype=bool, 

543 doc="Calculate variance?", 

544 default=True 

545 ) 

546 gain = pexConfig.Field( 

547 dtype=float, 

548 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

549 default=float("NaN"), 

550 ) 

551 readNoise = pexConfig.Field( 

552 dtype=float, 

553 doc="The read noise to use if no Detector is present in the Exposure", 

554 default=0.0, 

555 ) 

556 doEmpiricalReadNoise = pexConfig.Field( 

557 dtype=bool, 

558 default=False, 

559 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

560 ) 

561 

562 # Linearization. 

563 doLinearize = pexConfig.Field( 

564 dtype=bool, 

565 doc="Correct for nonlinearity of the detector's response?", 

566 default=True, 

567 ) 

568 

569 # Crosstalk. 

570 doCrosstalk = pexConfig.Field( 

571 dtype=bool, 

572 doc="Apply intra-CCD crosstalk correction?", 

573 default=False, 

574 ) 

575 doCrosstalkBeforeAssemble = pexConfig.Field( 

576 dtype=bool, 

577 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

578 default=False, 

579 ) 

580 crosstalk = pexConfig.ConfigurableField( 

581 target=CrosstalkTask, 

582 doc="Intra-CCD crosstalk correction", 

583 ) 

584 

585 # Masking options. 

586 doDefect = pexConfig.Field( 

587 dtype=bool, 

588 doc="Apply correction for CCD defects, e.g. hot pixels?", 

589 default=True, 

590 ) 

591 doNanMasking = pexConfig.Field( 

592 dtype=bool, 

593 doc="Mask NAN pixels?", 

594 default=True, 

595 ) 

596 doWidenSaturationTrails = pexConfig.Field( 

597 dtype=bool, 

598 doc="Widen bleed trails based on their width?", 

599 default=True 

600 ) 

601 

602 # Brighter-Fatter correction. 

603 doBrighterFatter = pexConfig.Field( 

604 dtype=bool, 

605 default=False, 

606 doc="Apply the brighter fatter correction" 

607 ) 

608 brighterFatterLevel = pexConfig.ChoiceField( 

609 dtype=str, 

610 default="DETECTOR", 

611 doc="The level at which to correct for brighter-fatter.", 

612 allowed={ 

613 "AMP": "Every amplifier treated separately.", 

614 "DETECTOR": "One kernel per detector", 

615 } 

616 ) 

617 brighterFatterMaxIter = pexConfig.Field( 

618 dtype=int, 

619 default=10, 

620 doc="Maximum number of iterations for the brighter fatter correction" 

621 ) 

622 brighterFatterThreshold = pexConfig.Field( 

623 dtype=float, 

624 default=1000, 

625 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

626 " absolute value of the difference between the current corrected image and the one" 

627 " from the previous iteration summed over all the pixels." 

628 ) 

629 brighterFatterApplyGain = pexConfig.Field( 

630 dtype=bool, 

631 default=True, 

632 doc="Should the gain be applied when applying the brighter fatter correction?" 

633 ) 

634 brighterFatterMaskGrowSize = pexConfig.Field( 

635 dtype=int, 

636 default=0, 

637 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

638 " when brighter-fatter correction is applied." 

639 ) 

640 

641 # Dark subtraction. 

642 doDark = pexConfig.Field( 

643 dtype=bool, 

644 doc="Apply dark frame correction?", 

645 default=True, 

646 ) 

647 darkDataProductName = pexConfig.Field( 

648 dtype=str, 

649 doc="Name of the dark data product", 

650 default="dark", 

651 ) 

652 

653 # Camera-specific stray light removal. 

654 doStrayLight = pexConfig.Field( 

655 dtype=bool, 

656 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

657 default=False, 

658 ) 

659 strayLight = pexConfig.ConfigurableField( 

660 target=StrayLightTask, 

661 doc="y-band stray light correction" 

662 ) 

663 

664 # Flat correction. 

665 doFlat = pexConfig.Field( 

666 dtype=bool, 

667 doc="Apply flat field correction?", 

668 default=True, 

669 ) 

670 flatDataProductName = pexConfig.Field( 

671 dtype=str, 

672 doc="Name of the flat data product", 

673 default="flat", 

674 ) 

675 flatScalingType = pexConfig.ChoiceField( 

676 dtype=str, 

677 doc="The method for scaling the flat on the fly.", 

678 default='USER', 

679 allowed={ 

680 "USER": "Scale by flatUserScale", 

681 "MEAN": "Scale by the inverse of the mean", 

682 "MEDIAN": "Scale by the inverse of the median", 

683 }, 

684 ) 

685 flatUserScale = pexConfig.Field( 

686 dtype=float, 

687 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

688 default=1.0, 

689 ) 

690 doTweakFlat = pexConfig.Field( 

691 dtype=bool, 

692 doc="Tweak flats to match observed amplifier ratios?", 

693 default=False 

694 ) 

695 

696 # Amplifier normalization based on gains instead of using flats configuration. 

697 doApplyGains = pexConfig.Field( 

698 dtype=bool, 

699 doc="Correct the amplifiers for their gains instead of applying flat correction", 

700 default=False, 

701 ) 

702 normalizeGains = pexConfig.Field( 

703 dtype=bool, 

704 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

705 default=False, 

706 ) 

707 

708 # Fringe correction. 

709 doFringe = pexConfig.Field( 

710 dtype=bool, 

711 doc="Apply fringe correction?", 

712 default=True, 

713 ) 

714 fringe = pexConfig.ConfigurableField( 

715 target=FringeTask, 

716 doc="Fringe subtraction task", 

717 ) 

718 fringeAfterFlat = pexConfig.Field( 

719 dtype=bool, 

720 doc="Do fringe subtraction after flat-fielding?", 

721 default=True, 

722 ) 

723 

724 # Initial CCD-level background statistics options. 

725 doMeasureBackground = pexConfig.Field( 

726 dtype=bool, 

727 doc="Measure the background level on the reduced image?", 

728 default=False, 

729 ) 

730 

731 # Camera-specific masking configuration. 

732 doCameraSpecificMasking = pexConfig.Field( 

733 dtype=bool, 

734 doc="Mask camera-specific bad regions?", 

735 default=False, 

736 ) 

737 masking = pexConfig.ConfigurableField( 

738 target=MaskingTask, 

739 doc="Masking task." 

740 ) 

741 

742 # Interpolation options. 

743 

744 doInterpolate = pexConfig.Field( 

745 dtype=bool, 

746 doc="Interpolate masked pixels?", 

747 default=True, 

748 ) 

749 doSaturationInterpolation = pexConfig.Field( 

750 dtype=bool, 

751 doc="Perform interpolation over pixels masked as saturated?" 

752 " NB: This is independent of doSaturation; if that is False this plane" 

753 " will likely be blank, resulting in a no-op here.", 

754 default=True, 

755 ) 

756 doNanInterpolation = pexConfig.Field( 

757 dtype=bool, 

758 doc="Perform interpolation over pixels masked as NaN?" 

759 " NB: This is independent of doNanMasking; if that is False this plane" 

760 " will likely be blank, resulting in a no-op here.", 

761 default=True, 

762 ) 

763 doNanInterpAfterFlat = pexConfig.Field( 

764 dtype=bool, 

765 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

766 "also have to interpolate them before flat-fielding."), 

767 default=False, 

768 ) 

769 maskListToInterpolate = pexConfig.ListField( 

770 dtype=str, 

771 doc="List of mask planes that should be interpolated.", 

772 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

773 ) 

774 doSaveInterpPixels = pexConfig.Field( 

775 dtype=bool, 

776 doc="Save a copy of the pre-interpolated pixel values?", 

777 default=False, 

778 ) 

779 

780 # Default photometric calibration options. 

781 fluxMag0T1 = pexConfig.DictField( 

782 keytype=str, 

783 itemtype=float, 

784 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

785 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

786 )) 

787 ) 

788 defaultFluxMag0T1 = pexConfig.Field( 

789 dtype=float, 

790 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

791 default=pow(10.0, 0.4*28.0) 

792 ) 

793 

794 # Vignette correction configuration. 

795 doVignette = pexConfig.Field( 

796 dtype=bool, 

797 doc="Apply vignetting parameters?", 

798 default=False, 

799 ) 

800 vignette = pexConfig.ConfigurableField( 

801 target=VignetteTask, 

802 doc="Vignetting task.", 

803 ) 

804 

805 # Transmission curve configuration. 

806 doAttachTransmissionCurve = pexConfig.Field( 

807 dtype=bool, 

808 default=False, 

809 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

810 ) 

811 doUseOpticsTransmission = pexConfig.Field( 

812 dtype=bool, 

813 default=True, 

814 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

815 ) 

816 doUseFilterTransmission = pexConfig.Field( 

817 dtype=bool, 

818 default=True, 

819 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

820 ) 

821 doUseSensorTransmission = pexConfig.Field( 

822 dtype=bool, 

823 default=True, 

824 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

825 ) 

826 doUseAtmosphereTransmission = pexConfig.Field( 

827 dtype=bool, 

828 default=True, 

829 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

830 ) 

831 

832 # Illumination correction. 

833 doIlluminationCorrection = pexConfig.Field( 

834 dtype=bool, 

835 default=False, 

836 doc="Perform illumination correction?" 

837 ) 

838 illuminationCorrectionDataProductName = pexConfig.Field( 

839 dtype=str, 

840 doc="Name of the illumination correction data product.", 

841 default="illumcor", 

842 ) 

843 illumScale = pexConfig.Field( 

844 dtype=float, 

845 doc="Scale factor for the illumination correction.", 

846 default=1.0, 

847 ) 

848 illumFilters = pexConfig.ListField( 

849 dtype=str, 

850 default=[], 

851 doc="Only perform illumination correction for these filters." 

852 ) 

853 

854 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

855 doWrite = pexConfig.Field( 

856 dtype=bool, 

857 doc="Persist postISRCCD?", 

858 default=True, 

859 ) 

860 

861 def validate(self): 

862 super().validate() 

863 if self.doFlat and self.doApplyGains: 

864 raise ValueError("You may not specify both doFlat and doApplyGains") 

865 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

866 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

867 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

868 self.maskListToInterpolate.append(self.saturatedMaskName) 

869 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

870 self.maskListToInterpolate.remove(self.saturatedMaskName) 

871 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

872 self.maskListToInterpolate.append("UNMASKEDNAN") 

873 

874 

875class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

876 """Apply common instrument signature correction algorithms to a raw frame. 

877 

878 The process for correcting imaging data is very similar from 

879 camera to camera. This task provides a vanilla implementation of 

880 doing these corrections, including the ability to turn certain 

881 corrections off if they are not needed. The inputs to the primary 

882 method, `run()`, are a raw exposure to be corrected and the 

883 calibration data products. The raw input is a single chip sized 

884 mosaic of all amps including overscans and other non-science 

885 pixels. The method `runDataRef()` identifies and defines the 

886 calibration data products, and is intended for use by a 

887 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

888 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

889 subclassed for different camera, although the most camera specific 

890 methods have been split into subtasks that can be redirected 

891 appropriately. 

892 

893 The __init__ method sets up the subtasks for ISR processing, using 

894 the defaults from `lsst.ip.isr`. 

895 

896 Parameters 

897 ---------- 

898 args : `list` 

899 Positional arguments passed to the Task constructor. None used at this time. 

900 kwargs : `dict`, optional 

901 Keyword arguments passed on to the Task constructor. None used at this time. 

902 """ 

903 ConfigClass = IsrTaskConfig 

904 _DefaultName = "isr" 

905 

906 def __init__(self, **kwargs): 

907 super().__init__(**kwargs) 

908 self.makeSubtask("assembleCcd") 

909 self.makeSubtask("crosstalk") 

910 self.makeSubtask("strayLight") 

911 self.makeSubtask("fringe") 

912 self.makeSubtask("masking") 

913 self.makeSubtask("overscan") 

914 self.makeSubtask("vignette") 

915 

916 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

917 inputs = butlerQC.get(inputRefs) 

918 

919 try: 

920 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

921 except Exception as e: 

922 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

923 (inputRefs, e)) 

924 

925 inputs['isGen3'] = True 

926 

927 detector = inputs['ccdExposure'].getDetector() 

928 

929 if self.config.doCrosstalk is True: 

930 # Crosstalk sources need to be defined by the pipeline 

931 # yaml if they exist. 

932 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

933 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

934 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

935 else: 

936 coeffVector = (self.config.crosstalk.crosstalkValues 

937 if self.config.crosstalk.useConfigCoefficients else None) 

938 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

939 inputs['crosstalk'] = crosstalkCalib 

940 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

941 if 'crosstalkSources' not in inputs: 

942 self.log.warn("No crosstalkSources found for chip with interChip terms!") 

943 

944 if self.doLinearize(detector) is True: 

945 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict): 

946 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

947 linearizer.fromYaml(inputs['linearizer']) 

948 else: 

949 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector, 

950 log=self.log) 

951 inputs['linearizer'] = linearizer 

952 

953 if self.config.doDefect is True: 

954 if "defects" in inputs and inputs['defects'] is not None: 

955 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

956 # masking expects a list of defects defined by their bounding box 

957 if not isinstance(inputs["defects"], Defects): 

958 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

959 

960 # Load the correct style of brighter fatter kernel, and repack 

961 # the information as a numpy array. 

962 if self.config.doBrighterFatter: 

963 brighterFatterKernel = inputs.pop('newBFKernel', None) 

964 if brighterFatterKernel is None: 

965 brighterFatterKernel = inputs.get('bfKernel', None) 

966 

967 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

968 detId = detector.getId() 

969 inputs['bfGains'] = brighterFatterKernel.gain 

970 # If the kernel is not an ndarray, it's the cp_pipe version 

971 # so extract the kernel for this detector, or raise an error 

972 if self.config.brighterFatterLevel == 'DETECTOR': 

973 if brighterFatterKernel.detectorKernel: 

974 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

975 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

976 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

977 else: 

978 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

979 else: 

980 # TODO DM-15631 for implementing this 

981 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

982 

983 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

984 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

985 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

986 expId=expId, 

987 assembler=self.assembleCcd 

988 if self.config.doAssembleIsrExposures else None) 

989 else: 

990 inputs['fringes'] = pipeBase.Struct(fringes=None) 

991 

992 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

993 if 'strayLightData' not in inputs: 

994 inputs['strayLightData'] = None 

995 

996 outputs = self.run(**inputs) 

997 butlerQC.put(outputs, outputRefs) 

998 

999 def readIsrData(self, dataRef, rawExposure): 

1000 """Retrieve necessary frames for instrument signature removal. 

1001 

1002 Pre-fetching all required ISR data products limits the IO 

1003 required by the ISR. Any conflict between the calibration data 

1004 available and that needed for ISR is also detected prior to 

1005 doing processing, allowing it to fail quickly. 

1006 

1007 Parameters 

1008 ---------- 

1009 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1010 Butler reference of the detector data to be processed 

1011 rawExposure : `afw.image.Exposure` 

1012 The raw exposure that will later be corrected with the 

1013 retrieved calibration data; should not be modified in this 

1014 method. 

1015 

1016 Returns 

1017 ------- 

1018 result : `lsst.pipe.base.Struct` 

1019 Result struct with components (which may be `None`): 

1020 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1021 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1022 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1023 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1024 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1025 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1026 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1027 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1028 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1029 - ``seed``: random seed derived from the ccdExposureId for random 

1030 number generator (`uint32`). 

1031 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1032 A ``TransmissionCurve`` that represents the throughput of the optics, 

1033 to be evaluated in focal-plane coordinates. 

1034 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1035 A ``TransmissionCurve`` that represents the throughput of the filter 

1036 itself, to be evaluated in focal-plane coordinates. 

1037 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1038 A ``TransmissionCurve`` that represents the throughput of the sensor 

1039 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1040 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1041 A ``TransmissionCurve`` that represents the throughput of the 

1042 atmosphere, assumed to be spatially constant. 

1043 - ``strayLightData`` : `object` 

1044 An opaque object containing calibration information for 

1045 stray-light correction. If `None`, no correction will be 

1046 performed. 

1047 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1048 

1049 Raises 

1050 ------ 

1051 NotImplementedError : 

1052 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1053 """ 

1054 try: 

1055 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1056 dateObs = dateObs.toPython().isoformat() 

1057 except RuntimeError: 

1058 self.log.warn("Unable to identify dateObs for rawExposure.") 

1059 dateObs = None 

1060 

1061 ccd = rawExposure.getDetector() 

1062 # TODO DM-28093: change this to: rawExposure.getFilterLabel().physicalLabel 

1063 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

1064 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1065 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1066 if self.config.doBias else None) 

1067 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1068 linearizer = (dataRef.get("linearizer", immediate=True) 

1069 if self.doLinearize(ccd) else None) 

1070 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1071 linearizer.log = self.log 

1072 if isinstance(linearizer, numpy.ndarray): 

1073 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1074 

1075 crosstalkCalib = None 

1076 if self.config.doCrosstalk: 

1077 try: 

1078 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1079 except NoResults: 

1080 coeffVector = (self.config.crosstalk.crosstalkValues 

1081 if self.config.crosstalk.useConfigCoefficients else None) 

1082 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1083 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1084 if self.config.doCrosstalk else None) 

1085 

1086 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1087 if self.config.doDark else None) 

1088 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1089 dateObs=dateObs) 

1090 if self.config.doFlat else None) 

1091 

1092 brighterFatterKernel = None 

1093 brighterFatterGains = None 

1094 if self.config.doBrighterFatter is True: 

1095 try: 

1096 # Use the new-style cp_pipe version of the kernel if it exists 

1097 # If using a new-style kernel, always use the self-consistent 

1098 # gains, i.e. the ones inside the kernel object itself 

1099 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1100 brighterFatterGains = brighterFatterKernel.gain 

1101 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

1102 except NoResults: 

1103 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1104 brighterFatterKernel = dataRef.get("bfKernel") 

1105 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

1106 except NoResults: 

1107 brighterFatterKernel = None 

1108 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1109 # If the kernel is not an ndarray, it's the cp_pipe version 

1110 # so extract the kernel for this detector, or raise an error 

1111 if self.config.brighterFatterLevel == 'DETECTOR': 

1112 if brighterFatterKernel.detectorKernel: 

1113 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

1114 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

1115 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

1116 else: 

1117 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1118 else: 

1119 # TODO DM-15631 for implementing this 

1120 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1121 

1122 defectList = (dataRef.get("defects") 

1123 if self.config.doDefect else None) 

1124 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

1125 if self.config.doAssembleIsrExposures else None) 

1126 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1127 else pipeBase.Struct(fringes=None)) 

1128 

1129 if self.config.doAttachTransmissionCurve: 

1130 opticsTransmission = (dataRef.get("transmission_optics") 

1131 if self.config.doUseOpticsTransmission else None) 

1132 filterTransmission = (dataRef.get("transmission_filter") 

1133 if self.config.doUseFilterTransmission else None) 

1134 sensorTransmission = (dataRef.get("transmission_sensor") 

1135 if self.config.doUseSensorTransmission else None) 

1136 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1137 if self.config.doUseAtmosphereTransmission else None) 

1138 else: 

1139 opticsTransmission = None 

1140 filterTransmission = None 

1141 sensorTransmission = None 

1142 atmosphereTransmission = None 

1143 

1144 if self.config.doStrayLight: 

1145 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1146 else: 

1147 strayLightData = None 

1148 

1149 illumMaskedImage = (self.getIsrExposure(dataRef, 

1150 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1151 if (self.config.doIlluminationCorrection 

1152 and filterName in self.config.illumFilters) 

1153 else None) 

1154 

1155 # Struct should include only kwargs to run() 

1156 return pipeBase.Struct(bias=biasExposure, 

1157 linearizer=linearizer, 

1158 crosstalk=crosstalkCalib, 

1159 crosstalkSources=crosstalkSources, 

1160 dark=darkExposure, 

1161 flat=flatExposure, 

1162 bfKernel=brighterFatterKernel, 

1163 bfGains=brighterFatterGains, 

1164 defects=defectList, 

1165 fringes=fringeStruct, 

1166 opticsTransmission=opticsTransmission, 

1167 filterTransmission=filterTransmission, 

1168 sensorTransmission=sensorTransmission, 

1169 atmosphereTransmission=atmosphereTransmission, 

1170 strayLightData=strayLightData, 

1171 illumMaskedImage=illumMaskedImage 

1172 ) 

1173 

1174 @pipeBase.timeMethod 

1175 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, 

1176 crosstalk=None, crosstalkSources=None, 

1177 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1178 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1179 sensorTransmission=None, atmosphereTransmission=None, 

1180 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1181 isGen3=False, 

1182 ): 

1183 """Perform instrument signature removal on an exposure. 

1184 

1185 Steps included in the ISR processing, in order performed, are: 

1186 - saturation and suspect pixel masking 

1187 - overscan subtraction 

1188 - CCD assembly of individual amplifiers 

1189 - bias subtraction 

1190 - variance image construction 

1191 - linearization of non-linear response 

1192 - crosstalk masking 

1193 - brighter-fatter correction 

1194 - dark subtraction 

1195 - fringe correction 

1196 - stray light subtraction 

1197 - flat correction 

1198 - masking of known defects and camera specific features 

1199 - vignette calculation 

1200 - appending transmission curve and distortion model 

1201 

1202 Parameters 

1203 ---------- 

1204 ccdExposure : `lsst.afw.image.Exposure` 

1205 The raw exposure that is to be run through ISR. The 

1206 exposure is modified by this method. 

1207 camera : `lsst.afw.cameraGeom.Camera`, optional 

1208 The camera geometry for this exposure. Required if ``isGen3`` is 

1209 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1210 ``flat`` does not have an associated detector. 

1211 bias : `lsst.afw.image.Exposure`, optional 

1212 Bias calibration frame. 

1213 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1214 Functor for linearization. 

1215 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1216 Calibration for crosstalk. 

1217 crosstalkSources : `list`, optional 

1218 List of possible crosstalk sources. 

1219 dark : `lsst.afw.image.Exposure`, optional 

1220 Dark calibration frame. 

1221 flat : `lsst.afw.image.Exposure`, optional 

1222 Flat calibration frame. 

1223 bfKernel : `numpy.ndarray`, optional 

1224 Brighter-fatter kernel. 

1225 bfGains : `dict` of `float`, optional 

1226 Gains used to override the detector's nominal gains for the 

1227 brighter-fatter correction. A dict keyed by amplifier name for 

1228 the detector in question. 

1229 defects : `lsst.ip.isr.Defects`, optional 

1230 List of defects. 

1231 fringes : `lsst.pipe.base.Struct`, optional 

1232 Struct containing the fringe correction data, with 

1233 elements: 

1234 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1235 - ``seed``: random seed derived from the ccdExposureId for random 

1236 number generator (`uint32`) 

1237 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1238 A ``TransmissionCurve`` that represents the throughput of the optics, 

1239 to be evaluated in focal-plane coordinates. 

1240 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1241 A ``TransmissionCurve`` that represents the throughput of the filter 

1242 itself, to be evaluated in focal-plane coordinates. 

1243 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1244 A ``TransmissionCurve`` that represents the throughput of the sensor 

1245 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1246 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1247 A ``TransmissionCurve`` that represents the throughput of the 

1248 atmosphere, assumed to be spatially constant. 

1249 detectorNum : `int`, optional 

1250 The integer number for the detector to process. 

1251 isGen3 : bool, optional 

1252 Flag this call to run() as using the Gen3 butler environment. 

1253 strayLightData : `object`, optional 

1254 Opaque object containing calibration information for stray-light 

1255 correction. If `None`, no correction will be performed. 

1256 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1257 Illumination correction image. 

1258 

1259 Returns 

1260 ------- 

1261 result : `lsst.pipe.base.Struct` 

1262 Result struct with component: 

1263 - ``exposure`` : `afw.image.Exposure` 

1264 The fully ISR corrected exposure. 

1265 - ``outputExposure`` : `afw.image.Exposure` 

1266 An alias for `exposure` 

1267 - ``ossThumb`` : `numpy.ndarray` 

1268 Thumbnail image of the exposure after overscan subtraction. 

1269 - ``flattenedThumb`` : `numpy.ndarray` 

1270 Thumbnail image of the exposure after flat-field correction. 

1271 

1272 Raises 

1273 ------ 

1274 RuntimeError 

1275 Raised if a configuration option is set to True, but the 

1276 required calibration data has not been specified. 

1277 

1278 Notes 

1279 ----- 

1280 The current processed exposure can be viewed by setting the 

1281 appropriate lsstDebug entries in the `debug.display` 

1282 dictionary. The names of these entries correspond to some of 

1283 the IsrTaskConfig Boolean options, with the value denoting the 

1284 frame to use. The exposure is shown inside the matching 

1285 option check and after the processing of that step has 

1286 finished. The steps with debug points are: 

1287 

1288 doAssembleCcd 

1289 doBias 

1290 doCrosstalk 

1291 doBrighterFatter 

1292 doDark 

1293 doFringe 

1294 doStrayLight 

1295 doFlat 

1296 

1297 In addition, setting the "postISRCCD" entry displays the 

1298 exposure after all ISR processing has finished. 

1299 

1300 """ 

1301 

1302 if isGen3 is True: 

1303 # Gen3 currently cannot automatically do configuration overrides. 

1304 # DM-15257 looks to discuss this issue. 

1305 # Configure input exposures; 

1306 if detectorNum is None: 

1307 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1308 

1309 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1310 bias = self.ensureExposure(bias, camera, detectorNum) 

1311 dark = self.ensureExposure(dark, camera, detectorNum) 

1312 flat = self.ensureExposure(flat, camera, detectorNum) 

1313 else: 

1314 if isinstance(ccdExposure, ButlerDataRef): 

1315 return self.runDataRef(ccdExposure) 

1316 

1317 ccd = ccdExposure.getDetector() 

1318 # TODO DM-28093: change this to: ccdExposure.getFilterLabel().physicalLabel 

1319 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1320 

1321 if not ccd: 

1322 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1323 ccd = [FakeAmp(ccdExposure, self.config)] 

1324 

1325 # Validate Input 

1326 if self.config.doBias and bias is None: 

1327 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1328 if self.doLinearize(ccd) and linearizer is None: 

1329 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1330 if self.config.doBrighterFatter and bfKernel is None: 

1331 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1332 if self.config.doDark and dark is None: 

1333 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1334 if self.config.doFlat and flat is None: 

1335 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1336 if self.config.doDefect and defects is None: 

1337 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1338 if (self.config.doFringe and filterName in self.fringe.config.filters 

1339 and fringes.fringes is None): 

1340 # The `fringes` object needs to be a pipeBase.Struct, as 

1341 # we use it as a `dict` for the parameters of 

1342 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1343 # not be `None` if `doFringe=True`. Otherwise, raise. 

1344 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1345 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters 

1346 and illumMaskedImage is None): 

1347 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1348 

1349 # Begin ISR processing. 

1350 if self.config.doConvertIntToFloat: 

1351 self.log.info("Converting exposure to floating point values.") 

1352 ccdExposure = self.convertIntToFloat(ccdExposure) 

1353 

1354 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1355 self.log.info("Applying bias correction.") 

1356 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1357 trimToFit=self.config.doTrimToMatchCalib) 

1358 self.debugView(ccdExposure, "doBias") 

1359 

1360 # Amplifier level processing. 

1361 overscans = [] 

1362 for amp in ccd: 

1363 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1364 if ccdExposure.getBBox().contains(amp.getBBox()): 

1365 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1366 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1367 

1368 if self.config.doOverscan and not badAmp: 

1369 # Overscan correction on amp-by-amp basis. 

1370 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1371 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1372 if overscanResults is not None and \ 

1373 self.config.qa is not None and self.config.qa.saveStats is True: 

1374 if isinstance(overscanResults.overscanFit, float): 

1375 qaMedian = overscanResults.overscanFit 

1376 qaStdev = float("NaN") 

1377 else: 

1378 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1379 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1380 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1381 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1382 

1383 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian) 

1384 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev) 

1385 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1386 amp.getName(), qaMedian, qaStdev) 

1387 

1388 # Residuals after overscan correction 

1389 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1390 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1391 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1392 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1393 

1394 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter) 

1395 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter) 

1396 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1397 amp.getName(), qaMedianAfter, qaStdevAfter) 

1398 

1399 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1400 else: 

1401 if badAmp: 

1402 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1403 overscanResults = None 

1404 

1405 overscans.append(overscanResults if overscanResults is not None else None) 

1406 else: 

1407 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1408 

1409 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1410 self.log.info("Applying crosstalk correction.") 

1411 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1412 crosstalkSources=crosstalkSources) 

1413 self.debugView(ccdExposure, "doCrosstalk") 

1414 

1415 if self.config.doAssembleCcd: 

1416 self.log.info("Assembling CCD from amplifiers.") 

1417 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1418 

1419 if self.config.expectWcs and not ccdExposure.getWcs(): 

1420 self.log.warn("No WCS found in input exposure.") 

1421 self.debugView(ccdExposure, "doAssembleCcd") 

1422 

1423 ossThumb = None 

1424 if self.config.qa.doThumbnailOss: 

1425 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1426 

1427 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1428 self.log.info("Applying bias correction.") 

1429 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1430 trimToFit=self.config.doTrimToMatchCalib) 

1431 self.debugView(ccdExposure, "doBias") 

1432 

1433 if self.config.doVariance: 

1434 for amp, overscanResults in zip(ccd, overscans): 

1435 if ccdExposure.getBBox().contains(amp.getBBox()): 

1436 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1437 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1438 if overscanResults is not None: 

1439 self.updateVariance(ampExposure, amp, 

1440 overscanImage=overscanResults.overscanImage) 

1441 else: 

1442 self.updateVariance(ampExposure, amp, 

1443 overscanImage=None) 

1444 if self.config.qa is not None and self.config.qa.saveStats is True: 

1445 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1446 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1447 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1448 qaStats.getValue(afwMath.MEDIAN)) 

1449 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1450 qaStats.getValue(afwMath.STDEVCLIP)) 

1451 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1452 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1453 qaStats.getValue(afwMath.STDEVCLIP)) 

1454 

1455 if self.doLinearize(ccd): 

1456 self.log.info("Applying linearizer.") 

1457 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1458 detector=ccd, log=self.log) 

1459 

1460 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1461 self.log.info("Applying crosstalk correction.") 

1462 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1463 crosstalkSources=crosstalkSources, isTrimmed=True) 

1464 self.debugView(ccdExposure, "doCrosstalk") 

1465 

1466 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1467 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1468 if self.config.doDefect: 

1469 self.log.info("Masking defects.") 

1470 self.maskDefect(ccdExposure, defects) 

1471 

1472 if self.config.numEdgeSuspect > 0: 

1473 self.log.info("Masking edges as SUSPECT.") 

1474 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1475 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1476 

1477 if self.config.doNanMasking: 

1478 self.log.info("Masking NAN value pixels.") 

1479 self.maskNan(ccdExposure) 

1480 

1481 if self.config.doWidenSaturationTrails: 

1482 self.log.info("Widening saturation trails.") 

1483 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1484 

1485 if self.config.doCameraSpecificMasking: 

1486 self.log.info("Masking regions for camera specific reasons.") 

1487 self.masking.run(ccdExposure) 

1488 

1489 if self.config.doBrighterFatter: 

1490 # We need to apply flats and darks before we can interpolate, and we 

1491 # need to interpolate before we do B-F, but we do B-F without the 

1492 # flats and darks applied so we can work in units of electrons or holes. 

1493 # This context manager applies and then removes the darks and flats. 

1494 # 

1495 # We also do not want to interpolate values here, so operate on temporary 

1496 # images so we can apply only the BF-correction and roll back the 

1497 # interpolation. 

1498 interpExp = ccdExposure.clone() 

1499 with self.flatContext(interpExp, flat, dark): 

1500 isrFunctions.interpolateFromMask( 

1501 maskedImage=interpExp.getMaskedImage(), 

1502 fwhm=self.config.fwhm, 

1503 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1504 maskNameList=self.config.maskListToInterpolate 

1505 ) 

1506 bfExp = interpExp.clone() 

1507 

1508 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1509 type(bfKernel), type(bfGains)) 

1510 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1511 self.config.brighterFatterMaxIter, 

1512 self.config.brighterFatterThreshold, 

1513 self.config.brighterFatterApplyGain, 

1514 bfGains) 

1515 if bfResults[1] == self.config.brighterFatterMaxIter: 

1516 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1517 bfResults[0]) 

1518 else: 

1519 self.log.info("Finished brighter fatter correction in %d iterations.", 

1520 bfResults[1]) 

1521 image = ccdExposure.getMaskedImage().getImage() 

1522 bfCorr = bfExp.getMaskedImage().getImage() 

1523 bfCorr -= interpExp.getMaskedImage().getImage() 

1524 image += bfCorr 

1525 

1526 # Applying the brighter-fatter correction applies a 

1527 # convolution to the science image. At the edges this 

1528 # convolution may not have sufficient valid pixels to 

1529 # produce a valid correction. Mark pixels within the size 

1530 # of the brighter-fatter kernel as EDGE to warn of this 

1531 # fact. 

1532 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1533 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1534 maskPlane="EDGE") 

1535 

1536 if self.config.brighterFatterMaskGrowSize > 0: 

1537 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1538 for maskPlane in self.config.maskListToInterpolate: 

1539 isrFunctions.growMasks(ccdExposure.getMask(), 

1540 radius=self.config.brighterFatterMaskGrowSize, 

1541 maskNameList=maskPlane, 

1542 maskValue=maskPlane) 

1543 

1544 self.debugView(ccdExposure, "doBrighterFatter") 

1545 

1546 if self.config.doDark: 

1547 self.log.info("Applying dark correction.") 

1548 self.darkCorrection(ccdExposure, dark) 

1549 self.debugView(ccdExposure, "doDark") 

1550 

1551 if self.config.doFringe and not self.config.fringeAfterFlat: 

1552 self.log.info("Applying fringe correction before flat.") 

1553 self.fringe.run(ccdExposure, **fringes.getDict()) 

1554 self.debugView(ccdExposure, "doFringe") 

1555 

1556 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1557 self.log.info("Checking strayLight correction.") 

1558 self.strayLight.run(ccdExposure, strayLightData) 

1559 self.debugView(ccdExposure, "doStrayLight") 

1560 

1561 if self.config.doFlat: 

1562 self.log.info("Applying flat correction.") 

1563 self.flatCorrection(ccdExposure, flat) 

1564 self.debugView(ccdExposure, "doFlat") 

1565 

1566 if self.config.doApplyGains: 

1567 self.log.info("Applying gain correction instead of flat.") 

1568 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1569 

1570 if self.config.doFringe and self.config.fringeAfterFlat: 

1571 self.log.info("Applying fringe correction after flat.") 

1572 self.fringe.run(ccdExposure, **fringes.getDict()) 

1573 

1574 if self.config.doVignette: 

1575 self.log.info("Constructing Vignette polygon.") 

1576 self.vignettePolygon = self.vignette.run(ccdExposure) 

1577 

1578 if self.config.vignette.doWriteVignettePolygon: 

1579 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1580 

1581 if self.config.doAttachTransmissionCurve: 

1582 self.log.info("Adding transmission curves.") 

1583 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1584 filterTransmission=filterTransmission, 

1585 sensorTransmission=sensorTransmission, 

1586 atmosphereTransmission=atmosphereTransmission) 

1587 

1588 flattenedThumb = None 

1589 if self.config.qa.doThumbnailFlattened: 

1590 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1591 

1592 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1593 self.log.info("Performing illumination correction.") 

1594 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1595 illumMaskedImage, illumScale=self.config.illumScale, 

1596 trimToFit=self.config.doTrimToMatchCalib) 

1597 

1598 preInterpExp = None 

1599 if self.config.doSaveInterpPixels: 

1600 preInterpExp = ccdExposure.clone() 

1601 

1602 # Reset and interpolate bad pixels. 

1603 # 

1604 # Large contiguous bad regions (which should have the BAD mask 

1605 # bit set) should have their values set to the image median. 

1606 # This group should include defects and bad amplifiers. As the 

1607 # area covered by these defects are large, there's little 

1608 # reason to expect that interpolation would provide a more 

1609 # useful value. 

1610 # 

1611 # Smaller defects can be safely interpolated after the larger 

1612 # regions have had their pixel values reset. This ensures 

1613 # that the remaining defects adjacent to bad amplifiers (as an 

1614 # example) do not attempt to interpolate extreme values. 

1615 if self.config.doSetBadRegions: 

1616 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1617 if badPixelCount > 0: 

1618 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1619 

1620 if self.config.doInterpolate: 

1621 self.log.info("Interpolating masked pixels.") 

1622 isrFunctions.interpolateFromMask( 

1623 maskedImage=ccdExposure.getMaskedImage(), 

1624 fwhm=self.config.fwhm, 

1625 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1626 maskNameList=list(self.config.maskListToInterpolate) 

1627 ) 

1628 

1629 self.roughZeroPoint(ccdExposure) 

1630 

1631 if self.config.doMeasureBackground: 

1632 self.log.info("Measuring background level.") 

1633 self.measureBackground(ccdExposure, self.config.qa) 

1634 

1635 if self.config.qa is not None and self.config.qa.saveStats is True: 

1636 for amp in ccd: 

1637 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1638 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1639 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1640 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1641 qaStats.getValue(afwMath.MEDIAN)) 

1642 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1643 qaStats.getValue(afwMath.STDEVCLIP)) 

1644 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1645 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1646 qaStats.getValue(afwMath.STDEVCLIP)) 

1647 

1648 self.debugView(ccdExposure, "postISRCCD") 

1649 

1650 return pipeBase.Struct( 

1651 exposure=ccdExposure, 

1652 ossThumb=ossThumb, 

1653 flattenedThumb=flattenedThumb, 

1654 

1655 preInterpolatedExposure=preInterpExp, 

1656 outputExposure=ccdExposure, 

1657 outputOssThumbnail=ossThumb, 

1658 outputFlattenedThumbnail=flattenedThumb, 

1659 ) 

1660 

1661 @pipeBase.timeMethod 

1662 def runDataRef(self, sensorRef): 

1663 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1664 

1665 This method contains the `CmdLineTask` interface to the ISR 

1666 processing. All IO is handled here, freeing the `run()` method 

1667 to manage only pixel-level calculations. The steps performed 

1668 are: 

1669 - Read in necessary detrending/isr/calibration data. 

1670 - Process raw exposure in `run()`. 

1671 - Persist the ISR-corrected exposure as "postISRCCD" if 

1672 config.doWrite=True. 

1673 

1674 Parameters 

1675 ---------- 

1676 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1677 DataRef of the detector data to be processed 

1678 

1679 Returns 

1680 ------- 

1681 result : `lsst.pipe.base.Struct` 

1682 Result struct with component: 

1683 - ``exposure`` : `afw.image.Exposure` 

1684 The fully ISR corrected exposure. 

1685 

1686 Raises 

1687 ------ 

1688 RuntimeError 

1689 Raised if a configuration option is set to True, but the 

1690 required calibration data does not exist. 

1691 

1692 """ 

1693 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1694 

1695 ccdExposure = sensorRef.get(self.config.datasetType) 

1696 

1697 camera = sensorRef.get("camera") 

1698 isrData = self.readIsrData(sensorRef, ccdExposure) 

1699 

1700 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1701 

1702 if self.config.doWrite: 

1703 sensorRef.put(result.exposure, "postISRCCD") 

1704 if result.preInterpolatedExposure is not None: 

1705 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1706 if result.ossThumb is not None: 

1707 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1708 if result.flattenedThumb is not None: 

1709 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1710 

1711 return result 

1712 

1713 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1714 """Retrieve a calibration dataset for removing instrument signature. 

1715 

1716 Parameters 

1717 ---------- 

1718 

1719 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1720 DataRef of the detector data to find calibration datasets 

1721 for. 

1722 datasetType : `str` 

1723 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1724 dateObs : `str`, optional 

1725 Date of the observation. Used to correct butler failures 

1726 when using fallback filters. 

1727 immediate : `Bool` 

1728 If True, disable butler proxies to enable error handling 

1729 within this routine. 

1730 

1731 Returns 

1732 ------- 

1733 exposure : `lsst.afw.image.Exposure` 

1734 Requested calibration frame. 

1735 

1736 Raises 

1737 ------ 

1738 RuntimeError 

1739 Raised if no matching calibration frame can be found. 

1740 """ 

1741 try: 

1742 exp = dataRef.get(datasetType, immediate=immediate) 

1743 except Exception as exc1: 

1744 if not self.config.fallbackFilterName: 

1745 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1746 try: 

1747 if self.config.useFallbackDate and dateObs: 

1748 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1749 dateObs=dateObs, immediate=immediate) 

1750 else: 

1751 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1752 except Exception as exc2: 

1753 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1754 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1755 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1756 

1757 if self.config.doAssembleIsrExposures: 

1758 exp = self.assembleCcd.assembleCcd(exp) 

1759 return exp 

1760 

1761 def ensureExposure(self, inputExp, camera, detectorNum): 

1762 """Ensure that the data returned by Butler is a fully constructed exposure. 

1763 

1764 ISR requires exposure-level image data for historical reasons, so if we did 

1765 not recieve that from Butler, construct it from what we have, modifying the 

1766 input in place. 

1767 

1768 Parameters 

1769 ---------- 

1770 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1771 `lsst.afw.image.ImageF` 

1772 The input data structure obtained from Butler. 

1773 camera : `lsst.afw.cameraGeom.camera` 

1774 The camera associated with the image. Used to find the appropriate 

1775 detector. 

1776 detectorNum : `int` 

1777 The detector this exposure should match. 

1778 

1779 Returns 

1780 ------- 

1781 inputExp : `lsst.afw.image.Exposure` 

1782 The re-constructed exposure, with appropriate detector parameters. 

1783 

1784 Raises 

1785 ------ 

1786 TypeError 

1787 Raised if the input data cannot be used to construct an exposure. 

1788 """ 

1789 if isinstance(inputExp, afwImage.DecoratedImageU): 

1790 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1791 elif isinstance(inputExp, afwImage.ImageF): 

1792 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1793 elif isinstance(inputExp, afwImage.MaskedImageF): 

1794 inputExp = afwImage.makeExposure(inputExp) 

1795 elif isinstance(inputExp, afwImage.Exposure): 

1796 pass 

1797 elif inputExp is None: 

1798 # Assume this will be caught by the setup if it is a problem. 

1799 return inputExp 

1800 else: 

1801 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1802 (type(inputExp), )) 

1803 

1804 if inputExp.getDetector() is None: 

1805 inputExp.setDetector(camera[detectorNum]) 

1806 

1807 return inputExp 

1808 

1809 def convertIntToFloat(self, exposure): 

1810 """Convert exposure image from uint16 to float. 

1811 

1812 If the exposure does not need to be converted, the input is 

1813 immediately returned. For exposures that are converted to use 

1814 floating point pixels, the variance is set to unity and the 

1815 mask to zero. 

1816 

1817 Parameters 

1818 ---------- 

1819 exposure : `lsst.afw.image.Exposure` 

1820 The raw exposure to be converted. 

1821 

1822 Returns 

1823 ------- 

1824 newexposure : `lsst.afw.image.Exposure` 

1825 The input ``exposure``, converted to floating point pixels. 

1826 

1827 Raises 

1828 ------ 

1829 RuntimeError 

1830 Raised if the exposure type cannot be converted to float. 

1831 

1832 """ 

1833 if isinstance(exposure, afwImage.ExposureF): 

1834 # Nothing to be done 

1835 self.log.debug("Exposure already of type float.") 

1836 return exposure 

1837 if not hasattr(exposure, "convertF"): 

1838 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1839 

1840 newexposure = exposure.convertF() 

1841 newexposure.variance[:] = 1 

1842 newexposure.mask[:] = 0x0 

1843 

1844 return newexposure 

1845 

1846 def maskAmplifier(self, ccdExposure, amp, defects): 

1847 """Identify bad amplifiers, saturated and suspect pixels. 

1848 

1849 Parameters 

1850 ---------- 

1851 ccdExposure : `lsst.afw.image.Exposure` 

1852 Input exposure to be masked. 

1853 amp : `lsst.afw.table.AmpInfoCatalog` 

1854 Catalog of parameters defining the amplifier on this 

1855 exposure to mask. 

1856 defects : `lsst.ip.isr.Defects` 

1857 List of defects. Used to determine if the entire 

1858 amplifier is bad. 

1859 

1860 Returns 

1861 ------- 

1862 badAmp : `Bool` 

1863 If this is true, the entire amplifier area is covered by 

1864 defects and unusable. 

1865 

1866 """ 

1867 maskedImage = ccdExposure.getMaskedImage() 

1868 

1869 badAmp = False 

1870 

1871 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1872 # comparison with current defects definition. 

1873 if defects is not None: 

1874 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1875 

1876 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1877 # association with pixels in current ccdExposure). 

1878 if badAmp: 

1879 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1880 afwImage.PARENT) 

1881 maskView = dataView.getMask() 

1882 maskView |= maskView.getPlaneBitMask("BAD") 

1883 del maskView 

1884 return badAmp 

1885 

1886 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1887 # Saturation and suspect pixels can be masked now, though. 

1888 limits = dict() 

1889 if self.config.doSaturation and not badAmp: 

1890 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1891 if self.config.doSuspect and not badAmp: 

1892 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1893 if math.isfinite(self.config.saturation): 

1894 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1895 

1896 for maskName, maskThreshold in limits.items(): 

1897 if not math.isnan(maskThreshold): 

1898 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1899 isrFunctions.makeThresholdMask( 

1900 maskedImage=dataView, 

1901 threshold=maskThreshold, 

1902 growFootprints=0, 

1903 maskName=maskName 

1904 ) 

1905 

1906 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1907 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1908 afwImage.PARENT) 

1909 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1910 self.config.suspectMaskName]) 

1911 if numpy.all(maskView.getArray() & maskVal > 0): 

1912 badAmp = True 

1913 maskView |= maskView.getPlaneBitMask("BAD") 

1914 

1915 return badAmp 

1916 

1917 def overscanCorrection(self, ccdExposure, amp): 

1918 """Apply overscan correction in place. 

1919 

1920 This method does initial pixel rejection of the overscan 

1921 region. The overscan can also be optionally segmented to 

1922 allow for discontinuous overscan responses to be fit 

1923 separately. The actual overscan subtraction is performed by 

1924 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1925 which is called here after the amplifier is preprocessed. 

1926 

1927 Parameters 

1928 ---------- 

1929 ccdExposure : `lsst.afw.image.Exposure` 

1930 Exposure to have overscan correction performed. 

1931 amp : `lsst.afw.cameraGeom.Amplifer` 

1932 The amplifier to consider while correcting the overscan. 

1933 

1934 Returns 

1935 ------- 

1936 overscanResults : `lsst.pipe.base.Struct` 

1937 Result struct with components: 

1938 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1939 Value or fit subtracted from the amplifier image data. 

1940 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1941 Value or fit subtracted from the overscan image data. 

1942 - ``overscanImage`` : `lsst.afw.image.Image` 

1943 Image of the overscan region with the overscan 

1944 correction applied. This quantity is used to estimate 

1945 the amplifier read noise empirically. 

1946 

1947 Raises 

1948 ------ 

1949 RuntimeError 

1950 Raised if the ``amp`` does not contain raw pixel information. 

1951 

1952 See Also 

1953 -------- 

1954 lsst.ip.isr.isrFunctions.overscanCorrection 

1955 """ 

1956 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1957 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1958 return None 

1959 

1960 statControl = afwMath.StatisticsControl() 

1961 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1962 

1963 # Determine the bounding boxes 

1964 dataBBox = amp.getRawDataBBox() 

1965 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1966 dx0 = 0 

1967 dx1 = 0 

1968 

1969 prescanBBox = amp.getRawPrescanBBox() 

1970 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1971 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1972 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1973 else: 

1974 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1975 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1976 

1977 # Determine if we need to work on subregions of the amplifier and overscan. 

1978 imageBBoxes = [] 

1979 overscanBBoxes = [] 

1980 

1981 if ((self.config.overscanBiasJump 

1982 and self.config.overscanBiasJumpLocation) 

1983 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

1984 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1985 self.config.overscanBiasJumpDevices)): 

1986 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1987 yLower = self.config.overscanBiasJumpLocation 

1988 yUpper = dataBBox.getHeight() - yLower 

1989 else: 

1990 yUpper = self.config.overscanBiasJumpLocation 

1991 yLower = dataBBox.getHeight() - yUpper 

1992 

1993 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1994 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1995 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1996 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1997 yLower))) 

1998 

1999 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

2000 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

2001 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

2002 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2003 yUpper))) 

2004 else: 

2005 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2006 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

2007 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2008 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2009 oscanBBox.getHeight()))) 

2010 

2011 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

2012 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

2013 ampImage = ccdExposure.maskedImage[imageBBox] 

2014 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2015 

2016 overscanArray = overscanImage.image.array 

2017 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2018 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2019 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2020 

2021 statControl = afwMath.StatisticsControl() 

2022 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2023 

2024 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2025 

2026 # Measure average overscan levels and record them in the metadata. 

2027 levelStat = afwMath.MEDIAN 

2028 sigmaStat = afwMath.STDEVCLIP 

2029 

2030 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2031 self.config.qa.flatness.nIter) 

2032 metadata = ccdExposure.getMetadata() 

2033 ampNum = amp.getName() 

2034 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2035 if isinstance(overscanResults.overscanFit, float): 

2036 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2037 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2038 else: 

2039 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2040 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2041 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2042 

2043 return overscanResults 

2044 

2045 def updateVariance(self, ampExposure, amp, overscanImage=None): 

2046 """Set the variance plane using the amplifier gain and read noise 

2047 

2048 The read noise is calculated from the ``overscanImage`` if the 

2049 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2050 the value from the amplifier data is used. 

2051 

2052 Parameters 

2053 ---------- 

2054 ampExposure : `lsst.afw.image.Exposure` 

2055 Exposure to process. 

2056 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2057 Amplifier detector data. 

2058 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2059 Image of overscan, required only for empirical read noise. 

2060 

2061 See also 

2062 -------- 

2063 lsst.ip.isr.isrFunctions.updateVariance 

2064 """ 

2065 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2066 gain = amp.getGain() 

2067 

2068 if math.isnan(gain): 

2069 gain = 1.0 

2070 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2071 elif gain <= 0: 

2072 patchedGain = 1.0 

2073 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

2074 amp.getName(), gain, patchedGain) 

2075 gain = patchedGain 

2076 

2077 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2078 self.log.info("Overscan is none for EmpiricalReadNoise.") 

2079 

2080 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2081 stats = afwMath.StatisticsControl() 

2082 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2083 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2084 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2085 amp.getName(), readNoise) 

2086 else: 

2087 readNoise = amp.getReadNoise() 

2088 

2089 isrFunctions.updateVariance( 

2090 maskedImage=ampExposure.getMaskedImage(), 

2091 gain=gain, 

2092 readNoise=readNoise, 

2093 ) 

2094 

2095 def darkCorrection(self, exposure, darkExposure, invert=False): 

2096 """Apply dark correction in place. 

2097 

2098 Parameters 

2099 ---------- 

2100 exposure : `lsst.afw.image.Exposure` 

2101 Exposure to process. 

2102 darkExposure : `lsst.afw.image.Exposure` 

2103 Dark exposure of the same size as ``exposure``. 

2104 invert : `Bool`, optional 

2105 If True, re-add the dark to an already corrected image. 

2106 

2107 Raises 

2108 ------ 

2109 RuntimeError 

2110 Raised if either ``exposure`` or ``darkExposure`` do not 

2111 have their dark time defined. 

2112 

2113 See Also 

2114 -------- 

2115 lsst.ip.isr.isrFunctions.darkCorrection 

2116 """ 

2117 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2118 if math.isnan(expScale): 

2119 raise RuntimeError("Exposure darktime is NAN.") 

2120 if darkExposure.getInfo().getVisitInfo() is not None \ 

2121 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2122 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2123 else: 

2124 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2125 # so getDarkTime() does not exist. 

2126 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2127 darkScale = 1.0 

2128 

2129 isrFunctions.darkCorrection( 

2130 maskedImage=exposure.getMaskedImage(), 

2131 darkMaskedImage=darkExposure.getMaskedImage(), 

2132 expScale=expScale, 

2133 darkScale=darkScale, 

2134 invert=invert, 

2135 trimToFit=self.config.doTrimToMatchCalib 

2136 ) 

2137 

2138 def doLinearize(self, detector): 

2139 """Check if linearization is needed for the detector cameraGeom. 

2140 

2141 Checks config.doLinearize and the linearity type of the first 

2142 amplifier. 

2143 

2144 Parameters 

2145 ---------- 

2146 detector : `lsst.afw.cameraGeom.Detector` 

2147 Detector to get linearity type from. 

2148 

2149 Returns 

2150 ------- 

2151 doLinearize : `Bool` 

2152 If True, linearization should be performed. 

2153 """ 

2154 return self.config.doLinearize and \ 

2155 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2156 

2157 def flatCorrection(self, exposure, flatExposure, invert=False): 

2158 """Apply flat correction in place. 

2159 

2160 Parameters 

2161 ---------- 

2162 exposure : `lsst.afw.image.Exposure` 

2163 Exposure to process. 

2164 flatExposure : `lsst.afw.image.Exposure` 

2165 Flat exposure of the same size as ``exposure``. 

2166 invert : `Bool`, optional 

2167 If True, unflatten an already flattened image. 

2168 

2169 See Also 

2170 -------- 

2171 lsst.ip.isr.isrFunctions.flatCorrection 

2172 """ 

2173 isrFunctions.flatCorrection( 

2174 maskedImage=exposure.getMaskedImage(), 

2175 flatMaskedImage=flatExposure.getMaskedImage(), 

2176 scalingType=self.config.flatScalingType, 

2177 userScale=self.config.flatUserScale, 

2178 invert=invert, 

2179 trimToFit=self.config.doTrimToMatchCalib 

2180 ) 

2181 

2182 def saturationDetection(self, exposure, amp): 

2183 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2184 

2185 Parameters 

2186 ---------- 

2187 exposure : `lsst.afw.image.Exposure` 

2188 Exposure to process. Only the amplifier DataSec is processed. 

2189 amp : `lsst.afw.table.AmpInfoCatalog` 

2190 Amplifier detector data. 

2191 

2192 See Also 

2193 -------- 

2194 lsst.ip.isr.isrFunctions.makeThresholdMask 

2195 """ 

2196 if not math.isnan(amp.getSaturation()): 

2197 maskedImage = exposure.getMaskedImage() 

2198 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2199 isrFunctions.makeThresholdMask( 

2200 maskedImage=dataView, 

2201 threshold=amp.getSaturation(), 

2202 growFootprints=0, 

2203 maskName=self.config.saturatedMaskName, 

2204 ) 

2205 

2206 def saturationInterpolation(self, exposure): 

2207 """Interpolate over saturated pixels, in place. 

2208 

2209 This method should be called after `saturationDetection`, to 

2210 ensure that the saturated pixels have been identified in the 

2211 SAT mask. It should also be called after `assembleCcd`, since 

2212 saturated regions may cross amplifier boundaries. 

2213 

2214 Parameters 

2215 ---------- 

2216 exposure : `lsst.afw.image.Exposure` 

2217 Exposure to process. 

2218 

2219 See Also 

2220 -------- 

2221 lsst.ip.isr.isrTask.saturationDetection 

2222 lsst.ip.isr.isrFunctions.interpolateFromMask 

2223 """ 

2224 isrFunctions.interpolateFromMask( 

2225 maskedImage=exposure.getMaskedImage(), 

2226 fwhm=self.config.fwhm, 

2227 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2228 maskNameList=list(self.config.saturatedMaskName), 

2229 ) 

2230 

2231 def suspectDetection(self, exposure, amp): 

2232 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2233 

2234 Parameters 

2235 ---------- 

2236 exposure : `lsst.afw.image.Exposure` 

2237 Exposure to process. Only the amplifier DataSec is processed. 

2238 amp : `lsst.afw.table.AmpInfoCatalog` 

2239 Amplifier detector data. 

2240 

2241 See Also 

2242 -------- 

2243 lsst.ip.isr.isrFunctions.makeThresholdMask 

2244 

2245 Notes 

2246 ----- 

2247 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2248 This is intended to indicate pixels that may be affected by unknown systematics; 

2249 for example if non-linearity corrections above a certain level are unstable 

2250 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2251 that no such level exists and no pixels are to be masked as suspicious. 

2252 """ 

2253 suspectLevel = amp.getSuspectLevel() 

2254 if math.isnan(suspectLevel): 

2255 return 

2256 

2257 maskedImage = exposure.getMaskedImage() 

2258 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2259 isrFunctions.makeThresholdMask( 

2260 maskedImage=dataView, 

2261 threshold=suspectLevel, 

2262 growFootprints=0, 

2263 maskName=self.config.suspectMaskName, 

2264 ) 

2265 

2266 def maskDefect(self, exposure, defectBaseList): 

2267 """Mask defects using mask plane "BAD", in place. 

2268 

2269 Parameters 

2270 ---------- 

2271 exposure : `lsst.afw.image.Exposure` 

2272 Exposure to process. 

2273 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2274 `lsst.afw.image.DefectBase`. 

2275 List of defects to mask. 

2276 

2277 Notes 

2278 ----- 

2279 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2280 """ 

2281 maskedImage = exposure.getMaskedImage() 

2282 if not isinstance(defectBaseList, Defects): 

2283 # Promotes DefectBase to Defect 

2284 defectList = Defects(defectBaseList) 

2285 else: 

2286 defectList = defectBaseList 

2287 defectList.maskPixels(maskedImage, maskName="BAD") 

2288 

2289 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2290 """Mask edge pixels with applicable mask plane. 

2291 

2292 Parameters 

2293 ---------- 

2294 exposure : `lsst.afw.image.Exposure` 

2295 Exposure to process. 

2296 numEdgePixels : `int`, optional 

2297 Number of edge pixels to mask. 

2298 maskPlane : `str`, optional 

2299 Mask plane name to use. 

2300 level : `str`, optional 

2301 Level at which to mask edges. 

2302 """ 

2303 maskedImage = exposure.getMaskedImage() 

2304 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2305 

2306 if numEdgePixels > 0: 

2307 if level == 'DETECTOR': 

2308 boxes = [maskedImage.getBBox()] 

2309 elif level == 'AMP': 

2310 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2311 

2312 for box in boxes: 

2313 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2314 subImage = maskedImage[box] 

2315 box.grow(-numEdgePixels) 

2316 # Mask pixels outside box 

2317 SourceDetectionTask.setEdgeBits( 

2318 subImage, 

2319 box, 

2320 maskBitMask) 

2321 

2322 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2323 """Mask and interpolate defects using mask plane "BAD", in place. 

2324 

2325 Parameters 

2326 ---------- 

2327 exposure : `lsst.afw.image.Exposure` 

2328 Exposure to process. 

2329 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2330 `lsst.afw.image.DefectBase`. 

2331 List of defects to mask and interpolate. 

2332 

2333 See Also 

2334 -------- 

2335 lsst.ip.isr.isrTask.maskDefect 

2336 """ 

2337 self.maskDefect(exposure, defectBaseList) 

2338 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2339 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2340 isrFunctions.interpolateFromMask( 

2341 maskedImage=exposure.getMaskedImage(), 

2342 fwhm=self.config.fwhm, 

2343 growSaturatedFootprints=0, 

2344 maskNameList=["BAD"], 

2345 ) 

2346 

2347 def maskNan(self, exposure): 

2348 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2349 

2350 Parameters 

2351 ---------- 

2352 exposure : `lsst.afw.image.Exposure` 

2353 Exposure to process. 

2354 

2355 Notes 

2356 ----- 

2357 We mask over all NaNs, including those that are masked with 

2358 other bits (because those may or may not be interpolated over 

2359 later, and we want to remove all NaNs). Despite this 

2360 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2361 the historical name. 

2362 """ 

2363 maskedImage = exposure.getMaskedImage() 

2364 

2365 # Find and mask NaNs 

2366 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2367 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2368 numNans = maskNans(maskedImage, maskVal) 

2369 self.metadata.set("NUMNANS", numNans) 

2370 if numNans > 0: 

2371 self.log.warn("There were %d unmasked NaNs.", numNans) 

2372 

2373 def maskAndInterpolateNan(self, exposure): 

2374 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2375 

2376 Parameters 

2377 ---------- 

2378 exposure : `lsst.afw.image.Exposure` 

2379 Exposure to process. 

2380 

2381 See Also 

2382 -------- 

2383 lsst.ip.isr.isrTask.maskNan 

2384 """ 

2385 self.maskNan(exposure) 

2386 isrFunctions.interpolateFromMask( 

2387 maskedImage=exposure.getMaskedImage(), 

2388 fwhm=self.config.fwhm, 

2389 growSaturatedFootprints=0, 

2390 maskNameList=["UNMASKEDNAN"], 

2391 ) 

2392 

2393 def measureBackground(self, exposure, IsrQaConfig=None): 

2394 """Measure the image background in subgrids, for quality control purposes. 

2395 

2396 Parameters 

2397 ---------- 

2398 exposure : `lsst.afw.image.Exposure` 

2399 Exposure to process. 

2400 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2401 Configuration object containing parameters on which background 

2402 statistics and subgrids to use. 

2403 """ 

2404 if IsrQaConfig is not None: 

2405 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2406 IsrQaConfig.flatness.nIter) 

2407 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2408 statsControl.setAndMask(maskVal) 

2409 maskedImage = exposure.getMaskedImage() 

2410 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2411 skyLevel = stats.getValue(afwMath.MEDIAN) 

2412 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2413 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2414 metadata = exposure.getMetadata() 

2415 metadata.set('SKYLEVEL', skyLevel) 

2416 metadata.set('SKYSIGMA', skySigma) 

2417 

2418 # calcluating flatlevel over the subgrids 

2419 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2420 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2421 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2422 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2423 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2424 skyLevels = numpy.zeros((nX, nY)) 

2425 

2426 for j in range(nY): 

2427 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2428 for i in range(nX): 

2429 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2430 

2431 xLLC = xc - meshXHalf 

2432 yLLC = yc - meshYHalf 

2433 xURC = xc + meshXHalf - 1 

2434 yURC = yc + meshYHalf - 1 

2435 

2436 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2437 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2438 

2439 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2440 

2441 good = numpy.where(numpy.isfinite(skyLevels)) 

2442 skyMedian = numpy.median(skyLevels[good]) 

2443 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2444 flatness_rms = numpy.std(flatness) 

2445 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2446 

2447 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2448 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2449 nX, nY, flatness_pp, flatness_rms) 

2450 

2451 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2452 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2453 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2454 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2455 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2456 

2457 def roughZeroPoint(self, exposure): 

2458 """Set an approximate magnitude zero point for the exposure. 

2459 

2460 Parameters 

2461 ---------- 

2462 exposure : `lsst.afw.image.Exposure` 

2463 Exposure to process. 

2464 """ 

2465 # TODO DM-28093: change this to: exposure.getFilterLabel().physicalLabel 

2466 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2467 if filterName in self.config.fluxMag0T1: 

2468 fluxMag0 = self.config.fluxMag0T1[filterName] 

2469 else: 

2470 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2471 fluxMag0 = self.config.defaultFluxMag0T1 

2472 

2473 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2474 if not expTime > 0: # handle NaN as well as <= 0 

2475 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2476 return 

2477 

2478 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2479 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2480 

2481 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2482 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2483 

2484 Parameters 

2485 ---------- 

2486 ccdExposure : `lsst.afw.image.Exposure` 

2487 Exposure to process. 

2488 fpPolygon : `lsst.afw.geom.Polygon` 

2489 Polygon in focal plane coordinates. 

2490 """ 

2491 # Get ccd corners in focal plane coordinates 

2492 ccd = ccdExposure.getDetector() 

2493 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2494 ccdPolygon = Polygon(fpCorners) 

2495 

2496 # Get intersection of ccd corners with fpPolygon 

2497 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2498 

2499 # Transform back to pixel positions and build new polygon 

2500 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2501 validPolygon = Polygon(ccdPoints) 

2502 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2503 

2504 @contextmanager 

2505 def flatContext(self, exp, flat, dark=None): 

2506 """Context manager that applies and removes flats and darks, 

2507 if the task is configured to apply them. 

2508 

2509 Parameters 

2510 ---------- 

2511 exp : `lsst.afw.image.Exposure` 

2512 Exposure to process. 

2513 flat : `lsst.afw.image.Exposure` 

2514 Flat exposure the same size as ``exp``. 

2515 dark : `lsst.afw.image.Exposure`, optional 

2516 Dark exposure the same size as ``exp``. 

2517 

2518 Yields 

2519 ------ 

2520 exp : `lsst.afw.image.Exposure` 

2521 The flat and dark corrected exposure. 

2522 """ 

2523 if self.config.doDark and dark is not None: 

2524 self.darkCorrection(exp, dark) 

2525 if self.config.doFlat: 

2526 self.flatCorrection(exp, flat) 

2527 try: 

2528 yield exp 

2529 finally: 

2530 if self.config.doFlat: 

2531 self.flatCorrection(exp, flat, invert=True) 

2532 if self.config.doDark and dark is not None: 

2533 self.darkCorrection(exp, dark, invert=True) 

2534 

2535 def debugView(self, exposure, stepname): 

2536 """Utility function to examine ISR exposure at different stages. 

2537 

2538 Parameters 

2539 ---------- 

2540 exposure : `lsst.afw.image.Exposure` 

2541 Exposure to view. 

2542 stepname : `str` 

2543 State of processing to view. 

2544 """ 

2545 frame = getDebugFrame(self._display, stepname) 

2546 if frame: 

2547 display = getDisplay(frame) 

2548 display.scale('asinh', 'zscale') 

2549 display.mtv(exposure) 

2550 prompt = "Press Enter to continue [c]... " 

2551 while True: 

2552 ans = input(prompt).lower() 

2553 if ans in ("", "c",): 

2554 break 

2555 

2556 

2557class FakeAmp(object): 

2558 """A Detector-like object that supports returning gain and saturation level 

2559 

2560 This is used when the input exposure does not have a detector. 

2561 

2562 Parameters 

2563 ---------- 

2564 exposure : `lsst.afw.image.Exposure` 

2565 Exposure to generate a fake amplifier for. 

2566 config : `lsst.ip.isr.isrTaskConfig` 

2567 Configuration to apply to the fake amplifier. 

2568 """ 

2569 

2570 def __init__(self, exposure, config): 

2571 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2572 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2573 self._gain = config.gain 

2574 self._readNoise = config.readNoise 

2575 self._saturation = config.saturation 

2576 

2577 def getBBox(self): 

2578 return self._bbox 

2579 

2580 def getRawBBox(self): 

2581 return self._bbox 

2582 

2583 def getRawHorizontalOverscanBBox(self): 

2584 return self._RawHorizontalOverscanBBox 

2585 

2586 def getGain(self): 

2587 return self._gain 

2588 

2589 def getReadNoise(self): 

2590 return self._readNoise 

2591 

2592 def getSaturation(self): 

2593 return self._saturation 

2594 

2595 def getSuspectLevel(self): 

2596 return float("NaN") 

2597 

2598 

2599class RunIsrConfig(pexConfig.Config): 

2600 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2601 

2602 

2603class RunIsrTask(pipeBase.CmdLineTask): 

2604 """Task to wrap the default IsrTask to allow it to be retargeted. 

2605 

2606 The standard IsrTask can be called directly from a command line 

2607 program, but doing so removes the ability of the task to be 

2608 retargeted. As most cameras override some set of the IsrTask 

2609 methods, this would remove those data-specific methods in the 

2610 output post-ISR images. This wrapping class fixes the issue, 

2611 allowing identical post-ISR images to be generated by both the 

2612 processCcd and isrTask code. 

2613 """ 

2614 ConfigClass = RunIsrConfig 

2615 _DefaultName = "runIsr" 

2616 

2617 def __init__(self, *args, **kwargs): 

2618 super().__init__(*args, **kwargs) 

2619 self.makeSubtask("isr") 

2620 

2621 def runDataRef(self, dataRef): 

2622 """ 

2623 Parameters 

2624 ---------- 

2625 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2626 data reference of the detector data to be processed 

2627 

2628 Returns 

2629 ------- 

2630 result : `pipeBase.Struct` 

2631 Result struct with component: 

2632 

2633 - exposure : `lsst.afw.image.Exposure` 

2634 Post-ISR processed exposure. 

2635 """ 

2636 return self.isr.runDataRef(dataRef)