Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42from lsst.meas.algorithms import Defects 

43 

44from . import isrFunctions 

45from . import isrQa 

46from . import linearize 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55 

56 

57__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

58 

59 

60class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

61 dimensions={"instrument", "exposure", "detector"}, 

62 defaultTemplates={}): 

63 ccdExposure = cT.Input( 

64 name="raw", 

65 doc="Input exposure to process.", 

66 storageClass="Exposure", 

67 dimensions=["instrument", "detector", "exposure"], 

68 ) 

69 camera = cT.PrerequisiteInput( 

70 name="camera", 

71 storageClass="Camera", 

72 doc="Input camera to construct complete exposures.", 

73 dimensions=["instrument", "calibration_label"], 

74 ) 

75 bias = cT.PrerequisiteInput( 

76 name="bias", 

77 doc="Input bias calibration.", 

78 storageClass="ExposureF", 

79 dimensions=["instrument", "calibration_label", "detector"], 

80 ) 

81 dark = cT.PrerequisiteInput( 

82 name='dark', 

83 doc="Input dark calibration.", 

84 storageClass="ExposureF", 

85 dimensions=["instrument", "calibration_label", "detector"], 

86 ) 

87 flat = cT.PrerequisiteInput( 

88 name="flat", 

89 doc="Input flat calibration.", 

90 storageClass="ExposureF", 

91 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

92 ) 

93 fringes = cT.PrerequisiteInput( 

94 name="fringe", 

95 doc="Input fringe calibration.", 

96 storageClass="ExposureF", 

97 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

98 ) 

99 strayLightData = cT.PrerequisiteInput( 

100 name='yBackground', 

101 doc="Input stray light calibration.", 

102 storageClass="StrayLightData", 

103 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

104 ) 

105 bfKernel = cT.PrerequisiteInput( 

106 name='bfKernel', 

107 doc="Input brighter-fatter kernel.", 

108 storageClass="NumpyArray", 

109 dimensions=["instrument", "calibration_label"], 

110 ) 

111 newBFKernel = cT.PrerequisiteInput( 

112 name='brighterFatterKernel', 

113 doc="Newer complete kernel + gain solutions.", 

114 storageClass="BrighterFatterKernel", 

115 dimensions=["instrument", "calibration_label", "detector"], 

116 ) 

117 defects = cT.PrerequisiteInput( 

118 name='defects', 

119 doc="Input defect tables.", 

120 storageClass="Defects", 

121 dimensions=["instrument", "calibration_label", "detector"], 

122 ) 

123 opticsTransmission = cT.PrerequisiteInput( 

124 name="transmission_optics", 

125 storageClass="TransmissionCurve", 

126 doc="Transmission curve due to the optics.", 

127 dimensions=["instrument", "calibration_label"], 

128 ) 

129 filterTransmission = cT.PrerequisiteInput( 

130 name="transmission_filter", 

131 storageClass="TransmissionCurve", 

132 doc="Transmission curve due to the filter.", 

133 dimensions=["instrument", "physical_filter", "calibration_label"], 

134 ) 

135 sensorTransmission = cT.PrerequisiteInput( 

136 name="transmission_sensor", 

137 storageClass="TransmissionCurve", 

138 doc="Transmission curve due to the sensor.", 

139 dimensions=["instrument", "calibration_label", "detector"], 

140 ) 

141 atmosphereTransmission = cT.PrerequisiteInput( 

142 name="transmission_atmosphere", 

143 storageClass="TransmissionCurve", 

144 doc="Transmission curve due to the atmosphere.", 

145 dimensions=["instrument"], 

146 ) 

147 illumMaskedImage = cT.PrerequisiteInput( 

148 name="illum", 

149 doc="Input illumination correction.", 

150 storageClass="MaskedImageF", 

151 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

152 ) 

153 

154 outputExposure = cT.Output( 

155 name='postISRCCD', 

156 doc="Output ISR processed exposure.", 

157 storageClass="ExposureF", 

158 dimensions=["instrument", "exposure", "detector"], 

159 ) 

160 preInterpExposure = cT.Output( 

161 name='preInterpISRCCD', 

162 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "exposure", "detector"], 

165 ) 

166 outputOssThumbnail = cT.Output( 

167 name="OssThumb", 

168 doc="Output Overscan-subtracted thumbnail image.", 

169 storageClass="Thumbnail", 

170 dimensions=["instrument", "exposure", "detector"], 

171 ) 

172 outputFlattenedThumbnail = cT.Output( 

173 name="FlattenedThumb", 

174 doc="Output flat-corrected thumbnail image.", 

175 storageClass="Thumbnail", 

176 dimensions=["instrument", "exposure", "detector"], 

177 ) 

178 

179 def __init__(self, *, config=None): 

180 super().__init__(config=config) 

181 

182 if config.doBias is not True: 

183 self.prerequisiteInputs.discard("bias") 

184 if config.doLinearize is not True: 

185 self.prerequisiteInputs.discard("linearizer") 

186 if config.doCrosstalk is not True: 

187 self.prerequisiteInputs.discard("crosstalkSources") 

188 if config.doBrighterFatter is not True: 

189 self.prerequisiteInputs.discard("bfKernel") 

190 self.prerequisiteInputs.discard("newBFKernel") 

191 if config.doDefect is not True: 

192 self.prerequisiteInputs.discard("defects") 

193 if config.doDark is not True: 

194 self.prerequisiteInputs.discard("dark") 

195 if config.doFlat is not True: 

196 self.prerequisiteInputs.discard("flat") 

197 if config.doAttachTransmissionCurve is not True: 

198 self.prerequisiteInputs.discard("opticsTransmission") 

199 self.prerequisiteInputs.discard("filterTransmission") 

200 self.prerequisiteInputs.discard("sensorTransmission") 

201 self.prerequisiteInputs.discard("atmosphereTransmission") 

202 if config.doUseOpticsTransmission is not True: 

203 self.prerequisiteInputs.discard("opticsTransmission") 

204 if config.doUseFilterTransmission is not True: 

205 self.prerequisiteInputs.discard("filterTransmission") 

206 if config.doUseSensorTransmission is not True: 

207 self.prerequisiteInputs.discard("sensorTransmission") 

208 if config.doUseAtmosphereTransmission is not True: 

209 self.prerequisiteInputs.discard("atmosphereTransmission") 

210 if config.doIlluminationCorrection is not True: 

211 self.prerequisiteInputs.discard("illumMaskedImage") 

212 

213 if config.doWrite is not True: 

214 self.outputs.discard("outputExposure") 

215 self.outputs.discard("preInterpExposure") 

216 self.outputs.discard("outputFlattenedThumbnail") 

217 self.outputs.discard("outputOssThumbnail") 

218 if config.doSaveInterpPixels is not True: 

219 self.outputs.discard("preInterpExposure") 

220 if config.qa.doThumbnailOss is not True: 

221 self.outputs.discard("outputOssThumbnail") 

222 if config.qa.doThumbnailFlattened is not True: 

223 self.outputs.discard("outputFlattenedThumbnail") 

224 

225 

226class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

227 pipelineConnections=IsrTaskConnections): 

228 """Configuration parameters for IsrTask. 

229 

230 Items are grouped in the order in which they are executed by the task. 

231 """ 

232 datasetType = pexConfig.Field( 

233 dtype=str, 

234 doc="Dataset type for input data; users will typically leave this alone, " 

235 "but camera-specific ISR tasks will override it", 

236 default="raw", 

237 ) 

238 

239 fallbackFilterName = pexConfig.Field( 

240 dtype=str, 

241 doc="Fallback default filter name for calibrations.", 

242 optional=True 

243 ) 

244 useFallbackDate = pexConfig.Field( 

245 dtype=bool, 

246 doc="Pass observation date when using fallback filter.", 

247 default=False, 

248 ) 

249 expectWcs = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

253 ) 

254 fwhm = pexConfig.Field( 

255 dtype=float, 

256 doc="FWHM of PSF in arcseconds.", 

257 default=1.0, 

258 ) 

259 qa = pexConfig.ConfigField( 

260 dtype=isrQa.IsrQaConfig, 

261 doc="QA related configuration options.", 

262 ) 

263 

264 # Image conversion configuration 

265 doConvertIntToFloat = pexConfig.Field( 

266 dtype=bool, 

267 doc="Convert integer raw images to floating point values?", 

268 default=True, 

269 ) 

270 

271 # Saturated pixel handling. 

272 doSaturation = pexConfig.Field( 

273 dtype=bool, 

274 doc="Mask saturated pixels? NB: this is totally independent of the" 

275 " interpolation option - this is ONLY setting the bits in the mask." 

276 " To have them interpolated make sure doSaturationInterpolation=True", 

277 default=True, 

278 ) 

279 saturatedMaskName = pexConfig.Field( 

280 dtype=str, 

281 doc="Name of mask plane to use in saturation detection and interpolation", 

282 default="SAT", 

283 ) 

284 saturation = pexConfig.Field( 

285 dtype=float, 

286 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

287 default=float("NaN"), 

288 ) 

289 growSaturationFootprintSize = pexConfig.Field( 

290 dtype=int, 

291 doc="Number of pixels by which to grow the saturation footprints", 

292 default=1, 

293 ) 

294 

295 # Suspect pixel handling. 

296 doSuspect = pexConfig.Field( 

297 dtype=bool, 

298 doc="Mask suspect pixels?", 

299 default=False, 

300 ) 

301 suspectMaskName = pexConfig.Field( 

302 dtype=str, 

303 doc="Name of mask plane to use for suspect pixels", 

304 default="SUSPECT", 

305 ) 

306 numEdgeSuspect = pexConfig.Field( 

307 dtype=int, 

308 doc="Number of edge pixels to be flagged as untrustworthy.", 

309 default=0, 

310 ) 

311 

312 # Initial masking options. 

313 doSetBadRegions = pexConfig.Field( 

314 dtype=bool, 

315 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

316 default=True, 

317 ) 

318 badStatistic = pexConfig.ChoiceField( 

319 dtype=str, 

320 doc="How to estimate the average value for BAD regions.", 

321 default='MEANCLIP', 

322 allowed={ 

323 "MEANCLIP": "Correct using the (clipped) mean of good data", 

324 "MEDIAN": "Correct using the median of the good data", 

325 }, 

326 ) 

327 

328 # Overscan subtraction configuration. 

329 doOverscan = pexConfig.Field( 

330 dtype=bool, 

331 doc="Do overscan subtraction?", 

332 default=True, 

333 ) 

334 overscanFitType = pexConfig.ChoiceField( 

335 dtype=str, 

336 doc="The method for fitting the overscan bias level.", 

337 default='MEDIAN', 

338 allowed={ 

339 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

340 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

341 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

342 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

343 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

344 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

345 "MEAN": "Correct using the mean of the overscan region", 

346 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

347 "MEDIAN": "Correct using the median of the overscan region", 

348 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

349 }, 

350 ) 

351 overscanOrder = pexConfig.Field( 

352 dtype=int, 

353 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " + 

354 "or number of spline knots if overscan fit type is a spline."), 

355 default=1, 

356 ) 

357 overscanNumSigmaClip = pexConfig.Field( 

358 dtype=float, 

359 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

360 default=3.0, 

361 ) 

362 overscanIsInt = pexConfig.Field( 

363 dtype=bool, 

364 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" + 

365 " and overscan.FitType=MEDIAN_PER_ROW.", 

366 default=True, 

367 ) 

368 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

369 dtype=int, 

370 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

371 default=0, 

372 ) 

373 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

374 dtype=int, 

375 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

376 default=0, 

377 ) 

378 overscanMaxDev = pexConfig.Field( 378 ↛ exitline 378 didn't jump to the function exit

379 dtype=float, 

380 doc="Maximum deviation from the median for overscan", 

381 default=1000.0, check=lambda x: x > 0 

382 ) 

383 overscanBiasJump = pexConfig.Field( 

384 dtype=bool, 

385 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

386 default=False, 

387 ) 

388 overscanBiasJumpKeyword = pexConfig.Field( 

389 dtype=str, 

390 doc="Header keyword containing information about devices.", 

391 default="NO_SUCH_KEY", 

392 ) 

393 overscanBiasJumpDevices = pexConfig.ListField( 

394 dtype=str, 

395 doc="List of devices that need piecewise overscan correction.", 

396 default=(), 

397 ) 

398 overscanBiasJumpLocation = pexConfig.Field( 

399 dtype=int, 

400 doc="Location of bias jump along y-axis.", 

401 default=0, 

402 ) 

403 

404 # Amplifier to CCD assembly configuration 

405 doAssembleCcd = pexConfig.Field( 

406 dtype=bool, 

407 default=True, 

408 doc="Assemble amp-level exposures into a ccd-level exposure?" 

409 ) 

410 assembleCcd = pexConfig.ConfigurableField( 

411 target=AssembleCcdTask, 

412 doc="CCD assembly task", 

413 ) 

414 

415 # General calibration configuration. 

416 doAssembleIsrExposures = pexConfig.Field( 

417 dtype=bool, 

418 default=False, 

419 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

420 ) 

421 doTrimToMatchCalib = pexConfig.Field( 

422 dtype=bool, 

423 default=False, 

424 doc="Trim raw data to match calibration bounding boxes?" 

425 ) 

426 

427 # Bias subtraction. 

428 doBias = pexConfig.Field( 

429 dtype=bool, 

430 doc="Apply bias frame correction?", 

431 default=True, 

432 ) 

433 biasDataProductName = pexConfig.Field( 

434 dtype=str, 

435 doc="Name of the bias data product", 

436 default="bias", 

437 ) 

438 

439 # Variance construction 

440 doVariance = pexConfig.Field( 

441 dtype=bool, 

442 doc="Calculate variance?", 

443 default=True 

444 ) 

445 gain = pexConfig.Field( 

446 dtype=float, 

447 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

448 default=float("NaN"), 

449 ) 

450 readNoise = pexConfig.Field( 

451 dtype=float, 

452 doc="The read noise to use if no Detector is present in the Exposure", 

453 default=0.0, 

454 ) 

455 doEmpiricalReadNoise = pexConfig.Field( 

456 dtype=bool, 

457 default=False, 

458 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

459 ) 

460 

461 # Linearization. 

462 doLinearize = pexConfig.Field( 

463 dtype=bool, 

464 doc="Correct for nonlinearity of the detector's response?", 

465 default=True, 

466 ) 

467 

468 # Crosstalk. 

469 doCrosstalk = pexConfig.Field( 

470 dtype=bool, 

471 doc="Apply intra-CCD crosstalk correction?", 

472 default=False, 

473 ) 

474 doCrosstalkBeforeAssemble = pexConfig.Field( 

475 dtype=bool, 

476 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

477 default=False, 

478 ) 

479 crosstalk = pexConfig.ConfigurableField( 

480 target=CrosstalkTask, 

481 doc="Intra-CCD crosstalk correction", 

482 ) 

483 

484 # Masking options. 

485 doDefect = pexConfig.Field( 

486 dtype=bool, 

487 doc="Apply correction for CCD defects, e.g. hot pixels?", 

488 default=True, 

489 ) 

490 doNanMasking = pexConfig.Field( 

491 dtype=bool, 

492 doc="Mask NAN pixels?", 

493 default=True, 

494 ) 

495 doWidenSaturationTrails = pexConfig.Field( 

496 dtype=bool, 

497 doc="Widen bleed trails based on their width?", 

498 default=True 

499 ) 

500 

501 # Brighter-Fatter correction. 

502 doBrighterFatter = pexConfig.Field( 

503 dtype=bool, 

504 default=False, 

505 doc="Apply the brighter fatter correction" 

506 ) 

507 brighterFatterLevel = pexConfig.ChoiceField( 

508 dtype=str, 

509 default="DETECTOR", 

510 doc="The level at which to correct for brighter-fatter.", 

511 allowed={ 

512 "AMP": "Every amplifier treated separately.", 

513 "DETECTOR": "One kernel per detector", 

514 } 

515 ) 

516 brighterFatterMaxIter = pexConfig.Field( 

517 dtype=int, 

518 default=10, 

519 doc="Maximum number of iterations for the brighter fatter correction" 

520 ) 

521 brighterFatterThreshold = pexConfig.Field( 

522 dtype=float, 

523 default=1000, 

524 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

525 " absolute value of the difference between the current corrected image and the one" 

526 " from the previous iteration summed over all the pixels." 

527 ) 

528 brighterFatterApplyGain = pexConfig.Field( 

529 dtype=bool, 

530 default=True, 

531 doc="Should the gain be applied when applying the brighter fatter correction?" 

532 ) 

533 brighterFatterMaskGrowSize = pexConfig.Field( 

534 dtype=int, 

535 default=0, 

536 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

537 " when brighter-fatter correction is applied." 

538 ) 

539 

540 # Dark subtraction. 

541 doDark = pexConfig.Field( 

542 dtype=bool, 

543 doc="Apply dark frame correction?", 

544 default=True, 

545 ) 

546 darkDataProductName = pexConfig.Field( 

547 dtype=str, 

548 doc="Name of the dark data product", 

549 default="dark", 

550 ) 

551 

552 # Camera-specific stray light removal. 

553 doStrayLight = pexConfig.Field( 

554 dtype=bool, 

555 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

556 default=False, 

557 ) 

558 strayLight = pexConfig.ConfigurableField( 

559 target=StrayLightTask, 

560 doc="y-band stray light correction" 

561 ) 

562 

563 # Flat correction. 

564 doFlat = pexConfig.Field( 

565 dtype=bool, 

566 doc="Apply flat field correction?", 

567 default=True, 

568 ) 

569 flatDataProductName = pexConfig.Field( 

570 dtype=str, 

571 doc="Name of the flat data product", 

572 default="flat", 

573 ) 

574 flatScalingType = pexConfig.ChoiceField( 

575 dtype=str, 

576 doc="The method for scaling the flat on the fly.", 

577 default='USER', 

578 allowed={ 

579 "USER": "Scale by flatUserScale", 

580 "MEAN": "Scale by the inverse of the mean", 

581 "MEDIAN": "Scale by the inverse of the median", 

582 }, 

583 ) 

584 flatUserScale = pexConfig.Field( 

585 dtype=float, 

586 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

587 default=1.0, 

588 ) 

589 doTweakFlat = pexConfig.Field( 

590 dtype=bool, 

591 doc="Tweak flats to match observed amplifier ratios?", 

592 default=False 

593 ) 

594 

595 # Amplifier normalization based on gains instead of using flats configuration. 

596 doApplyGains = pexConfig.Field( 

597 dtype=bool, 

598 doc="Correct the amplifiers for their gains instead of applying flat correction", 

599 default=False, 

600 ) 

601 normalizeGains = pexConfig.Field( 

602 dtype=bool, 

603 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

604 default=False, 

605 ) 

606 

607 # Fringe correction. 

608 doFringe = pexConfig.Field( 

609 dtype=bool, 

610 doc="Apply fringe correction?", 

611 default=True, 

612 ) 

613 fringe = pexConfig.ConfigurableField( 

614 target=FringeTask, 

615 doc="Fringe subtraction task", 

616 ) 

617 fringeAfterFlat = pexConfig.Field( 

618 dtype=bool, 

619 doc="Do fringe subtraction after flat-fielding?", 

620 default=True, 

621 ) 

622 

623 # Distortion model application. 

624 doAddDistortionModel = pexConfig.Field( 

625 dtype=bool, 

626 doc="Apply a distortion model based on camera geometry to the WCS?", 

627 default=True, 

628 deprecated=("Camera geometry is incorporated when reading the raw files." 

629 " This option no longer is used, and will be removed after v19.") 

630 ) 

631 

632 # Initial CCD-level background statistics options. 

633 doMeasureBackground = pexConfig.Field( 

634 dtype=bool, 

635 doc="Measure the background level on the reduced image?", 

636 default=False, 

637 ) 

638 

639 # Camera-specific masking configuration. 

640 doCameraSpecificMasking = pexConfig.Field( 

641 dtype=bool, 

642 doc="Mask camera-specific bad regions?", 

643 default=False, 

644 ) 

645 masking = pexConfig.ConfigurableField( 

646 target=MaskingTask, 

647 doc="Masking task." 

648 ) 

649 

650 # Interpolation options. 

651 

652 doInterpolate = pexConfig.Field( 

653 dtype=bool, 

654 doc="Interpolate masked pixels?", 

655 default=True, 

656 ) 

657 doSaturationInterpolation = pexConfig.Field( 

658 dtype=bool, 

659 doc="Perform interpolation over pixels masked as saturated?" 

660 " NB: This is independent of doSaturation; if that is False this plane" 

661 " will likely be blank, resulting in a no-op here.", 

662 default=True, 

663 ) 

664 doNanInterpolation = pexConfig.Field( 

665 dtype=bool, 

666 doc="Perform interpolation over pixels masked as NaN?" 

667 " NB: This is independent of doNanMasking; if that is False this plane" 

668 " will likely be blank, resulting in a no-op here.", 

669 default=True, 

670 ) 

671 doNanInterpAfterFlat = pexConfig.Field( 

672 dtype=bool, 

673 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

674 "also have to interpolate them before flat-fielding."), 

675 default=False, 

676 ) 

677 maskListToInterpolate = pexConfig.ListField( 

678 dtype=str, 

679 doc="List of mask planes that should be interpolated.", 

680 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

681 ) 

682 doSaveInterpPixels = pexConfig.Field( 

683 dtype=bool, 

684 doc="Save a copy of the pre-interpolated pixel values?", 

685 default=False, 

686 ) 

687 

688 # Default photometric calibration options. 

689 fluxMag0T1 = pexConfig.DictField( 

690 keytype=str, 

691 itemtype=float, 

692 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

693 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

694 )) 

695 ) 

696 defaultFluxMag0T1 = pexConfig.Field( 

697 dtype=float, 

698 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

699 default=pow(10.0, 0.4*28.0) 

700 ) 

701 

702 # Vignette correction configuration. 

703 doVignette = pexConfig.Field( 

704 dtype=bool, 

705 doc="Apply vignetting parameters?", 

706 default=False, 

707 ) 

708 vignette = pexConfig.ConfigurableField( 

709 target=VignetteTask, 

710 doc="Vignetting task.", 

711 ) 

712 

713 # Transmission curve configuration. 

714 doAttachTransmissionCurve = pexConfig.Field( 

715 dtype=bool, 

716 default=False, 

717 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

718 ) 

719 doUseOpticsTransmission = pexConfig.Field( 

720 dtype=bool, 

721 default=True, 

722 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

723 ) 

724 doUseFilterTransmission = pexConfig.Field( 

725 dtype=bool, 

726 default=True, 

727 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

728 ) 

729 doUseSensorTransmission = pexConfig.Field( 

730 dtype=bool, 

731 default=True, 

732 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

733 ) 

734 doUseAtmosphereTransmission = pexConfig.Field( 

735 dtype=bool, 

736 default=True, 

737 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

738 ) 

739 

740 # Illumination correction. 

741 doIlluminationCorrection = pexConfig.Field( 

742 dtype=bool, 

743 default=False, 

744 doc="Perform illumination correction?" 

745 ) 

746 illuminationCorrectionDataProductName = pexConfig.Field( 

747 dtype=str, 

748 doc="Name of the illumination correction data product.", 

749 default="illumcor", 

750 ) 

751 illumScale = pexConfig.Field( 

752 dtype=float, 

753 doc="Scale factor for the illumination correction.", 

754 default=1.0, 

755 ) 

756 illumFilters = pexConfig.ListField( 

757 dtype=str, 

758 default=[], 

759 doc="Only perform illumination correction for these filters." 

760 ) 

761 

762 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

763 doWrite = pexConfig.Field( 

764 dtype=bool, 

765 doc="Persist postISRCCD?", 

766 default=True, 

767 ) 

768 

769 def validate(self): 

770 super().validate() 

771 if self.doFlat and self.doApplyGains: 

772 raise ValueError("You may not specify both doFlat and doApplyGains") 

773 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate: 

774 self.config.maskListToInterpolate.append("SAT") 

775 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

776 self.config.maskListToInterpolate.append("UNMASKEDNAN") 

777 

778 

779class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

780 """Apply common instrument signature correction algorithms to a raw frame. 

781 

782 The process for correcting imaging data is very similar from 

783 camera to camera. This task provides a vanilla implementation of 

784 doing these corrections, including the ability to turn certain 

785 corrections off if they are not needed. The inputs to the primary 

786 method, `run()`, are a raw exposure to be corrected and the 

787 calibration data products. The raw input is a single chip sized 

788 mosaic of all amps including overscans and other non-science 

789 pixels. The method `runDataRef()` identifies and defines the 

790 calibration data products, and is intended for use by a 

791 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

792 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

793 subclassed for different camera, although the most camera specific 

794 methods have been split into subtasks that can be redirected 

795 appropriately. 

796 

797 The __init__ method sets up the subtasks for ISR processing, using 

798 the defaults from `lsst.ip.isr`. 

799 

800 Parameters 

801 ---------- 

802 args : `list` 

803 Positional arguments passed to the Task constructor. None used at this time. 

804 kwargs : `dict`, optional 

805 Keyword arguments passed on to the Task constructor. None used at this time. 

806 """ 

807 ConfigClass = IsrTaskConfig 

808 _DefaultName = "isr" 

809 

810 def __init__(self, **kwargs): 

811 super().__init__(**kwargs) 

812 self.makeSubtask("assembleCcd") 

813 self.makeSubtask("crosstalk") 

814 self.makeSubtask("strayLight") 

815 self.makeSubtask("fringe") 

816 self.makeSubtask("masking") 

817 self.makeSubtask("vignette") 

818 

819 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

820 inputs = butlerQC.get(inputRefs) 

821 

822 try: 

823 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

824 except Exception as e: 

825 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

826 (inputRefs, e)) 

827 

828 inputs['isGen3'] = True 

829 

830 detector = inputs['ccdExposure'].getDetector() 

831 

832 if self.doLinearize(detector) is True: 

833 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict): 

834 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

835 linearizer.fromYaml(inputs['linearizer']) 

836 else: 

837 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector, 

838 log=self.log) 

839 inputs['linearizer'] = linearizer 

840 

841 if self.config.doDefect is True: 

842 if "defects" in inputs and inputs['defects'] is not None: 

843 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

844 # masking expects a list of defects defined by their bounding box 

845 if not isinstance(inputs["defects"], Defects): 

846 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

847 

848 # Load the correct style of brighter fatter kernel, and repack 

849 # the information as a numpy array. 

850 if self.config.doBrighterFatter: 

851 brighterFatterKernel = inputs.pop('newBFKernel', None) 

852 if brighterFatterKernel is None: 

853 brighterFatterKernel = inputs.get('bfKernel', None) 

854 

855 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

856 detId = detector.getId() 

857 inputs['bfGains'] = brighterFatterKernel.gain 

858 # If the kernel is not an ndarray, it's the cp_pipe version 

859 # so extract the kernel for this detector, or raise an error 

860 if self.config.brighterFatterLevel == 'DETECTOR': 

861 if brighterFatterKernel.detectorKernel: 

862 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

863 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

864 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

865 else: 

866 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

867 else: 

868 # TODO DM-15631 for implementing this 

869 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

870 

871 # Broken: DM-17169 

872 # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be 

873 # fixed for non-HSC cameras in the future. 

874 # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure']) 

875 # if self.config.doCrosstalk else None) 

876 

877 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

878 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

879 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

880 expId=expId, 

881 assembler=self.assembleCcd 

882 if self.config.doAssembleIsrExposures else None) 

883 else: 

884 inputs['fringes'] = pipeBase.Struct(fringes=None) 

885 

886 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

887 if 'strayLightData' not in inputs: 

888 inputs['strayLightData'] = None 

889 

890 outputs = self.run(**inputs) 

891 butlerQC.put(outputs, outputRefs) 

892 

893 def readIsrData(self, dataRef, rawExposure): 

894 """!Retrieve necessary frames for instrument signature removal. 

895 

896 Pre-fetching all required ISR data products limits the IO 

897 required by the ISR. Any conflict between the calibration data 

898 available and that needed for ISR is also detected prior to 

899 doing processing, allowing it to fail quickly. 

900 

901 Parameters 

902 ---------- 

903 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

904 Butler reference of the detector data to be processed 

905 rawExposure : `afw.image.Exposure` 

906 The raw exposure that will later be corrected with the 

907 retrieved calibration data; should not be modified in this 

908 method. 

909 

910 Returns 

911 ------- 

912 result : `lsst.pipe.base.Struct` 

913 Result struct with components (which may be `None`): 

914 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

915 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

916 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

917 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

918 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

919 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

920 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 

921 - ``fringes``: `lsst.pipe.base.Struct` with components: 

922 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

923 - ``seed``: random seed derived from the ccdExposureId for random 

924 number generator (`uint32`). 

925 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

926 A ``TransmissionCurve`` that represents the throughput of the optics, 

927 to be evaluated in focal-plane coordinates. 

928 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

929 A ``TransmissionCurve`` that represents the throughput of the filter 

930 itself, to be evaluated in focal-plane coordinates. 

931 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

932 A ``TransmissionCurve`` that represents the throughput of the sensor 

933 itself, to be evaluated in post-assembly trimmed detector coordinates. 

934 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

935 A ``TransmissionCurve`` that represents the throughput of the 

936 atmosphere, assumed to be spatially constant. 

937 - ``strayLightData`` : `object` 

938 An opaque object containing calibration information for 

939 stray-light correction. If `None`, no correction will be 

940 performed. 

941 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

942 

943 Raises 

944 ------ 

945 NotImplementedError : 

946 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

947 """ 

948 try: 

949 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

950 dateObs = dateObs.toPython().isoformat() 

951 except RuntimeError: 

952 self.log.warn("Unable to identify dateObs for rawExposure.") 

953 dateObs = None 

954 

955 ccd = rawExposure.getDetector() 

956 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

957 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

958 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

959 if self.config.doBias else None) 

960 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

961 linearizer = (dataRef.get("linearizer", immediate=True) 

962 if self.doLinearize(ccd) else None) 

963 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

964 linearizer.log = self.log 

965 if isinstance(linearizer, numpy.ndarray): 

966 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

967 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef) 

968 if self.config.doCrosstalk else None) 

969 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

970 if self.config.doDark else None) 

971 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

972 dateObs=dateObs) 

973 if self.config.doFlat else None) 

974 

975 brighterFatterKernel = None 

976 brighterFatterGains = None 

977 if self.config.doBrighterFatter is True: 

978 try: 

979 # Use the new-style cp_pipe version of the kernel if it exists 

980 # If using a new-style kernel, always use the self-consistent 

981 # gains, i.e. the ones inside the kernel object itself 

982 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

983 brighterFatterGains = brighterFatterKernel.gain 

984 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

985 except NoResults: 

986 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

987 brighterFatterKernel = dataRef.get("bfKernel") 

988 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

989 except NoResults: 

990 brighterFatterKernel = None 

991 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

992 # If the kernel is not an ndarray, it's the cp_pipe version 

993 # so extract the kernel for this detector, or raise an error 

994 if self.config.brighterFatterLevel == 'DETECTOR': 

995 if brighterFatterKernel.detectorKernel: 

996 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

997 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

998 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

999 else: 

1000 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1001 else: 

1002 # TODO DM-15631 for implementing this 

1003 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1004 

1005 defectList = (dataRef.get("defects") 

1006 if self.config.doDefect else None) 

1007 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

1008 if self.config.doAssembleIsrExposures else None) 

1009 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1010 else pipeBase.Struct(fringes=None)) 

1011 

1012 if self.config.doAttachTransmissionCurve: 

1013 opticsTransmission = (dataRef.get("transmission_optics") 

1014 if self.config.doUseOpticsTransmission else None) 

1015 filterTransmission = (dataRef.get("transmission_filter") 

1016 if self.config.doUseFilterTransmission else None) 

1017 sensorTransmission = (dataRef.get("transmission_sensor") 

1018 if self.config.doUseSensorTransmission else None) 

1019 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1020 if self.config.doUseAtmosphereTransmission else None) 

1021 else: 

1022 opticsTransmission = None 

1023 filterTransmission = None 

1024 sensorTransmission = None 

1025 atmosphereTransmission = None 

1026 

1027 if self.config.doStrayLight: 

1028 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1029 else: 

1030 strayLightData = None 

1031 

1032 illumMaskedImage = (self.getIsrExposure(dataRef, 

1033 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1034 if (self.config.doIlluminationCorrection and 

1035 filterName in self.config.illumFilters) 

1036 else None) 

1037 

1038 # Struct should include only kwargs to run() 

1039 return pipeBase.Struct(bias=biasExposure, 

1040 linearizer=linearizer, 

1041 crosstalkSources=crosstalkSources, 

1042 dark=darkExposure, 

1043 flat=flatExposure, 

1044 bfKernel=brighterFatterKernel, 

1045 bfGains=brighterFatterGains, 

1046 defects=defectList, 

1047 fringes=fringeStruct, 

1048 opticsTransmission=opticsTransmission, 

1049 filterTransmission=filterTransmission, 

1050 sensorTransmission=sensorTransmission, 

1051 atmosphereTransmission=atmosphereTransmission, 

1052 strayLightData=strayLightData, 

1053 illumMaskedImage=illumMaskedImage 

1054 ) 

1055 

1056 @pipeBase.timeMethod 

1057 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, 

1058 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1059 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1060 sensorTransmission=None, atmosphereTransmission=None, 

1061 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1062 isGen3=False, 

1063 ): 

1064 """!Perform instrument signature removal on an exposure. 

1065 

1066 Steps included in the ISR processing, in order performed, are: 

1067 - saturation and suspect pixel masking 

1068 - overscan subtraction 

1069 - CCD assembly of individual amplifiers 

1070 - bias subtraction 

1071 - variance image construction 

1072 - linearization of non-linear response 

1073 - crosstalk masking 

1074 - brighter-fatter correction 

1075 - dark subtraction 

1076 - fringe correction 

1077 - stray light subtraction 

1078 - flat correction 

1079 - masking of known defects and camera specific features 

1080 - vignette calculation 

1081 - appending transmission curve and distortion model 

1082 

1083 Parameters 

1084 ---------- 

1085 ccdExposure : `lsst.afw.image.Exposure` 

1086 The raw exposure that is to be run through ISR. The 

1087 exposure is modified by this method. 

1088 camera : `lsst.afw.cameraGeom.Camera`, optional 

1089 The camera geometry for this exposure. Used to select the 

1090 distortion model appropriate for this data. 

1091 bias : `lsst.afw.image.Exposure`, optional 

1092 Bias calibration frame. 

1093 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1094 Functor for linearization. 

1095 crosstalkSources : `list`, optional 

1096 List of possible crosstalk sources. 

1097 dark : `lsst.afw.image.Exposure`, optional 

1098 Dark calibration frame. 

1099 flat : `lsst.afw.image.Exposure`, optional 

1100 Flat calibration frame. 

1101 bfKernel : `numpy.ndarray`, optional 

1102 Brighter-fatter kernel. 

1103 bfGains : `dict` of `float`, optional 

1104 Gains used to override the detector's nominal gains for the 

1105 brighter-fatter correction. A dict keyed by amplifier name for 

1106 the detector in question. 

1107 defects : `lsst.meas.algorithms.Defects`, optional 

1108 List of defects. 

1109 fringes : `lsst.pipe.base.Struct`, optional 

1110 Struct containing the fringe correction data, with 

1111 elements: 

1112 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1113 - ``seed``: random seed derived from the ccdExposureId for random 

1114 number generator (`uint32`) 

1115 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1116 A ``TransmissionCurve`` that represents the throughput of the optics, 

1117 to be evaluated in focal-plane coordinates. 

1118 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1119 A ``TransmissionCurve`` that represents the throughput of the filter 

1120 itself, to be evaluated in focal-plane coordinates. 

1121 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1122 A ``TransmissionCurve`` that represents the throughput of the sensor 

1123 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1124 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1125 A ``TransmissionCurve`` that represents the throughput of the 

1126 atmosphere, assumed to be spatially constant. 

1127 detectorNum : `int`, optional 

1128 The integer number for the detector to process. 

1129 isGen3 : bool, optional 

1130 Flag this call to run() as using the Gen3 butler environment. 

1131 strayLightData : `object`, optional 

1132 Opaque object containing calibration information for stray-light 

1133 correction. If `None`, no correction will be performed. 

1134 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1135 Illumination correction image. 

1136 

1137 Returns 

1138 ------- 

1139 result : `lsst.pipe.base.Struct` 

1140 Result struct with component: 

1141 - ``exposure`` : `afw.image.Exposure` 

1142 The fully ISR corrected exposure. 

1143 - ``outputExposure`` : `afw.image.Exposure` 

1144 An alias for `exposure` 

1145 - ``ossThumb`` : `numpy.ndarray` 

1146 Thumbnail image of the exposure after overscan subtraction. 

1147 - ``flattenedThumb`` : `numpy.ndarray` 

1148 Thumbnail image of the exposure after flat-field correction. 

1149 

1150 Raises 

1151 ------ 

1152 RuntimeError 

1153 Raised if a configuration option is set to True, but the 

1154 required calibration data has not been specified. 

1155 

1156 Notes 

1157 ----- 

1158 The current processed exposure can be viewed by setting the 

1159 appropriate lsstDebug entries in the `debug.display` 

1160 dictionary. The names of these entries correspond to some of 

1161 the IsrTaskConfig Boolean options, with the value denoting the 

1162 frame to use. The exposure is shown inside the matching 

1163 option check and after the processing of that step has 

1164 finished. The steps with debug points are: 

1165 

1166 doAssembleCcd 

1167 doBias 

1168 doCrosstalk 

1169 doBrighterFatter 

1170 doDark 

1171 doFringe 

1172 doStrayLight 

1173 doFlat 

1174 

1175 In addition, setting the "postISRCCD" entry displays the 

1176 exposure after all ISR processing has finished. 

1177 

1178 """ 

1179 

1180 if isGen3 is True: 

1181 # Gen3 currently cannot automatically do configuration overrides. 

1182 # DM-15257 looks to discuss this issue. 

1183 # Configure input exposures; 

1184 if detectorNum is None: 

1185 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1186 

1187 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1188 bias = self.ensureExposure(bias, camera, detectorNum) 

1189 dark = self.ensureExposure(dark, camera, detectorNum) 

1190 flat = self.ensureExposure(flat, camera, detectorNum) 

1191 else: 

1192 if isinstance(ccdExposure, ButlerDataRef): 

1193 return self.runDataRef(ccdExposure) 

1194 

1195 ccd = ccdExposure.getDetector() 

1196 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1197 

1198 if not ccd: 

1199 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1200 ccd = [FakeAmp(ccdExposure, self.config)] 

1201 

1202 # Validate Input 

1203 if self.config.doBias and bias is None: 

1204 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1205 if self.doLinearize(ccd) and linearizer is None: 

1206 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1207 if self.config.doBrighterFatter and bfKernel is None: 

1208 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1209 if self.config.doDark and dark is None: 

1210 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1211 if self.config.doFlat and flat is None: 

1212 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1213 if self.config.doDefect and defects is None: 

1214 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1215 if (self.config.doFringe and filterName in self.fringe.config.filters and 

1216 fringes.fringes is None): 

1217 # The `fringes` object needs to be a pipeBase.Struct, as 

1218 # we use it as a `dict` for the parameters of 

1219 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1220 # not be `None` if `doFringe=True`. Otherwise, raise. 

1221 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1222 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and 

1223 illumMaskedImage is None): 

1224 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1225 

1226 # Begin ISR processing. 

1227 if self.config.doConvertIntToFloat: 

1228 self.log.info("Converting exposure to floating point values.") 

1229 ccdExposure = self.convertIntToFloat(ccdExposure) 

1230 

1231 # Amplifier level processing. 

1232 overscans = [] 

1233 for amp in ccd: 

1234 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1235 if ccdExposure.getBBox().contains(amp.getBBox()): 

1236 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1237 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1238 

1239 if self.config.doOverscan and not badAmp: 

1240 # Overscan correction on amp-by-amp basis. 

1241 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1242 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1243 if overscanResults is not None and \ 

1244 self.config.qa is not None and self.config.qa.saveStats is True: 

1245 if isinstance(overscanResults.overscanFit, float): 

1246 qaMedian = overscanResults.overscanFit 

1247 qaStdev = float("NaN") 

1248 else: 

1249 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1250 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1251 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1252 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1253 

1254 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian) 

1255 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev) 

1256 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1257 amp.getName(), qaMedian, qaStdev) 

1258 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1259 else: 

1260 if badAmp: 

1261 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1262 overscanResults = None 

1263 

1264 overscans.append(overscanResults if overscanResults is not None else None) 

1265 else: 

1266 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1267 

1268 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1269 self.log.info("Applying crosstalk correction.") 

1270 self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources) 

1271 self.debugView(ccdExposure, "doCrosstalk") 

1272 

1273 if self.config.doAssembleCcd: 

1274 self.log.info("Assembling CCD from amplifiers.") 

1275 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1276 

1277 if self.config.expectWcs and not ccdExposure.getWcs(): 

1278 self.log.warn("No WCS found in input exposure.") 

1279 self.debugView(ccdExposure, "doAssembleCcd") 

1280 

1281 ossThumb = None 

1282 if self.config.qa.doThumbnailOss: 

1283 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1284 

1285 if self.config.doBias: 

1286 self.log.info("Applying bias correction.") 

1287 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1288 trimToFit=self.config.doTrimToMatchCalib) 

1289 self.debugView(ccdExposure, "doBias") 

1290 

1291 if self.config.doVariance: 

1292 for amp, overscanResults in zip(ccd, overscans): 

1293 if ccdExposure.getBBox().contains(amp.getBBox()): 

1294 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1295 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1296 if overscanResults is not None: 

1297 self.updateVariance(ampExposure, amp, 

1298 overscanImage=overscanResults.overscanImage) 

1299 else: 

1300 self.updateVariance(ampExposure, amp, 

1301 overscanImage=None) 

1302 if self.config.qa is not None and self.config.qa.saveStats is True: 

1303 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1304 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1305 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1306 qaStats.getValue(afwMath.MEDIAN)) 

1307 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1308 qaStats.getValue(afwMath.STDEVCLIP)) 

1309 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1310 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1311 qaStats.getValue(afwMath.STDEVCLIP)) 

1312 

1313 if self.doLinearize(ccd): 

1314 self.log.info("Applying linearizer.") 

1315 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1316 detector=ccd, log=self.log) 

1317 

1318 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1319 self.log.info("Applying crosstalk correction.") 

1320 self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True) 

1321 self.debugView(ccdExposure, "doCrosstalk") 

1322 

1323 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1324 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1325 if self.config.doDefect: 

1326 self.log.info("Masking defects.") 

1327 self.maskDefect(ccdExposure, defects) 

1328 

1329 if self.config.numEdgeSuspect > 0: 

1330 self.log.info("Masking edges as SUSPECT.") 

1331 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1332 maskPlane="SUSPECT") 

1333 

1334 if self.config.doNanMasking: 

1335 self.log.info("Masking NAN value pixels.") 

1336 self.maskNan(ccdExposure) 

1337 

1338 if self.config.doWidenSaturationTrails: 

1339 self.log.info("Widening saturation trails.") 

1340 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1341 

1342 if self.config.doCameraSpecificMasking: 

1343 self.log.info("Masking regions for camera specific reasons.") 

1344 self.masking.run(ccdExposure) 

1345 

1346 if self.config.doBrighterFatter: 

1347 # We need to apply flats and darks before we can interpolate, and we 

1348 # need to interpolate before we do B-F, but we do B-F without the 

1349 # flats and darks applied so we can work in units of electrons or holes. 

1350 # This context manager applies and then removes the darks and flats. 

1351 # 

1352 # We also do not want to interpolate values here, so operate on temporary 

1353 # images so we can apply only the BF-correction and roll back the 

1354 # interpolation. 

1355 interpExp = ccdExposure.clone() 

1356 with self.flatContext(interpExp, flat, dark): 

1357 isrFunctions.interpolateFromMask( 

1358 maskedImage=interpExp.getMaskedImage(), 

1359 fwhm=self.config.fwhm, 

1360 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1361 maskNameList=self.config.maskListToInterpolate 

1362 ) 

1363 bfExp = interpExp.clone() 

1364 

1365 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1366 type(bfKernel), type(bfGains)) 

1367 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1368 self.config.brighterFatterMaxIter, 

1369 self.config.brighterFatterThreshold, 

1370 self.config.brighterFatterApplyGain, 

1371 bfGains) 

1372 if bfResults[1] == self.config.brighterFatterMaxIter: 

1373 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1374 bfResults[0]) 

1375 else: 

1376 self.log.info("Finished brighter fatter correction in %d iterations.", 

1377 bfResults[1]) 

1378 image = ccdExposure.getMaskedImage().getImage() 

1379 bfCorr = bfExp.getMaskedImage().getImage() 

1380 bfCorr -= interpExp.getMaskedImage().getImage() 

1381 image += bfCorr 

1382 

1383 # Applying the brighter-fatter correction applies a 

1384 # convolution to the science image. At the edges this 

1385 # convolution may not have sufficient valid pixels to 

1386 # produce a valid correction. Mark pixels within the size 

1387 # of the brighter-fatter kernel as EDGE to warn of this 

1388 # fact. 

1389 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1390 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1391 maskPlane="EDGE") 

1392 

1393 if self.config.brighterFatterMaskGrowSize > 0: 

1394 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1395 for maskPlane in self.config.maskListToInterpolate: 

1396 isrFunctions.growMasks(ccdExposure.getMask(), 

1397 radius=self.config.brighterFatterMaskGrowSize, 

1398 maskNameList=maskPlane, 

1399 maskValue=maskPlane) 

1400 

1401 self.debugView(ccdExposure, "doBrighterFatter") 

1402 

1403 if self.config.doDark: 

1404 self.log.info("Applying dark correction.") 

1405 self.darkCorrection(ccdExposure, dark) 

1406 self.debugView(ccdExposure, "doDark") 

1407 

1408 if self.config.doFringe and not self.config.fringeAfterFlat: 

1409 self.log.info("Applying fringe correction before flat.") 

1410 self.fringe.run(ccdExposure, **fringes.getDict()) 

1411 self.debugView(ccdExposure, "doFringe") 

1412 

1413 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1414 self.log.info("Checking strayLight correction.") 

1415 self.strayLight.run(ccdExposure, strayLightData) 

1416 self.debugView(ccdExposure, "doStrayLight") 

1417 

1418 if self.config.doFlat: 

1419 self.log.info("Applying flat correction.") 

1420 self.flatCorrection(ccdExposure, flat) 

1421 self.debugView(ccdExposure, "doFlat") 

1422 

1423 if self.config.doApplyGains: 

1424 self.log.info("Applying gain correction instead of flat.") 

1425 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1426 

1427 if self.config.doFringe and self.config.fringeAfterFlat: 

1428 self.log.info("Applying fringe correction after flat.") 

1429 self.fringe.run(ccdExposure, **fringes.getDict()) 

1430 

1431 if self.config.doVignette: 

1432 self.log.info("Constructing Vignette polygon.") 

1433 self.vignettePolygon = self.vignette.run(ccdExposure) 

1434 

1435 if self.config.vignette.doWriteVignettePolygon: 

1436 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1437 

1438 if self.config.doAttachTransmissionCurve: 

1439 self.log.info("Adding transmission curves.") 

1440 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1441 filterTransmission=filterTransmission, 

1442 sensorTransmission=sensorTransmission, 

1443 atmosphereTransmission=atmosphereTransmission) 

1444 

1445 flattenedThumb = None 

1446 if self.config.qa.doThumbnailFlattened: 

1447 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1448 

1449 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1450 self.log.info("Performing illumination correction.") 

1451 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1452 illumMaskedImage, illumScale=self.config.illumScale, 

1453 trimToFit=self.config.doTrimToMatchCalib) 

1454 

1455 preInterpExp = None 

1456 if self.config.doSaveInterpPixels: 

1457 preInterpExp = ccdExposure.clone() 

1458 

1459 # Reset and interpolate bad pixels. 

1460 # 

1461 # Large contiguous bad regions (which should have the BAD mask 

1462 # bit set) should have their values set to the image median. 

1463 # This group should include defects and bad amplifiers. As the 

1464 # area covered by these defects are large, there's little 

1465 # reason to expect that interpolation would provide a more 

1466 # useful value. 

1467 # 

1468 # Smaller defects can be safely interpolated after the larger 

1469 # regions have had their pixel values reset. This ensures 

1470 # that the remaining defects adjacent to bad amplifiers (as an 

1471 # example) do not attempt to interpolate extreme values. 

1472 if self.config.doSetBadRegions: 

1473 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1474 if badPixelCount > 0: 

1475 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1476 

1477 if self.config.doInterpolate: 

1478 self.log.info("Interpolating masked pixels.") 

1479 isrFunctions.interpolateFromMask( 

1480 maskedImage=ccdExposure.getMaskedImage(), 

1481 fwhm=self.config.fwhm, 

1482 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1483 maskNameList=list(self.config.maskListToInterpolate) 

1484 ) 

1485 

1486 self.roughZeroPoint(ccdExposure) 

1487 

1488 if self.config.doMeasureBackground: 

1489 self.log.info("Measuring background level.") 

1490 self.measureBackground(ccdExposure, self.config.qa) 

1491 

1492 if self.config.qa is not None and self.config.qa.saveStats is True: 

1493 for amp in ccd: 

1494 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1495 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1496 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1497 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1498 qaStats.getValue(afwMath.MEDIAN)) 

1499 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1500 qaStats.getValue(afwMath.STDEVCLIP)) 

1501 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1502 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1503 qaStats.getValue(afwMath.STDEVCLIP)) 

1504 

1505 self.debugView(ccdExposure, "postISRCCD") 

1506 

1507 return pipeBase.Struct( 

1508 exposure=ccdExposure, 

1509 ossThumb=ossThumb, 

1510 flattenedThumb=flattenedThumb, 

1511 

1512 preInterpolatedExposure=preInterpExp, 

1513 outputExposure=ccdExposure, 

1514 outputOssThumbnail=ossThumb, 

1515 outputFlattenedThumbnail=flattenedThumb, 

1516 ) 

1517 

1518 @pipeBase.timeMethod 

1519 def runDataRef(self, sensorRef): 

1520 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1521 

1522 This method contains the `CmdLineTask` interface to the ISR 

1523 processing. All IO is handled here, freeing the `run()` method 

1524 to manage only pixel-level calculations. The steps performed 

1525 are: 

1526 - Read in necessary detrending/isr/calibration data. 

1527 - Process raw exposure in `run()`. 

1528 - Persist the ISR-corrected exposure as "postISRCCD" if 

1529 config.doWrite=True. 

1530 

1531 Parameters 

1532 ---------- 

1533 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1534 DataRef of the detector data to be processed 

1535 

1536 Returns 

1537 ------- 

1538 result : `lsst.pipe.base.Struct` 

1539 Result struct with component: 

1540 - ``exposure`` : `afw.image.Exposure` 

1541 The fully ISR corrected exposure. 

1542 

1543 Raises 

1544 ------ 

1545 RuntimeError 

1546 Raised if a configuration option is set to True, but the 

1547 required calibration data does not exist. 

1548 

1549 """ 

1550 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1551 

1552 ccdExposure = sensorRef.get(self.config.datasetType) 

1553 

1554 camera = sensorRef.get("camera") 

1555 isrData = self.readIsrData(sensorRef, ccdExposure) 

1556 

1557 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1558 

1559 if self.config.doWrite: 

1560 sensorRef.put(result.exposure, "postISRCCD") 

1561 if result.preInterpolatedExposure is not None: 

1562 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1563 if result.ossThumb is not None: 

1564 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1565 if result.flattenedThumb is not None: 

1566 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1567 

1568 return result 

1569 

1570 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1571 """!Retrieve a calibration dataset for removing instrument signature. 

1572 

1573 Parameters 

1574 ---------- 

1575 

1576 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1577 DataRef of the detector data to find calibration datasets 

1578 for. 

1579 datasetType : `str` 

1580 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1581 dateObs : `str`, optional 

1582 Date of the observation. Used to correct butler failures 

1583 when using fallback filters. 

1584 immediate : `Bool` 

1585 If True, disable butler proxies to enable error handling 

1586 within this routine. 

1587 

1588 Returns 

1589 ------- 

1590 exposure : `lsst.afw.image.Exposure` 

1591 Requested calibration frame. 

1592 

1593 Raises 

1594 ------ 

1595 RuntimeError 

1596 Raised if no matching calibration frame can be found. 

1597 """ 

1598 try: 

1599 exp = dataRef.get(datasetType, immediate=immediate) 

1600 except Exception as exc1: 

1601 if not self.config.fallbackFilterName: 

1602 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1603 try: 

1604 if self.config.useFallbackDate and dateObs: 

1605 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1606 dateObs=dateObs, immediate=immediate) 

1607 else: 

1608 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1609 except Exception as exc2: 

1610 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1611 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1612 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1613 

1614 if self.config.doAssembleIsrExposures: 

1615 exp = self.assembleCcd.assembleCcd(exp) 

1616 return exp 

1617 

1618 def ensureExposure(self, inputExp, camera, detectorNum): 

1619 """Ensure that the data returned by Butler is a fully constructed exposure. 

1620 

1621 ISR requires exposure-level image data for historical reasons, so if we did 

1622 not recieve that from Butler, construct it from what we have, modifying the 

1623 input in place. 

1624 

1625 Parameters 

1626 ---------- 

1627 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1628 `lsst.afw.image.ImageF` 

1629 The input data structure obtained from Butler. 

1630 camera : `lsst.afw.cameraGeom.camera` 

1631 The camera associated with the image. Used to find the appropriate 

1632 detector. 

1633 detectorNum : `int` 

1634 The detector this exposure should match. 

1635 

1636 Returns 

1637 ------- 

1638 inputExp : `lsst.afw.image.Exposure` 

1639 The re-constructed exposure, with appropriate detector parameters. 

1640 

1641 Raises 

1642 ------ 

1643 TypeError 

1644 Raised if the input data cannot be used to construct an exposure. 

1645 """ 

1646 if isinstance(inputExp, afwImage.DecoratedImageU): 

1647 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1648 elif isinstance(inputExp, afwImage.ImageF): 

1649 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1650 elif isinstance(inputExp, afwImage.MaskedImageF): 

1651 inputExp = afwImage.makeExposure(inputExp) 

1652 elif isinstance(inputExp, afwImage.Exposure): 

1653 pass 

1654 elif inputExp is None: 

1655 # Assume this will be caught by the setup if it is a problem. 

1656 return inputExp 

1657 else: 

1658 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1659 (type(inputExp), )) 

1660 

1661 if inputExp.getDetector() is None: 

1662 inputExp.setDetector(camera[detectorNum]) 

1663 

1664 return inputExp 

1665 

1666 def convertIntToFloat(self, exposure): 

1667 """Convert exposure image from uint16 to float. 

1668 

1669 If the exposure does not need to be converted, the input is 

1670 immediately returned. For exposures that are converted to use 

1671 floating point pixels, the variance is set to unity and the 

1672 mask to zero. 

1673 

1674 Parameters 

1675 ---------- 

1676 exposure : `lsst.afw.image.Exposure` 

1677 The raw exposure to be converted. 

1678 

1679 Returns 

1680 ------- 

1681 newexposure : `lsst.afw.image.Exposure` 

1682 The input ``exposure``, converted to floating point pixels. 

1683 

1684 Raises 

1685 ------ 

1686 RuntimeError 

1687 Raised if the exposure type cannot be converted to float. 

1688 

1689 """ 

1690 if isinstance(exposure, afwImage.ExposureF): 

1691 # Nothing to be done 

1692 self.log.debug("Exposure already of type float.") 

1693 return exposure 

1694 if not hasattr(exposure, "convertF"): 

1695 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1696 

1697 newexposure = exposure.convertF() 

1698 newexposure.variance[:] = 1 

1699 newexposure.mask[:] = 0x0 

1700 

1701 return newexposure 

1702 

1703 def maskAmplifier(self, ccdExposure, amp, defects): 

1704 """Identify bad amplifiers, saturated and suspect pixels. 

1705 

1706 Parameters 

1707 ---------- 

1708 ccdExposure : `lsst.afw.image.Exposure` 

1709 Input exposure to be masked. 

1710 amp : `lsst.afw.table.AmpInfoCatalog` 

1711 Catalog of parameters defining the amplifier on this 

1712 exposure to mask. 

1713 defects : `lsst.meas.algorithms.Defects` 

1714 List of defects. Used to determine if the entire 

1715 amplifier is bad. 

1716 

1717 Returns 

1718 ------- 

1719 badAmp : `Bool` 

1720 If this is true, the entire amplifier area is covered by 

1721 defects and unusable. 

1722 

1723 """ 

1724 maskedImage = ccdExposure.getMaskedImage() 

1725 

1726 badAmp = False 

1727 

1728 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1729 # comparison with current defects definition. 

1730 if defects is not None: 

1731 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1732 

1733 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1734 # association with pixels in current ccdExposure). 

1735 if badAmp: 

1736 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1737 afwImage.PARENT) 

1738 maskView = dataView.getMask() 

1739 maskView |= maskView.getPlaneBitMask("BAD") 

1740 del maskView 

1741 return badAmp 

1742 

1743 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1744 # Saturation and suspect pixels can be masked now, though. 

1745 limits = dict() 

1746 if self.config.doSaturation and not badAmp: 

1747 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1748 if self.config.doSuspect and not badAmp: 

1749 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1750 if math.isfinite(self.config.saturation): 

1751 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1752 

1753 for maskName, maskThreshold in limits.items(): 

1754 if not math.isnan(maskThreshold): 

1755 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1756 isrFunctions.makeThresholdMask( 

1757 maskedImage=dataView, 

1758 threshold=maskThreshold, 

1759 growFootprints=0, 

1760 maskName=maskName 

1761 ) 

1762 

1763 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1764 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1765 afwImage.PARENT) 

1766 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1767 self.config.suspectMaskName]) 

1768 if numpy.all(maskView.getArray() & maskVal > 0): 

1769 badAmp = True 

1770 maskView |= maskView.getPlaneBitMask("BAD") 

1771 

1772 return badAmp 

1773 

1774 def overscanCorrection(self, ccdExposure, amp): 

1775 """Apply overscan correction in place. 

1776 

1777 This method does initial pixel rejection of the overscan 

1778 region. The overscan can also be optionally segmented to 

1779 allow for discontinuous overscan responses to be fit 

1780 separately. The actual overscan subtraction is performed by 

1781 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1782 which is called here after the amplifier is preprocessed. 

1783 

1784 Parameters 

1785 ---------- 

1786 ccdExposure : `lsst.afw.image.Exposure` 

1787 Exposure to have overscan correction performed. 

1788 amp : `lsst.afw.table.AmpInfoCatalog` 

1789 The amplifier to consider while correcting the overscan. 

1790 

1791 Returns 

1792 ------- 

1793 overscanResults : `lsst.pipe.base.Struct` 

1794 Result struct with components: 

1795 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1796 Value or fit subtracted from the amplifier image data. 

1797 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1798 Value or fit subtracted from the overscan image data. 

1799 - ``overscanImage`` : `lsst.afw.image.Image` 

1800 Image of the overscan region with the overscan 

1801 correction applied. This quantity is used to estimate 

1802 the amplifier read noise empirically. 

1803 

1804 Raises 

1805 ------ 

1806 RuntimeError 

1807 Raised if the ``amp`` does not contain raw pixel information. 

1808 

1809 See Also 

1810 -------- 

1811 lsst.ip.isr.isrFunctions.overscanCorrection 

1812 """ 

1813 if not amp.getHasRawInfo(): 

1814 raise RuntimeError("This method must be executed on an amp with raw information.") 

1815 

1816 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1817 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1818 return None 

1819 

1820 statControl = afwMath.StatisticsControl() 

1821 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1822 

1823 # Determine the bounding boxes 

1824 dataBBox = amp.getRawDataBBox() 

1825 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1826 dx0 = 0 

1827 dx1 = 0 

1828 

1829 prescanBBox = amp.getRawPrescanBBox() 

1830 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1831 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1832 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1833 else: 

1834 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1835 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1836 

1837 # Determine if we need to work on subregions of the amplifier and overscan. 

1838 imageBBoxes = [] 

1839 overscanBBoxes = [] 

1840 

1841 if ((self.config.overscanBiasJump and 

1842 self.config.overscanBiasJumpLocation) and 

1843 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and 

1844 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1845 self.config.overscanBiasJumpDevices)): 

1846 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1847 yLower = self.config.overscanBiasJumpLocation 

1848 yUpper = dataBBox.getHeight() - yLower 

1849 else: 

1850 yUpper = self.config.overscanBiasJumpLocation 

1851 yLower = dataBBox.getHeight() - yUpper 

1852 

1853 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1854 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1855 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + 

1856 lsst.geom.Extent2I(dx0, 0), 

1857 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1858 yLower))) 

1859 

1860 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1861 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1862 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1863 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1864 yUpper))) 

1865 else: 

1866 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1867 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1868 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1869 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1870 oscanBBox.getHeight()))) 

1871 

1872 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

1873 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1874 ampImage = ccdExposure.maskedImage[imageBBox] 

1875 overscanImage = ccdExposure.maskedImage[overscanBBox] 

1876 

1877 overscanArray = overscanImage.image.array 

1878 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

1879 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

1880 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

1881 

1882 statControl = afwMath.StatisticsControl() 

1883 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1884 

1885 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage, 

1886 overscanImage=overscanImage, 

1887 fitType=self.config.overscanFitType, 

1888 order=self.config.overscanOrder, 

1889 collapseRej=self.config.overscanNumSigmaClip, 

1890 statControl=statControl, 

1891 overscanIsInt=self.config.overscanIsInt 

1892 ) 

1893 

1894 # Measure average overscan levels and record them in the metadata. 

1895 levelStat = afwMath.MEDIAN 

1896 sigmaStat = afwMath.STDEVCLIP 

1897 

1898 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

1899 self.config.qa.flatness.nIter) 

1900 metadata = ccdExposure.getMetadata() 

1901 ampNum = amp.getName() 

1902 if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

1903 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

1904 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

1905 else: 

1906 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

1907 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

1908 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

1909 

1910 return overscanResults 

1911 

1912 def updateVariance(self, ampExposure, amp, overscanImage=None): 

1913 """Set the variance plane using the amplifier gain and read noise 

1914 

1915 The read noise is calculated from the ``overscanImage`` if the 

1916 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1917 the value from the amplifier data is used. 

1918 

1919 Parameters 

1920 ---------- 

1921 ampExposure : `lsst.afw.image.Exposure` 

1922 Exposure to process. 

1923 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

1924 Amplifier detector data. 

1925 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

1926 Image of overscan, required only for empirical read noise. 

1927 

1928 See also 

1929 -------- 

1930 lsst.ip.isr.isrFunctions.updateVariance 

1931 """ 

1932 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

1933 gain = amp.getGain() 

1934 

1935 if math.isnan(gain): 

1936 gain = 1.0 

1937 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

1938 elif gain <= 0: 

1939 patchedGain = 1.0 

1940 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

1941 amp.getName(), gain, patchedGain) 

1942 gain = patchedGain 

1943 

1944 if self.config.doEmpiricalReadNoise and overscanImage is None: 

1945 self.log.info("Overscan is none for EmpiricalReadNoise.") 

1946 

1947 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

1948 stats = afwMath.StatisticsControl() 

1949 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

1950 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

1951 self.log.info("Calculated empirical read noise for amp %s: %f.", 

1952 amp.getName(), readNoise) 

1953 else: 

1954 readNoise = amp.getReadNoise() 

1955 

1956 isrFunctions.updateVariance( 

1957 maskedImage=ampExposure.getMaskedImage(), 

1958 gain=gain, 

1959 readNoise=readNoise, 

1960 ) 

1961 

1962 def darkCorrection(self, exposure, darkExposure, invert=False): 

1963 """!Apply dark correction in place. 

1964 

1965 Parameters 

1966 ---------- 

1967 exposure : `lsst.afw.image.Exposure` 

1968 Exposure to process. 

1969 darkExposure : `lsst.afw.image.Exposure` 

1970 Dark exposure of the same size as ``exposure``. 

1971 invert : `Bool`, optional 

1972 If True, re-add the dark to an already corrected image. 

1973 

1974 Raises 

1975 ------ 

1976 RuntimeError 

1977 Raised if either ``exposure`` or ``darkExposure`` do not 

1978 have their dark time defined. 

1979 

1980 See Also 

1981 -------- 

1982 lsst.ip.isr.isrFunctions.darkCorrection 

1983 """ 

1984 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

1985 if math.isnan(expScale): 

1986 raise RuntimeError("Exposure darktime is NAN.") 

1987 if darkExposure.getInfo().getVisitInfo() is not None \ 

1988 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

1989 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

1990 else: 

1991 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

1992 # so getDarkTime() does not exist. 

1993 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

1994 darkScale = 1.0 

1995 

1996 isrFunctions.darkCorrection( 

1997 maskedImage=exposure.getMaskedImage(), 

1998 darkMaskedImage=darkExposure.getMaskedImage(), 

1999 expScale=expScale, 

2000 darkScale=darkScale, 

2001 invert=invert, 

2002 trimToFit=self.config.doTrimToMatchCalib 

2003 ) 

2004 

2005 def doLinearize(self, detector): 

2006 """!Check if linearization is needed for the detector cameraGeom. 

2007 

2008 Checks config.doLinearize and the linearity type of the first 

2009 amplifier. 

2010 

2011 Parameters 

2012 ---------- 

2013 detector : `lsst.afw.cameraGeom.Detector` 

2014 Detector to get linearity type from. 

2015 

2016 Returns 

2017 ------- 

2018 doLinearize : `Bool` 

2019 If True, linearization should be performed. 

2020 """ 

2021 return self.config.doLinearize and \ 

2022 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2023 

2024 def flatCorrection(self, exposure, flatExposure, invert=False): 

2025 """!Apply flat correction in place. 

2026 

2027 Parameters 

2028 ---------- 

2029 exposure : `lsst.afw.image.Exposure` 

2030 Exposure to process. 

2031 flatExposure : `lsst.afw.image.Exposure` 

2032 Flat exposure of the same size as ``exposure``. 

2033 invert : `Bool`, optional 

2034 If True, unflatten an already flattened image. 

2035 

2036 See Also 

2037 -------- 

2038 lsst.ip.isr.isrFunctions.flatCorrection 

2039 """ 

2040 isrFunctions.flatCorrection( 

2041 maskedImage=exposure.getMaskedImage(), 

2042 flatMaskedImage=flatExposure.getMaskedImage(), 

2043 scalingType=self.config.flatScalingType, 

2044 userScale=self.config.flatUserScale, 

2045 invert=invert, 

2046 trimToFit=self.config.doTrimToMatchCalib 

2047 ) 

2048 

2049 def saturationDetection(self, exposure, amp): 

2050 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2051 

2052 Parameters 

2053 ---------- 

2054 exposure : `lsst.afw.image.Exposure` 

2055 Exposure to process. Only the amplifier DataSec is processed. 

2056 amp : `lsst.afw.table.AmpInfoCatalog` 

2057 Amplifier detector data. 

2058 

2059 See Also 

2060 -------- 

2061 lsst.ip.isr.isrFunctions.makeThresholdMask 

2062 """ 

2063 if not math.isnan(amp.getSaturation()): 

2064 maskedImage = exposure.getMaskedImage() 

2065 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2066 isrFunctions.makeThresholdMask( 

2067 maskedImage=dataView, 

2068 threshold=amp.getSaturation(), 

2069 growFootprints=0, 

2070 maskName=self.config.saturatedMaskName, 

2071 ) 

2072 

2073 def saturationInterpolation(self, exposure): 

2074 """!Interpolate over saturated pixels, in place. 

2075 

2076 This method should be called after `saturationDetection`, to 

2077 ensure that the saturated pixels have been identified in the 

2078 SAT mask. It should also be called after `assembleCcd`, since 

2079 saturated regions may cross amplifier boundaries. 

2080 

2081 Parameters 

2082 ---------- 

2083 exposure : `lsst.afw.image.Exposure` 

2084 Exposure to process. 

2085 

2086 See Also 

2087 -------- 

2088 lsst.ip.isr.isrTask.saturationDetection 

2089 lsst.ip.isr.isrFunctions.interpolateFromMask 

2090 """ 

2091 isrFunctions.interpolateFromMask( 

2092 maskedImage=exposure.getMaskedImage(), 

2093 fwhm=self.config.fwhm, 

2094 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2095 maskNameList=list(self.config.saturatedMaskName), 

2096 ) 

2097 

2098 def suspectDetection(self, exposure, amp): 

2099 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2100 

2101 Parameters 

2102 ---------- 

2103 exposure : `lsst.afw.image.Exposure` 

2104 Exposure to process. Only the amplifier DataSec is processed. 

2105 amp : `lsst.afw.table.AmpInfoCatalog` 

2106 Amplifier detector data. 

2107 

2108 See Also 

2109 -------- 

2110 lsst.ip.isr.isrFunctions.makeThresholdMask 

2111 

2112 Notes 

2113 ----- 

2114 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2115 This is intended to indicate pixels that may be affected by unknown systematics; 

2116 for example if non-linearity corrections above a certain level are unstable 

2117 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2118 that no such level exists and no pixels are to be masked as suspicious. 

2119 """ 

2120 suspectLevel = amp.getSuspectLevel() 

2121 if math.isnan(suspectLevel): 

2122 return 

2123 

2124 maskedImage = exposure.getMaskedImage() 

2125 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2126 isrFunctions.makeThresholdMask( 

2127 maskedImage=dataView, 

2128 threshold=suspectLevel, 

2129 growFootprints=0, 

2130 maskName=self.config.suspectMaskName, 

2131 ) 

2132 

2133 def maskDefect(self, exposure, defectBaseList): 

2134 """!Mask defects using mask plane "BAD", in place. 

2135 

2136 Parameters 

2137 ---------- 

2138 exposure : `lsst.afw.image.Exposure` 

2139 Exposure to process. 

2140 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2141 `lsst.afw.image.DefectBase`. 

2142 List of defects to mask. 

2143 

2144 Notes 

2145 ----- 

2146 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2147 """ 

2148 maskedImage = exposure.getMaskedImage() 

2149 if not isinstance(defectBaseList, Defects): 

2150 # Promotes DefectBase to Defect 

2151 defectList = Defects(defectBaseList) 

2152 else: 

2153 defectList = defectBaseList 

2154 defectList.maskPixels(maskedImage, maskName="BAD") 

2155 

2156 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"): 

2157 """!Mask edge pixels with applicable mask plane. 

2158 

2159 Parameters 

2160 ---------- 

2161 exposure : `lsst.afw.image.Exposure` 

2162 Exposure to process. 

2163 numEdgePixels : `int`, optional 

2164 Number of edge pixels to mask. 

2165 maskPlane : `str`, optional 

2166 Mask plane name to use. 

2167 """ 

2168 maskedImage = exposure.getMaskedImage() 

2169 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2170 

2171 if numEdgePixels > 0: 

2172 goodBBox = maskedImage.getBBox() 

2173 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2174 goodBBox.grow(-numEdgePixels) 

2175 # Mask pixels outside goodBBox 

2176 SourceDetectionTask.setEdgeBits( 

2177 maskedImage, 

2178 goodBBox, 

2179 maskBitMask 

2180 ) 

2181 

2182 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2183 """Mask and interpolate defects using mask plane "BAD", in place. 

2184 

2185 Parameters 

2186 ---------- 

2187 exposure : `lsst.afw.image.Exposure` 

2188 Exposure to process. 

2189 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2190 `lsst.afw.image.DefectBase`. 

2191 List of defects to mask and interpolate. 

2192 

2193 See Also 

2194 -------- 

2195 lsst.ip.isr.isrTask.maskDefect() 

2196 """ 

2197 self.maskDefect(exposure, defectBaseList) 

2198 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2199 maskPlane="SUSPECT") 

2200 isrFunctions.interpolateFromMask( 

2201 maskedImage=exposure.getMaskedImage(), 

2202 fwhm=self.config.fwhm, 

2203 growSaturatedFootprints=0, 

2204 maskNameList=["BAD"], 

2205 ) 

2206 

2207 def maskNan(self, exposure): 

2208 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2209 

2210 Parameters 

2211 ---------- 

2212 exposure : `lsst.afw.image.Exposure` 

2213 Exposure to process. 

2214 

2215 Notes 

2216 ----- 

2217 We mask over all NaNs, including those that are masked with 

2218 other bits (because those may or may not be interpolated over 

2219 later, and we want to remove all NaNs). Despite this 

2220 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2221 the historical name. 

2222 """ 

2223 maskedImage = exposure.getMaskedImage() 

2224 

2225 # Find and mask NaNs 

2226 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2227 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2228 numNans = maskNans(maskedImage, maskVal) 

2229 self.metadata.set("NUMNANS", numNans) 

2230 if numNans > 0: 

2231 self.log.warn("There were %d unmasked NaNs.", numNans) 

2232 

2233 def maskAndInterpolateNan(self, exposure): 

2234 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2235 

2236 Parameters 

2237 ---------- 

2238 exposure : `lsst.afw.image.Exposure` 

2239 Exposure to process. 

2240 

2241 See Also 

2242 -------- 

2243 lsst.ip.isr.isrTask.maskNan() 

2244 """ 

2245 self.maskNan(exposure) 

2246 isrFunctions.interpolateFromMask( 

2247 maskedImage=exposure.getMaskedImage(), 

2248 fwhm=self.config.fwhm, 

2249 growSaturatedFootprints=0, 

2250 maskNameList=["UNMASKEDNAN"], 

2251 ) 

2252 

2253 def measureBackground(self, exposure, IsrQaConfig=None): 

2254 """Measure the image background in subgrids, for quality control purposes. 

2255 

2256 Parameters 

2257 ---------- 

2258 exposure : `lsst.afw.image.Exposure` 

2259 Exposure to process. 

2260 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2261 Configuration object containing parameters on which background 

2262 statistics and subgrids to use. 

2263 """ 

2264 if IsrQaConfig is not None: 

2265 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2266 IsrQaConfig.flatness.nIter) 

2267 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2268 statsControl.setAndMask(maskVal) 

2269 maskedImage = exposure.getMaskedImage() 

2270 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2271 skyLevel = stats.getValue(afwMath.MEDIAN) 

2272 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2273 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2274 metadata = exposure.getMetadata() 

2275 metadata.set('SKYLEVEL', skyLevel) 

2276 metadata.set('SKYSIGMA', skySigma) 

2277 

2278 # calcluating flatlevel over the subgrids 

2279 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2280 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2281 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2282 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2283 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2284 skyLevels = numpy.zeros((nX, nY)) 

2285 

2286 for j in range(nY): 

2287 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2288 for i in range(nX): 

2289 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2290 

2291 xLLC = xc - meshXHalf 

2292 yLLC = yc - meshYHalf 

2293 xURC = xc + meshXHalf - 1 

2294 yURC = yc + meshYHalf - 1 

2295 

2296 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2297 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2298 

2299 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2300 

2301 good = numpy.where(numpy.isfinite(skyLevels)) 

2302 skyMedian = numpy.median(skyLevels[good]) 

2303 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2304 flatness_rms = numpy.std(flatness) 

2305 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2306 

2307 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2308 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2309 nX, nY, flatness_pp, flatness_rms) 

2310 

2311 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2312 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2313 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2314 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2315 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2316 

2317 def roughZeroPoint(self, exposure): 

2318 """Set an approximate magnitude zero point for the exposure. 

2319 

2320 Parameters 

2321 ---------- 

2322 exposure : `lsst.afw.image.Exposure` 

2323 Exposure to process. 

2324 """ 

2325 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2326 if filterName in self.config.fluxMag0T1: 

2327 fluxMag0 = self.config.fluxMag0T1[filterName] 

2328 else: 

2329 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2330 fluxMag0 = self.config.defaultFluxMag0T1 

2331 

2332 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2333 if not expTime > 0: # handle NaN as well as <= 0 

2334 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2335 return 

2336 

2337 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2338 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2339 

2340 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2341 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2342 

2343 Parameters 

2344 ---------- 

2345 ccdExposure : `lsst.afw.image.Exposure` 

2346 Exposure to process. 

2347 fpPolygon : `lsst.afw.geom.Polygon` 

2348 Polygon in focal plane coordinates. 

2349 """ 

2350 # Get ccd corners in focal plane coordinates 

2351 ccd = ccdExposure.getDetector() 

2352 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2353 ccdPolygon = Polygon(fpCorners) 

2354 

2355 # Get intersection of ccd corners with fpPolygon 

2356 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2357 

2358 # Transform back to pixel positions and build new polygon 

2359 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2360 validPolygon = Polygon(ccdPoints) 

2361 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2362 

2363 @contextmanager 

2364 def flatContext(self, exp, flat, dark=None): 

2365 """Context manager that applies and removes flats and darks, 

2366 if the task is configured to apply them. 

2367 

2368 Parameters 

2369 ---------- 

2370 exp : `lsst.afw.image.Exposure` 

2371 Exposure to process. 

2372 flat : `lsst.afw.image.Exposure` 

2373 Flat exposure the same size as ``exp``. 

2374 dark : `lsst.afw.image.Exposure`, optional 

2375 Dark exposure the same size as ``exp``. 

2376 

2377 Yields 

2378 ------ 

2379 exp : `lsst.afw.image.Exposure` 

2380 The flat and dark corrected exposure. 

2381 """ 

2382 if self.config.doDark and dark is not None: 

2383 self.darkCorrection(exp, dark) 

2384 if self.config.doFlat: 

2385 self.flatCorrection(exp, flat) 

2386 try: 

2387 yield exp 

2388 finally: 

2389 if self.config.doFlat: 

2390 self.flatCorrection(exp, flat, invert=True) 

2391 if self.config.doDark and dark is not None: 

2392 self.darkCorrection(exp, dark, invert=True) 

2393 

2394 def debugView(self, exposure, stepname): 

2395 """Utility function to examine ISR exposure at different stages. 

2396 

2397 Parameters 

2398 ---------- 

2399 exposure : `lsst.afw.image.Exposure` 

2400 Exposure to view. 

2401 stepname : `str` 

2402 State of processing to view. 

2403 """ 

2404 frame = getDebugFrame(self._display, stepname) 

2405 if frame: 

2406 display = getDisplay(frame) 

2407 display.scale('asinh', 'zscale') 

2408 display.mtv(exposure) 

2409 prompt = "Press Enter to continue [c]... " 

2410 while True: 

2411 ans = input(prompt).lower() 

2412 if ans in ("", "c",): 

2413 break 

2414 

2415 

2416class FakeAmp(object): 

2417 """A Detector-like object that supports returning gain and saturation level 

2418 

2419 This is used when the input exposure does not have a detector. 

2420 

2421 Parameters 

2422 ---------- 

2423 exposure : `lsst.afw.image.Exposure` 

2424 Exposure to generate a fake amplifier for. 

2425 config : `lsst.ip.isr.isrTaskConfig` 

2426 Configuration to apply to the fake amplifier. 

2427 """ 

2428 

2429 def __init__(self, exposure, config): 

2430 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2431 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2432 self._gain = config.gain 

2433 self._readNoise = config.readNoise 

2434 self._saturation = config.saturation 

2435 

2436 def getBBox(self): 

2437 return self._bbox 

2438 

2439 def getRawBBox(self): 

2440 return self._bbox 

2441 

2442 def getHasRawInfo(self): 

2443 return True # but see getRawHorizontalOverscanBBox() 

2444 

2445 def getRawHorizontalOverscanBBox(self): 

2446 return self._RawHorizontalOverscanBBox 

2447 

2448 def getGain(self): 

2449 return self._gain 

2450 

2451 def getReadNoise(self): 

2452 return self._readNoise 

2453 

2454 def getSaturation(self): 

2455 return self._saturation 

2456 

2457 def getSuspectLevel(self): 

2458 return float("NaN") 

2459 

2460 

2461class RunIsrConfig(pexConfig.Config): 

2462 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2463 

2464 

2465class RunIsrTask(pipeBase.CmdLineTask): 

2466 """Task to wrap the default IsrTask to allow it to be retargeted. 

2467 

2468 The standard IsrTask can be called directly from a command line 

2469 program, but doing so removes the ability of the task to be 

2470 retargeted. As most cameras override some set of the IsrTask 

2471 methods, this would remove those data-specific methods in the 

2472 output post-ISR images. This wrapping class fixes the issue, 

2473 allowing identical post-ISR images to be generated by both the 

2474 processCcd and isrTask code. 

2475 """ 

2476 ConfigClass = RunIsrConfig 

2477 _DefaultName = "runIsr" 

2478 

2479 def __init__(self, *args, **kwargs): 

2480 super().__init__(*args, **kwargs) 

2481 self.makeSubtask("isr") 

2482 

2483 def runDataRef(self, dataRef): 

2484 """ 

2485 Parameters 

2486 ---------- 

2487 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2488 data reference of the detector data to be processed 

2489 

2490 Returns 

2491 ------- 

2492 result : `pipeBase.Struct` 

2493 Result struct with component: 

2494 

2495 - exposure : `lsst.afw.image.Exposure` 

2496 Post-ISR processed exposure. 

2497 """ 

2498 return self.isr.runDataRef(dataRef)