Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42from lsst.meas.algorithms import Defects 

43 

44from . import isrFunctions 

45from . import isrQa 

46from . import linearize 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55 

56 

57__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

58 

59 

60class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

61 dimensions={"instrument", "visit", "detector"}, 

62 defaultTemplates={}): 

63 ccdExposure = cT.PrerequisiteInput( 

64 name="raw", 

65 doc="Input exposure to process.", 

66 storageClass="Exposure", 

67 dimensions=["instrument", "visit", "detector"], 

68 ) 

69 camera = cT.PrerequisiteInput( 

70 name="camera", 

71 storageClass="Camera", 

72 doc="Input camera to construct complete exposures.", 

73 dimensions=["instrument", "calibration_label"], 

74 ) 

75 bias = cT.PrerequisiteInput( 

76 name="bias", 

77 doc="Input bias calibration.", 

78 storageClass="ImageF", 

79 dimensions=["instrument", "calibration_label", "detector"], 

80 ) 

81 dark = cT.PrerequisiteInput( 

82 name='dark', 

83 doc="Input dark calibration.", 

84 storageClass="ImageF", 

85 dimensions=["instrument", "calibration_label", "detector"], 

86 ) 

87 flat = cT.PrerequisiteInput( 

88 name="flat", 

89 doc="Input flat calibration.", 

90 storageClass="MaskedImageF", 

91 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

92 ) 

93 fringes = cT.PrerequisiteInput( 

94 name="fringe", 

95 doc="Input fringe calibration.", 

96 storageClass="ExposureF", 

97 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

98 ) 

99 strayLightData = cT.PrerequisiteInput( 

100 name='yBackground', 

101 doc="Input stray light calibration.", 

102 storageClass="StrayLightData", 

103 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

104 ) 

105 bfKernel = cT.PrerequisiteInput( 

106 name='bfKernel', 

107 doc="Input brighter-fatter kernel.", 

108 storageClass="NumpyArray", 

109 dimensions=["instrument", "calibration_label"], 

110 ) 

111 newBFKernel = cT.PrerequisiteInput( 

112 name='brighterFatterKernel', 

113 doc="Newer complete kernel + gain solutions.", 

114 storageClass="BrighterFatterKernel", 

115 dimensions=["instrument", "calibration_label", "detector"], 

116 ) 

117 defects = cT.PrerequisiteInput( 

118 name='defects', 

119 doc="Input defect tables.", 

120 storageClass="DefectsList", 

121 dimensions=["instrument", "calibration_label", "detector"], 

122 ) 

123 opticsTransmission = cT.PrerequisiteInput( 

124 name="transmission_optics", 

125 storageClass="TransmissionCurve", 

126 doc="Transmission curve due to the optics.", 

127 dimensions=["instrument", "calibration_label"], 

128 ) 

129 filterTransmission = cT.PrerequisiteInput( 

130 name="transmission_filter", 

131 storageClass="TransmissionCurve", 

132 doc="Transmission curve due to the filter.", 

133 dimensions=["instrument", "physical_filter", "calibration_label"], 

134 ) 

135 sensorTransmission = cT.PrerequisiteInput( 

136 name="transmission_sensor", 

137 storageClass="TransmissionCurve", 

138 doc="Transmission curve due to the sensor.", 

139 dimensions=["instrument", "calibration_label", "detector"], 

140 ) 

141 atmosphereTransmission = cT.PrerequisiteInput( 

142 name="transmission_atmosphere", 

143 storageClass="TransmissionCurve", 

144 doc="Transmission curve due to the atmosphere.", 

145 dimensions=["instrument"], 

146 ) 

147 illumMaskedImage = cT.PrerequisiteInput( 

148 name="illum", 

149 doc="Input illumination correction.", 

150 storageClass="MaskedImageF", 

151 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

152 ) 

153 

154 outputExposure = cT.Output( 

155 name='postISRCCD', 

156 doc="Output ISR processed exposure.", 

157 storageClass="ExposureF", 

158 dimensions=["instrument", "visit", "detector"], 

159 ) 

160 preInterpExposure = cT.Output( 

161 name='preInterpISRCCD', 

162 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "visit", "detector"], 

165 ) 

166 outputOssThumbnail = cT.Output( 

167 name="OssThumb", 

168 doc="Output Overscan-subtracted thumbnail image.", 

169 storageClass="Thumbnail", 

170 dimensions=["instrument", "visit", "detector"], 

171 ) 

172 outputFlattenedThumbnail = cT.Output( 

173 name="FlattenedThumb", 

174 doc="Output flat-corrected thumbnail image.", 

175 storageClass="Thumbnail", 

176 dimensions=["instrument", "visit", "detector"], 

177 ) 

178 

179 def __init__(self, *, config=None): 

180 super().__init__(config=config) 

181 

182 if config.doBias is not True: 

183 self.prerequisiteInputs.discard("bias") 

184 if config.doLinearize is not True: 

185 self.prerequisiteInputs.discard("linearizer") 

186 if config.doCrosstalk is not True: 

187 self.prerequisiteInputs.discard("crosstalkSources") 

188 if config.doBrighterFatter is not True: 

189 self.prerequisiteInputs.discard("bfKernel") 

190 self.prerequisiteInputs.discard("newBFKernel") 

191 if config.doDefect is not True: 

192 self.prerequisiteInputs.discard("defects") 

193 if config.doDark is not True: 

194 self.prerequisiteInputs.discard("dark") 

195 if config.doFlat is not True: 

196 self.prerequisiteInputs.discard("flat") 

197 if config.doAttachTransmissionCurve is not True: 

198 self.prerequisiteInputs.discard("opticsTransmission") 

199 self.prerequisiteInputs.discard("filterTransmission") 

200 self.prerequisiteInputs.discard("sensorTransmission") 

201 self.prerequisiteInputs.discard("atmosphereTransmission") 

202 if config.doUseOpticsTransmission is not True: 

203 self.prerequisiteInputs.discard("opticsTransmission") 

204 if config.doUseFilterTransmission is not True: 

205 self.prerequisiteInputs.discard("filterTransmission") 

206 if config.doUseSensorTransmission is not True: 

207 self.prerequisiteInputs.discard("sensorTransmission") 

208 if config.doUseAtmosphereTransmission is not True: 

209 self.prerequisiteInputs.discard("atmosphereTransmission") 

210 if config.doIlluminationCorrection is not True: 

211 self.prerequisiteInputs.discard("illumMaskedImage") 

212 

213 if config.doWrite is not True: 

214 self.outputs.discard("outputExposure") 

215 self.outputs.discard("preInterpExposure") 

216 self.outputs.discard("outputFlattenedThumbnail") 

217 self.outputs.discard("outputOssThumbnail") 

218 if config.doSaveInterpPixels is not True: 

219 self.outputs.discard("preInterpExposure") 

220 if config.qa.doThumbnailOss is not True: 

221 self.outputs.discard("outputOssThumbnail") 

222 if config.qa.doThumbnailFlattened is not True: 

223 self.outputs.discard("outputFlattenedThumbnail") 

224 

225 

226class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

227 pipelineConnections=IsrTaskConnections): 

228 """Configuration parameters for IsrTask. 

229 

230 Items are grouped in the order in which they are executed by the task. 

231 """ 

232 datasetType = pexConfig.Field( 

233 dtype=str, 

234 doc="Dataset type for input data; users will typically leave this alone, " 

235 "but camera-specific ISR tasks will override it", 

236 default="raw", 

237 ) 

238 

239 fallbackFilterName = pexConfig.Field( 

240 dtype=str, 

241 doc="Fallback default filter name for calibrations.", 

242 optional=True 

243 ) 

244 useFallbackDate = pexConfig.Field( 

245 dtype=bool, 

246 doc="Pass observation date when using fallback filter.", 

247 default=False, 

248 ) 

249 expectWcs = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

253 ) 

254 fwhm = pexConfig.Field( 

255 dtype=float, 

256 doc="FWHM of PSF in arcseconds.", 

257 default=1.0, 

258 ) 

259 qa = pexConfig.ConfigField( 

260 dtype=isrQa.IsrQaConfig, 

261 doc="QA related configuration options.", 

262 ) 

263 

264 # Image conversion configuration 

265 doConvertIntToFloat = pexConfig.Field( 

266 dtype=bool, 

267 doc="Convert integer raw images to floating point values?", 

268 default=True, 

269 ) 

270 

271 # Saturated pixel handling. 

272 doSaturation = pexConfig.Field( 

273 dtype=bool, 

274 doc="Mask saturated pixels? NB: this is totally independent of the" 

275 " interpolation option - this is ONLY setting the bits in the mask." 

276 " To have them interpolated make sure doSaturationInterpolation=True", 

277 default=True, 

278 ) 

279 saturatedMaskName = pexConfig.Field( 

280 dtype=str, 

281 doc="Name of mask plane to use in saturation detection and interpolation", 

282 default="SAT", 

283 ) 

284 saturation = pexConfig.Field( 

285 dtype=float, 

286 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

287 default=float("NaN"), 

288 ) 

289 growSaturationFootprintSize = pexConfig.Field( 

290 dtype=int, 

291 doc="Number of pixels by which to grow the saturation footprints", 

292 default=1, 

293 ) 

294 

295 # Suspect pixel handling. 

296 doSuspect = pexConfig.Field( 

297 dtype=bool, 

298 doc="Mask suspect pixels?", 

299 default=False, 

300 ) 

301 suspectMaskName = pexConfig.Field( 

302 dtype=str, 

303 doc="Name of mask plane to use for suspect pixels", 

304 default="SUSPECT", 

305 ) 

306 numEdgeSuspect = pexConfig.Field( 

307 dtype=int, 

308 doc="Number of edge pixels to be flagged as untrustworthy.", 

309 default=0, 

310 ) 

311 

312 # Initial masking options. 

313 doSetBadRegions = pexConfig.Field( 

314 dtype=bool, 

315 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

316 default=True, 

317 ) 

318 badStatistic = pexConfig.ChoiceField( 

319 dtype=str, 

320 doc="How to estimate the average value for BAD regions.", 

321 default='MEANCLIP', 

322 allowed={ 

323 "MEANCLIP": "Correct using the (clipped) mean of good data", 

324 "MEDIAN": "Correct using the median of the good data", 

325 }, 

326 ) 

327 

328 # Overscan subtraction configuration. 

329 doOverscan = pexConfig.Field( 

330 dtype=bool, 

331 doc="Do overscan subtraction?", 

332 default=True, 

333 ) 

334 overscanFitType = pexConfig.ChoiceField( 

335 dtype=str, 

336 doc="The method for fitting the overscan bias level.", 

337 default='MEDIAN', 

338 allowed={ 

339 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

340 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

341 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

342 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

343 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

344 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

345 "MEAN": "Correct using the mean of the overscan region", 

346 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

347 "MEDIAN": "Correct using the median of the overscan region", 

348 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

349 }, 

350 ) 

351 overscanOrder = pexConfig.Field( 

352 dtype=int, 

353 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " + 

354 "or number of spline knots if overscan fit type is a spline."), 

355 default=1, 

356 ) 

357 overscanNumSigmaClip = pexConfig.Field( 

358 dtype=float, 

359 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

360 default=3.0, 

361 ) 

362 overscanIsInt = pexConfig.Field( 

363 dtype=bool, 

364 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" + 

365 " and overscan.FitType=MEDIAN_PER_ROW.", 

366 default=True, 

367 ) 

368 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

369 dtype=int, 

370 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

371 default=0, 

372 ) 

373 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

374 dtype=int, 

375 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

376 default=0, 

377 ) 

378 overscanMaxDev = pexConfig.Field( 378 ↛ exitline 378 didn't jump to the function exit

379 dtype=float, 

380 doc="Maximum deviation from the median for overscan", 

381 default=1000.0, check=lambda x: x > 0 

382 ) 

383 overscanBiasJump = pexConfig.Field( 

384 dtype=bool, 

385 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

386 default=False, 

387 ) 

388 overscanBiasJumpKeyword = pexConfig.Field( 

389 dtype=str, 

390 doc="Header keyword containing information about devices.", 

391 default="NO_SUCH_KEY", 

392 ) 

393 overscanBiasJumpDevices = pexConfig.ListField( 

394 dtype=str, 

395 doc="List of devices that need piecewise overscan correction.", 

396 default=(), 

397 ) 

398 overscanBiasJumpLocation = pexConfig.Field( 

399 dtype=int, 

400 doc="Location of bias jump along y-axis.", 

401 default=0, 

402 ) 

403 

404 # Amplifier to CCD assembly configuration 

405 doAssembleCcd = pexConfig.Field( 

406 dtype=bool, 

407 default=True, 

408 doc="Assemble amp-level exposures into a ccd-level exposure?" 

409 ) 

410 assembleCcd = pexConfig.ConfigurableField( 

411 target=AssembleCcdTask, 

412 doc="CCD assembly task", 

413 ) 

414 

415 # General calibration configuration. 

416 doAssembleIsrExposures = pexConfig.Field( 

417 dtype=bool, 

418 default=False, 

419 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

420 ) 

421 doTrimToMatchCalib = pexConfig.Field( 

422 dtype=bool, 

423 default=False, 

424 doc="Trim raw data to match calibration bounding boxes?" 

425 ) 

426 

427 # Bias subtraction. 

428 doBias = pexConfig.Field( 

429 dtype=bool, 

430 doc="Apply bias frame correction?", 

431 default=True, 

432 ) 

433 biasDataProductName = pexConfig.Field( 

434 dtype=str, 

435 doc="Name of the bias data product", 

436 default="bias", 

437 ) 

438 

439 # Variance construction 

440 doVariance = pexConfig.Field( 

441 dtype=bool, 

442 doc="Calculate variance?", 

443 default=True 

444 ) 

445 gain = pexConfig.Field( 

446 dtype=float, 

447 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

448 default=float("NaN"), 

449 ) 

450 readNoise = pexConfig.Field( 

451 dtype=float, 

452 doc="The read noise to use if no Detector is present in the Exposure", 

453 default=0.0, 

454 ) 

455 doEmpiricalReadNoise = pexConfig.Field( 

456 dtype=bool, 

457 default=False, 

458 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

459 ) 

460 

461 # Linearization. 

462 doLinearize = pexConfig.Field( 

463 dtype=bool, 

464 doc="Correct for nonlinearity of the detector's response?", 

465 default=True, 

466 ) 

467 

468 # Crosstalk. 

469 doCrosstalk = pexConfig.Field( 

470 dtype=bool, 

471 doc="Apply intra-CCD crosstalk correction?", 

472 default=False, 

473 ) 

474 doCrosstalkBeforeAssemble = pexConfig.Field( 

475 dtype=bool, 

476 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

477 default=False, 

478 ) 

479 crosstalk = pexConfig.ConfigurableField( 

480 target=CrosstalkTask, 

481 doc="Intra-CCD crosstalk correction", 

482 ) 

483 

484 # Masking options. 

485 doDefect = pexConfig.Field( 

486 dtype=bool, 

487 doc="Apply correction for CCD defects, e.g. hot pixels?", 

488 default=True, 

489 ) 

490 doNanMasking = pexConfig.Field( 

491 dtype=bool, 

492 doc="Mask NAN pixels?", 

493 default=True, 

494 ) 

495 doWidenSaturationTrails = pexConfig.Field( 

496 dtype=bool, 

497 doc="Widen bleed trails based on their width?", 

498 default=True 

499 ) 

500 

501 # Brighter-Fatter correction. 

502 doBrighterFatter = pexConfig.Field( 

503 dtype=bool, 

504 default=False, 

505 doc="Apply the brighter fatter correction" 

506 ) 

507 brighterFatterLevel = pexConfig.ChoiceField( 

508 dtype=str, 

509 default="DETECTOR", 

510 doc="The level at which to correct for brighter-fatter.", 

511 allowed={ 

512 "AMP": "Every amplifier treated separately.", 

513 "DETECTOR": "One kernel per detector", 

514 } 

515 ) 

516 brighterFatterMaxIter = pexConfig.Field( 

517 dtype=int, 

518 default=10, 

519 doc="Maximum number of iterations for the brighter fatter correction" 

520 ) 

521 brighterFatterThreshold = pexConfig.Field( 

522 dtype=float, 

523 default=1000, 

524 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

525 " absolute value of the difference between the current corrected image and the one" 

526 " from the previous iteration summed over all the pixels." 

527 ) 

528 brighterFatterApplyGain = pexConfig.Field( 

529 dtype=bool, 

530 default=True, 

531 doc="Should the gain be applied when applying the brighter fatter correction?" 

532 ) 

533 brighterFatterMaskGrowSize = pexConfig.Field( 

534 dtype=int, 

535 default=0, 

536 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

537 " when brighter-fatter correction is applied." 

538 ) 

539 

540 # Dark subtraction. 

541 doDark = pexConfig.Field( 

542 dtype=bool, 

543 doc="Apply dark frame correction?", 

544 default=True, 

545 ) 

546 darkDataProductName = pexConfig.Field( 

547 dtype=str, 

548 doc="Name of the dark data product", 

549 default="dark", 

550 ) 

551 

552 # Camera-specific stray light removal. 

553 doStrayLight = pexConfig.Field( 

554 dtype=bool, 

555 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

556 default=False, 

557 ) 

558 strayLight = pexConfig.ConfigurableField( 

559 target=StrayLightTask, 

560 doc="y-band stray light correction" 

561 ) 

562 

563 # Flat correction. 

564 doFlat = pexConfig.Field( 

565 dtype=bool, 

566 doc="Apply flat field correction?", 

567 default=True, 

568 ) 

569 flatDataProductName = pexConfig.Field( 

570 dtype=str, 

571 doc="Name of the flat data product", 

572 default="flat", 

573 ) 

574 flatScalingType = pexConfig.ChoiceField( 

575 dtype=str, 

576 doc="The method for scaling the flat on the fly.", 

577 default='USER', 

578 allowed={ 

579 "USER": "Scale by flatUserScale", 

580 "MEAN": "Scale by the inverse of the mean", 

581 "MEDIAN": "Scale by the inverse of the median", 

582 }, 

583 ) 

584 flatUserScale = pexConfig.Field( 

585 dtype=float, 

586 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

587 default=1.0, 

588 ) 

589 doTweakFlat = pexConfig.Field( 

590 dtype=bool, 

591 doc="Tweak flats to match observed amplifier ratios?", 

592 default=False 

593 ) 

594 

595 # Amplifier normalization based on gains instead of using flats configuration. 

596 doApplyGains = pexConfig.Field( 

597 dtype=bool, 

598 doc="Correct the amplifiers for their gains instead of applying flat correction", 

599 default=False, 

600 ) 

601 normalizeGains = pexConfig.Field( 

602 dtype=bool, 

603 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

604 default=False, 

605 ) 

606 

607 # Fringe correction. 

608 doFringe = pexConfig.Field( 

609 dtype=bool, 

610 doc="Apply fringe correction?", 

611 default=True, 

612 ) 

613 fringe = pexConfig.ConfigurableField( 

614 target=FringeTask, 

615 doc="Fringe subtraction task", 

616 ) 

617 fringeAfterFlat = pexConfig.Field( 

618 dtype=bool, 

619 doc="Do fringe subtraction after flat-fielding?", 

620 default=True, 

621 ) 

622 

623 # Distortion model application. 

624 doAddDistortionModel = pexConfig.Field( 

625 dtype=bool, 

626 doc="Apply a distortion model based on camera geometry to the WCS?", 

627 default=True, 

628 deprecated=("Camera geometry is incorporated when reading the raw files." 

629 " This option no longer is used, and will be removed after v19.") 

630 ) 

631 

632 # Initial CCD-level background statistics options. 

633 doMeasureBackground = pexConfig.Field( 

634 dtype=bool, 

635 doc="Measure the background level on the reduced image?", 

636 default=False, 

637 ) 

638 

639 # Camera-specific masking configuration. 

640 doCameraSpecificMasking = pexConfig.Field( 

641 dtype=bool, 

642 doc="Mask camera-specific bad regions?", 

643 default=False, 

644 ) 

645 masking = pexConfig.ConfigurableField( 

646 target=MaskingTask, 

647 doc="Masking task." 

648 ) 

649 

650 # Interpolation options. 

651 

652 doInterpolate = pexConfig.Field( 

653 dtype=bool, 

654 doc="Interpolate masked pixels?", 

655 default=True, 

656 ) 

657 doSaturationInterpolation = pexConfig.Field( 

658 dtype=bool, 

659 doc="Perform interpolation over pixels masked as saturated?" 

660 " NB: This is independent of doSaturation; if that is False this plane" 

661 " will likely be blank, resulting in a no-op here.", 

662 default=True, 

663 ) 

664 doNanInterpolation = pexConfig.Field( 

665 dtype=bool, 

666 doc="Perform interpolation over pixels masked as NaN?" 

667 " NB: This is independent of doNanMasking; if that is False this plane" 

668 " will likely be blank, resulting in a no-op here.", 

669 default=True, 

670 ) 

671 doNanInterpAfterFlat = pexConfig.Field( 

672 dtype=bool, 

673 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

674 "also have to interpolate them before flat-fielding."), 

675 default=False, 

676 ) 

677 maskListToInterpolate = pexConfig.ListField( 

678 dtype=str, 

679 doc="List of mask planes that should be interpolated.", 

680 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

681 ) 

682 doSaveInterpPixels = pexConfig.Field( 

683 dtype=bool, 

684 doc="Save a copy of the pre-interpolated pixel values?", 

685 default=False, 

686 ) 

687 

688 # Default photometric calibration options. 

689 fluxMag0T1 = pexConfig.DictField( 

690 keytype=str, 

691 itemtype=float, 

692 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

693 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

694 )) 

695 ) 

696 defaultFluxMag0T1 = pexConfig.Field( 

697 dtype=float, 

698 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

699 default=pow(10.0, 0.4*28.0) 

700 ) 

701 

702 # Vignette correction configuration. 

703 doVignette = pexConfig.Field( 

704 dtype=bool, 

705 doc="Apply vignetting parameters?", 

706 default=False, 

707 ) 

708 vignette = pexConfig.ConfigurableField( 

709 target=VignetteTask, 

710 doc="Vignetting task.", 

711 ) 

712 

713 # Transmission curve configuration. 

714 doAttachTransmissionCurve = pexConfig.Field( 

715 dtype=bool, 

716 default=False, 

717 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

718 ) 

719 doUseOpticsTransmission = pexConfig.Field( 

720 dtype=bool, 

721 default=True, 

722 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

723 ) 

724 doUseFilterTransmission = pexConfig.Field( 

725 dtype=bool, 

726 default=True, 

727 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

728 ) 

729 doUseSensorTransmission = pexConfig.Field( 

730 dtype=bool, 

731 default=True, 

732 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

733 ) 

734 doUseAtmosphereTransmission = pexConfig.Field( 

735 dtype=bool, 

736 default=True, 

737 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

738 ) 

739 

740 # Illumination correction. 

741 doIlluminationCorrection = pexConfig.Field( 

742 dtype=bool, 

743 default=False, 

744 doc="Perform illumination correction?" 

745 ) 

746 illuminationCorrectionDataProductName = pexConfig.Field( 

747 dtype=str, 

748 doc="Name of the illumination correction data product.", 

749 default="illumcor", 

750 ) 

751 illumScale = pexConfig.Field( 

752 dtype=float, 

753 doc="Scale factor for the illumination correction.", 

754 default=1.0, 

755 ) 

756 illumFilters = pexConfig.ListField( 

757 dtype=str, 

758 default=[], 

759 doc="Only perform illumination correction for these filters." 

760 ) 

761 

762 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

763 doWrite = pexConfig.Field( 

764 dtype=bool, 

765 doc="Persist postISRCCD?", 

766 default=True, 

767 ) 

768 

769 def validate(self): 

770 super().validate() 

771 if self.doFlat and self.doApplyGains: 

772 raise ValueError("You may not specify both doFlat and doApplyGains") 

773 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate: 

774 self.config.maskListToInterpolate.append("SAT") 

775 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

776 self.config.maskListToInterpolate.append("UNMASKEDNAN") 

777 

778 

779class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

780 """Apply common instrument signature correction algorithms to a raw frame. 

781 

782 The process for correcting imaging data is very similar from 

783 camera to camera. This task provides a vanilla implementation of 

784 doing these corrections, including the ability to turn certain 

785 corrections off if they are not needed. The inputs to the primary 

786 method, `run()`, are a raw exposure to be corrected and the 

787 calibration data products. The raw input is a single chip sized 

788 mosaic of all amps including overscans and other non-science 

789 pixels. The method `runDataRef()` identifies and defines the 

790 calibration data products, and is intended for use by a 

791 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

792 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

793 subclassed for different camera, although the most camera specific 

794 methods have been split into subtasks that can be redirected 

795 appropriately. 

796 

797 The __init__ method sets up the subtasks for ISR processing, using 

798 the defaults from `lsst.ip.isr`. 

799 

800 Parameters 

801 ---------- 

802 args : `list` 

803 Positional arguments passed to the Task constructor. None used at this time. 

804 kwargs : `dict`, optional 

805 Keyword arguments passed on to the Task constructor. None used at this time. 

806 """ 

807 ConfigClass = IsrTaskConfig 

808 _DefaultName = "isr" 

809 

810 def __init__(self, **kwargs): 

811 super().__init__(**kwargs) 

812 self.makeSubtask("assembleCcd") 

813 self.makeSubtask("crosstalk") 

814 self.makeSubtask("strayLight") 

815 self.makeSubtask("fringe") 

816 self.makeSubtask("masking") 

817 self.makeSubtask("vignette") 

818 

819 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

820 inputs = butlerQC.get(inputRefs) 

821 

822 try: 

823 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

824 except Exception as e: 

825 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

826 (inputRefs, e)) 

827 

828 inputs['isGen3'] = True 

829 

830 detector = inputs['ccdExposure'].getDetector() 

831 

832 if self.doLinearize(detector) is True: 

833 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict): 

834 linearizer = linearize.Linearizer(detector=detector) 

835 linearizer.fromYaml(inputs['linearizer']) 

836 else: 

837 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector) 

838 inputs['linearizer'] = linearizer 

839 

840 if self.config.doDefect is True: 

841 if "defects" in inputs and inputs['defects'] is not None: 

842 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

843 # masking expects a list of defects defined by their bounding box 

844 if not isinstance(inputs["defects"], Defects): 

845 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

846 

847 # Load the correct style of brighter fatter kernel, and repack 

848 # the information as a numpy array. 

849 if self.config.doBrighterFatter: 

850 brighterFatterKernel = inputs.pop('newBFKernel', None) 

851 if brighterFatterKernel is None: 

852 brighterFatterKernel = inputs.get('bfKernel', None) 

853 

854 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

855 detId = detector.getId() 

856 inputs['bfGains'] = brighterFatterKernel.gain 

857 # If the kernel is not an ndarray, it's the cp_pipe version 

858 # so extract the kernel for this detector, or raise an error 

859 if self.config.brighterFatterLevel == 'DETECTOR': 

860 if brighterFatterKernel.detectorKernel: 

861 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

862 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

863 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

864 else: 

865 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

866 else: 

867 # TODO DM-15631 for implementing this 

868 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

869 

870 # Broken: DM-17169 

871 # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be 

872 # fixed for non-HSC cameras in the future. 

873 # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure']) 

874 # if self.config.doCrosstalk else None) 

875 

876 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

877 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

878 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

879 expId=expId, 

880 assembler=self.assembleCcd 

881 if self.config.doAssembleIsrExposures else None) 

882 else: 

883 inputs['fringes'] = pipeBase.Struct(fringes=None) 

884 

885 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

886 if 'strayLightData' not in inputs: 

887 inputs['strayLightData'] = None 

888 

889 outputs = self.run(**inputs) 

890 butlerQC.put(outputs, outputRefs) 

891 

892 def readIsrData(self, dataRef, rawExposure): 

893 """!Retrieve necessary frames for instrument signature removal. 

894 

895 Pre-fetching all required ISR data products limits the IO 

896 required by the ISR. Any conflict between the calibration data 

897 available and that needed for ISR is also detected prior to 

898 doing processing, allowing it to fail quickly. 

899 

900 Parameters 

901 ---------- 

902 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

903 Butler reference of the detector data to be processed 

904 rawExposure : `afw.image.Exposure` 

905 The raw exposure that will later be corrected with the 

906 retrieved calibration data; should not be modified in this 

907 method. 

908 

909 Returns 

910 ------- 

911 result : `lsst.pipe.base.Struct` 

912 Result struct with components (which may be `None`): 

913 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

914 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

915 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

916 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

917 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

918 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

919 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 

920 - ``fringes``: `lsst.pipe.base.Struct` with components: 

921 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

922 - ``seed``: random seed derived from the ccdExposureId for random 

923 number generator (`uint32`). 

924 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

925 A ``TransmissionCurve`` that represents the throughput of the optics, 

926 to be evaluated in focal-plane coordinates. 

927 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

928 A ``TransmissionCurve`` that represents the throughput of the filter 

929 itself, to be evaluated in focal-plane coordinates. 

930 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

931 A ``TransmissionCurve`` that represents the throughput of the sensor 

932 itself, to be evaluated in post-assembly trimmed detector coordinates. 

933 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

934 A ``TransmissionCurve`` that represents the throughput of the 

935 atmosphere, assumed to be spatially constant. 

936 - ``strayLightData`` : `object` 

937 An opaque object containing calibration information for 

938 stray-light correction. If `None`, no correction will be 

939 performed. 

940 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

941 

942 Raises 

943 ------ 

944 NotImplementedError : 

945 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

946 """ 

947 try: 

948 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

949 dateObs = dateObs.toPython().isoformat() 

950 except RuntimeError: 

951 self.log.warn("Unable to identify dateObs for rawExposure.") 

952 dateObs = None 

953 

954 ccd = rawExposure.getDetector() 

955 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

956 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

957 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

958 if self.config.doBias else None) 

959 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

960 linearizer = (dataRef.get("linearizer", immediate=True) 

961 if self.doLinearize(ccd) else None) 

962 if isinstance(linearizer, numpy.ndarray): 

963 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

964 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef) 

965 if self.config.doCrosstalk else None) 

966 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

967 if self.config.doDark else None) 

968 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

969 dateObs=dateObs) 

970 if self.config.doFlat else None) 

971 

972 brighterFatterKernel = None 

973 brighterFatterGains = None 

974 if self.config.doBrighterFatter is True: 

975 try: 

976 # Use the new-style cp_pipe version of the kernel if it exists 

977 # If using a new-style kernel, always use the self-consistent 

978 # gains, i.e. the ones inside the kernel object itself 

979 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

980 brighterFatterGains = brighterFatterKernel.gain 

981 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

982 except NoResults: 

983 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

984 brighterFatterKernel = dataRef.get("bfKernel") 

985 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

986 except NoResults: 

987 brighterFatterKernel = None 

988 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

989 # If the kernel is not an ndarray, it's the cp_pipe version 

990 # so extract the kernel for this detector, or raise an error 

991 if self.config.brighterFatterLevel == 'DETECTOR': 

992 if brighterFatterKernel.detectorKernel: 

993 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

994 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

995 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

996 else: 

997 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

998 else: 

999 # TODO DM-15631 for implementing this 

1000 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1001 

1002 defectList = (dataRef.get("defects") 

1003 if self.config.doDefect else None) 

1004 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

1005 if self.config.doAssembleIsrExposures else None) 

1006 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1007 else pipeBase.Struct(fringes=None)) 

1008 

1009 if self.config.doAttachTransmissionCurve: 

1010 opticsTransmission = (dataRef.get("transmission_optics") 

1011 if self.config.doUseOpticsTransmission else None) 

1012 filterTransmission = (dataRef.get("transmission_filter") 

1013 if self.config.doUseFilterTransmission else None) 

1014 sensorTransmission = (dataRef.get("transmission_sensor") 

1015 if self.config.doUseSensorTransmission else None) 

1016 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1017 if self.config.doUseAtmosphereTransmission else None) 

1018 else: 

1019 opticsTransmission = None 

1020 filterTransmission = None 

1021 sensorTransmission = None 

1022 atmosphereTransmission = None 

1023 

1024 if self.config.doStrayLight: 

1025 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1026 else: 

1027 strayLightData = None 

1028 

1029 illumMaskedImage = (self.getIsrExposure(dataRef, 

1030 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1031 if (self.config.doIlluminationCorrection and 

1032 filterName in self.config.illumFilters) 

1033 else None) 

1034 

1035 # Struct should include only kwargs to run() 

1036 return pipeBase.Struct(bias=biasExposure, 

1037 linearizer=linearizer, 

1038 crosstalkSources=crosstalkSources, 

1039 dark=darkExposure, 

1040 flat=flatExposure, 

1041 bfKernel=brighterFatterKernel, 

1042 bfGains=brighterFatterGains, 

1043 defects=defectList, 

1044 fringes=fringeStruct, 

1045 opticsTransmission=opticsTransmission, 

1046 filterTransmission=filterTransmission, 

1047 sensorTransmission=sensorTransmission, 

1048 atmosphereTransmission=atmosphereTransmission, 

1049 strayLightData=strayLightData, 

1050 illumMaskedImage=illumMaskedImage 

1051 ) 

1052 

1053 @pipeBase.timeMethod 

1054 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, 

1055 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1056 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1057 sensorTransmission=None, atmosphereTransmission=None, 

1058 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1059 isGen3=False, 

1060 ): 

1061 """!Perform instrument signature removal on an exposure. 

1062 

1063 Steps included in the ISR processing, in order performed, are: 

1064 - saturation and suspect pixel masking 

1065 - overscan subtraction 

1066 - CCD assembly of individual amplifiers 

1067 - bias subtraction 

1068 - variance image construction 

1069 - linearization of non-linear response 

1070 - crosstalk masking 

1071 - brighter-fatter correction 

1072 - dark subtraction 

1073 - fringe correction 

1074 - stray light subtraction 

1075 - flat correction 

1076 - masking of known defects and camera specific features 

1077 - vignette calculation 

1078 - appending transmission curve and distortion model 

1079 

1080 Parameters 

1081 ---------- 

1082 ccdExposure : `lsst.afw.image.Exposure` 

1083 The raw exposure that is to be run through ISR. The 

1084 exposure is modified by this method. 

1085 camera : `lsst.afw.cameraGeom.Camera`, optional 

1086 The camera geometry for this exposure. Used to select the 

1087 distortion model appropriate for this data. 

1088 bias : `lsst.afw.image.Exposure`, optional 

1089 Bias calibration frame. 

1090 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1091 Functor for linearization. 

1092 crosstalkSources : `list`, optional 

1093 List of possible crosstalk sources. 

1094 dark : `lsst.afw.image.Exposure`, optional 

1095 Dark calibration frame. 

1096 flat : `lsst.afw.image.Exposure`, optional 

1097 Flat calibration frame. 

1098 bfKernel : `numpy.ndarray`, optional 

1099 Brighter-fatter kernel. 

1100 bfGains : `dict` of `float`, optional 

1101 Gains used to override the detector's nominal gains for the 

1102 brighter-fatter correction. A dict keyed by amplifier name for 

1103 the detector in question. 

1104 defects : `lsst.meas.algorithms.Defects`, optional 

1105 List of defects. 

1106 fringes : `lsst.pipe.base.Struct`, optional 

1107 Struct containing the fringe correction data, with 

1108 elements: 

1109 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1110 - ``seed``: random seed derived from the ccdExposureId for random 

1111 number generator (`uint32`) 

1112 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1113 A ``TransmissionCurve`` that represents the throughput of the optics, 

1114 to be evaluated in focal-plane coordinates. 

1115 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1116 A ``TransmissionCurve`` that represents the throughput of the filter 

1117 itself, to be evaluated in focal-plane coordinates. 

1118 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1119 A ``TransmissionCurve`` that represents the throughput of the sensor 

1120 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1121 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1122 A ``TransmissionCurve`` that represents the throughput of the 

1123 atmosphere, assumed to be spatially constant. 

1124 detectorNum : `int`, optional 

1125 The integer number for the detector to process. 

1126 isGen3 : bool, optional 

1127 Flag this call to run() as using the Gen3 butler environment. 

1128 strayLightData : `object`, optional 

1129 Opaque object containing calibration information for stray-light 

1130 correction. If `None`, no correction will be performed. 

1131 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1132 Illumination correction image. 

1133 

1134 Returns 

1135 ------- 

1136 result : `lsst.pipe.base.Struct` 

1137 Result struct with component: 

1138 - ``exposure`` : `afw.image.Exposure` 

1139 The fully ISR corrected exposure. 

1140 - ``outputExposure`` : `afw.image.Exposure` 

1141 An alias for `exposure` 

1142 - ``ossThumb`` : `numpy.ndarray` 

1143 Thumbnail image of the exposure after overscan subtraction. 

1144 - ``flattenedThumb`` : `numpy.ndarray` 

1145 Thumbnail image of the exposure after flat-field correction. 

1146 

1147 Raises 

1148 ------ 

1149 RuntimeError 

1150 Raised if a configuration option is set to True, but the 

1151 required calibration data has not been specified. 

1152 

1153 Notes 

1154 ----- 

1155 The current processed exposure can be viewed by setting the 

1156 appropriate lsstDebug entries in the `debug.display` 

1157 dictionary. The names of these entries correspond to some of 

1158 the IsrTaskConfig Boolean options, with the value denoting the 

1159 frame to use. The exposure is shown inside the matching 

1160 option check and after the processing of that step has 

1161 finished. The steps with debug points are: 

1162 

1163 doAssembleCcd 

1164 doBias 

1165 doCrosstalk 

1166 doBrighterFatter 

1167 doDark 

1168 doFringe 

1169 doStrayLight 

1170 doFlat 

1171 

1172 In addition, setting the "postISRCCD" entry displays the 

1173 exposure after all ISR processing has finished. 

1174 

1175 """ 

1176 

1177 if isGen3 is True: 

1178 # Gen3 currently cannot automatically do configuration overrides. 

1179 # DM-15257 looks to discuss this issue. 

1180 # Configure input exposures; 

1181 if detectorNum is None: 

1182 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1183 

1184 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1185 bias = self.ensureExposure(bias, camera, detectorNum) 

1186 dark = self.ensureExposure(dark, camera, detectorNum) 

1187 flat = self.ensureExposure(flat, camera, detectorNum) 

1188 else: 

1189 if isinstance(ccdExposure, ButlerDataRef): 

1190 return self.runDataRef(ccdExposure) 

1191 

1192 ccd = ccdExposure.getDetector() 

1193 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1194 

1195 if not ccd: 

1196 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1197 ccd = [FakeAmp(ccdExposure, self.config)] 

1198 

1199 # Validate Input 

1200 if self.config.doBias and bias is None: 

1201 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1202 if self.doLinearize(ccd) and linearizer is None: 

1203 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1204 if self.config.doBrighterFatter and bfKernel is None: 

1205 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1206 if self.config.doDark and dark is None: 

1207 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1208 if self.config.doFlat and flat is None: 

1209 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1210 if self.config.doDefect and defects is None: 

1211 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1212 if (self.config.doFringe and filterName in self.fringe.config.filters and 

1213 fringes.fringes is None): 

1214 # The `fringes` object needs to be a pipeBase.Struct, as 

1215 # we use it as a `dict` for the parameters of 

1216 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1217 # not be `None` if `doFringe=True`. Otherwise, raise. 

1218 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1219 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and 

1220 illumMaskedImage is None): 

1221 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1222 

1223 # Begin ISR processing. 

1224 if self.config.doConvertIntToFloat: 

1225 self.log.info("Converting exposure to floating point values.") 

1226 ccdExposure = self.convertIntToFloat(ccdExposure) 

1227 

1228 # Amplifier level processing. 

1229 overscans = [] 

1230 for amp in ccd: 

1231 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1232 if ccdExposure.getBBox().contains(amp.getBBox()): 

1233 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1234 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1235 

1236 if self.config.doOverscan and not badAmp: 

1237 # Overscan correction on amp-by-amp basis. 

1238 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1239 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1240 if overscanResults is not None and \ 

1241 self.config.qa is not None and self.config.qa.saveStats is True: 

1242 if isinstance(overscanResults.overscanFit, float): 

1243 qaMedian = overscanResults.overscanFit 

1244 qaStdev = float("NaN") 

1245 else: 

1246 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1247 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1248 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1249 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1250 

1251 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian) 

1252 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev) 

1253 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1254 amp.getName(), qaMedian, qaStdev) 

1255 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1256 else: 

1257 if badAmp: 

1258 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1259 overscanResults = None 

1260 

1261 overscans.append(overscanResults if overscanResults is not None else None) 

1262 else: 

1263 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1264 

1265 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1266 self.log.info("Applying crosstalk correction.") 

1267 self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources) 

1268 self.debugView(ccdExposure, "doCrosstalk") 

1269 

1270 if self.config.doAssembleCcd: 

1271 self.log.info("Assembling CCD from amplifiers.") 

1272 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1273 

1274 if self.config.expectWcs and not ccdExposure.getWcs(): 

1275 self.log.warn("No WCS found in input exposure.") 

1276 self.debugView(ccdExposure, "doAssembleCcd") 

1277 

1278 ossThumb = None 

1279 if self.config.qa.doThumbnailOss: 

1280 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1281 

1282 if self.config.doBias: 

1283 self.log.info("Applying bias correction.") 

1284 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1285 trimToFit=self.config.doTrimToMatchCalib) 

1286 self.debugView(ccdExposure, "doBias") 

1287 

1288 if self.config.doVariance: 

1289 for amp, overscanResults in zip(ccd, overscans): 

1290 if ccdExposure.getBBox().contains(amp.getBBox()): 

1291 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1292 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1293 if overscanResults is not None: 

1294 self.updateVariance(ampExposure, amp, 

1295 overscanImage=overscanResults.overscanImage) 

1296 else: 

1297 self.updateVariance(ampExposure, amp, 

1298 overscanImage=None) 

1299 if self.config.qa is not None and self.config.qa.saveStats is True: 

1300 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1301 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1302 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1303 qaStats.getValue(afwMath.MEDIAN)) 

1304 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1305 qaStats.getValue(afwMath.STDEVCLIP)) 

1306 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1307 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1308 qaStats.getValue(afwMath.STDEVCLIP)) 

1309 

1310 if self.doLinearize(ccd): 

1311 self.log.info("Applying linearizer.") 

1312 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1313 detector=ccd, log=self.log) 

1314 

1315 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1316 self.log.info("Applying crosstalk correction.") 

1317 self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True) 

1318 self.debugView(ccdExposure, "doCrosstalk") 

1319 

1320 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1321 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1322 if self.config.doDefect: 

1323 self.log.info("Masking defects.") 

1324 self.maskDefect(ccdExposure, defects) 

1325 

1326 if self.config.numEdgeSuspect > 0: 

1327 self.log.info("Masking edges as SUSPECT.") 

1328 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1329 maskPlane="SUSPECT") 

1330 

1331 if self.config.doNanMasking: 

1332 self.log.info("Masking NAN value pixels.") 

1333 self.maskNan(ccdExposure) 

1334 

1335 if self.config.doWidenSaturationTrails: 

1336 self.log.info("Widening saturation trails.") 

1337 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1338 

1339 if self.config.doCameraSpecificMasking: 

1340 self.log.info("Masking regions for camera specific reasons.") 

1341 self.masking.run(ccdExposure) 

1342 

1343 if self.config.doBrighterFatter: 

1344 # We need to apply flats and darks before we can interpolate, and we 

1345 # need to interpolate before we do B-F, but we do B-F without the 

1346 # flats and darks applied so we can work in units of electrons or holes. 

1347 # This context manager applies and then removes the darks and flats. 

1348 # 

1349 # We also do not want to interpolate values here, so operate on temporary 

1350 # images so we can apply only the BF-correction and roll back the 

1351 # interpolation. 

1352 interpExp = ccdExposure.clone() 

1353 with self.flatContext(interpExp, flat, dark): 

1354 isrFunctions.interpolateFromMask( 

1355 maskedImage=interpExp.getMaskedImage(), 

1356 fwhm=self.config.fwhm, 

1357 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1358 maskNameList=self.config.maskListToInterpolate 

1359 ) 

1360 bfExp = interpExp.clone() 

1361 

1362 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1363 type(bfKernel), type(bfGains)) 

1364 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1365 self.config.brighterFatterMaxIter, 

1366 self.config.brighterFatterThreshold, 

1367 self.config.brighterFatterApplyGain, 

1368 bfGains) 

1369 if bfResults[1] == self.config.brighterFatterMaxIter: 

1370 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1371 bfResults[0]) 

1372 else: 

1373 self.log.info("Finished brighter fatter correction in %d iterations.", 

1374 bfResults[1]) 

1375 image = ccdExposure.getMaskedImage().getImage() 

1376 bfCorr = bfExp.getMaskedImage().getImage() 

1377 bfCorr -= interpExp.getMaskedImage().getImage() 

1378 image += bfCorr 

1379 

1380 # Applying the brighter-fatter correction applies a 

1381 # convolution to the science image. At the edges this 

1382 # convolution may not have sufficient valid pixels to 

1383 # produce a valid correction. Mark pixels within the size 

1384 # of the brighter-fatter kernel as EDGE to warn of this 

1385 # fact. 

1386 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1387 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1388 maskPlane="EDGE") 

1389 

1390 if self.config.brighterFatterMaskGrowSize > 0: 

1391 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1392 for maskPlane in self.config.maskListToInterpolate: 

1393 isrFunctions.growMasks(ccdExposure.getMask(), 

1394 radius=self.config.brighterFatterMaskGrowSize, 

1395 maskNameList=maskPlane, 

1396 maskValue=maskPlane) 

1397 

1398 self.debugView(ccdExposure, "doBrighterFatter") 

1399 

1400 if self.config.doDark: 

1401 self.log.info("Applying dark correction.") 

1402 self.darkCorrection(ccdExposure, dark) 

1403 self.debugView(ccdExposure, "doDark") 

1404 

1405 if self.config.doFringe and not self.config.fringeAfterFlat: 

1406 self.log.info("Applying fringe correction before flat.") 

1407 self.fringe.run(ccdExposure, **fringes.getDict()) 

1408 self.debugView(ccdExposure, "doFringe") 

1409 

1410 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1411 self.log.info("Checking strayLight correction.") 

1412 self.strayLight.run(ccdExposure, strayLightData) 

1413 self.debugView(ccdExposure, "doStrayLight") 

1414 

1415 if self.config.doFlat: 

1416 self.log.info("Applying flat correction.") 

1417 self.flatCorrection(ccdExposure, flat) 

1418 self.debugView(ccdExposure, "doFlat") 

1419 

1420 if self.config.doApplyGains: 

1421 self.log.info("Applying gain correction instead of flat.") 

1422 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1423 

1424 if self.config.doFringe and self.config.fringeAfterFlat: 

1425 self.log.info("Applying fringe correction after flat.") 

1426 self.fringe.run(ccdExposure, **fringes.getDict()) 

1427 

1428 if self.config.doVignette: 

1429 self.log.info("Constructing Vignette polygon.") 

1430 self.vignettePolygon = self.vignette.run(ccdExposure) 

1431 

1432 if self.config.vignette.doWriteVignettePolygon: 

1433 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1434 

1435 if self.config.doAttachTransmissionCurve: 

1436 self.log.info("Adding transmission curves.") 

1437 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1438 filterTransmission=filterTransmission, 

1439 sensorTransmission=sensorTransmission, 

1440 atmosphereTransmission=atmosphereTransmission) 

1441 

1442 flattenedThumb = None 

1443 if self.config.qa.doThumbnailFlattened: 

1444 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1445 

1446 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1447 self.log.info("Performing illumination correction.") 

1448 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1449 illumMaskedImage, illumScale=self.config.illumScale, 

1450 trimToFit=self.config.doTrimToMatchCalib) 

1451 

1452 preInterpExp = None 

1453 if self.config.doSaveInterpPixels: 

1454 preInterpExp = ccdExposure.clone() 

1455 

1456 # Reset and interpolate bad pixels. 

1457 # 

1458 # Large contiguous bad regions (which should have the BAD mask 

1459 # bit set) should have their values set to the image median. 

1460 # This group should include defects and bad amplifiers. As the 

1461 # area covered by these defects are large, there's little 

1462 # reason to expect that interpolation would provide a more 

1463 # useful value. 

1464 # 

1465 # Smaller defects can be safely interpolated after the larger 

1466 # regions have had their pixel values reset. This ensures 

1467 # that the remaining defects adjacent to bad amplifiers (as an 

1468 # example) do not attempt to interpolate extreme values. 

1469 if self.config.doSetBadRegions: 

1470 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1471 if badPixelCount > 0: 

1472 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1473 

1474 if self.config.doInterpolate: 

1475 self.log.info("Interpolating masked pixels.") 

1476 isrFunctions.interpolateFromMask( 

1477 maskedImage=ccdExposure.getMaskedImage(), 

1478 fwhm=self.config.fwhm, 

1479 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1480 maskNameList=list(self.config.maskListToInterpolate) 

1481 ) 

1482 

1483 self.roughZeroPoint(ccdExposure) 

1484 

1485 if self.config.doMeasureBackground: 

1486 self.log.info("Measuring background level.") 

1487 self.measureBackground(ccdExposure, self.config.qa) 

1488 

1489 if self.config.qa is not None and self.config.qa.saveStats is True: 

1490 for amp in ccd: 

1491 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1492 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1493 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1494 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1495 qaStats.getValue(afwMath.MEDIAN)) 

1496 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1497 qaStats.getValue(afwMath.STDEVCLIP)) 

1498 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1499 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1500 qaStats.getValue(afwMath.STDEVCLIP)) 

1501 

1502 self.debugView(ccdExposure, "postISRCCD") 

1503 

1504 return pipeBase.Struct( 

1505 exposure=ccdExposure, 

1506 ossThumb=ossThumb, 

1507 flattenedThumb=flattenedThumb, 

1508 

1509 preInterpolatedExposure=preInterpExp, 

1510 outputExposure=ccdExposure, 

1511 outputOssThumbnail=ossThumb, 

1512 outputFlattenedThumbnail=flattenedThumb, 

1513 ) 

1514 

1515 @pipeBase.timeMethod 

1516 def runDataRef(self, sensorRef): 

1517 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1518 

1519 This method contains the `CmdLineTask` interface to the ISR 

1520 processing. All IO is handled here, freeing the `run()` method 

1521 to manage only pixel-level calculations. The steps performed 

1522 are: 

1523 - Read in necessary detrending/isr/calibration data. 

1524 - Process raw exposure in `run()`. 

1525 - Persist the ISR-corrected exposure as "postISRCCD" if 

1526 config.doWrite=True. 

1527 

1528 Parameters 

1529 ---------- 

1530 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1531 DataRef of the detector data to be processed 

1532 

1533 Returns 

1534 ------- 

1535 result : `lsst.pipe.base.Struct` 

1536 Result struct with component: 

1537 - ``exposure`` : `afw.image.Exposure` 

1538 The fully ISR corrected exposure. 

1539 

1540 Raises 

1541 ------ 

1542 RuntimeError 

1543 Raised if a configuration option is set to True, but the 

1544 required calibration data does not exist. 

1545 

1546 """ 

1547 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1548 

1549 ccdExposure = sensorRef.get(self.config.datasetType) 

1550 

1551 camera = sensorRef.get("camera") 

1552 isrData = self.readIsrData(sensorRef, ccdExposure) 

1553 

1554 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1555 

1556 if self.config.doWrite: 

1557 sensorRef.put(result.exposure, "postISRCCD") 

1558 if result.preInterpolatedExposure is not None: 

1559 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1560 if result.ossThumb is not None: 

1561 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1562 if result.flattenedThumb is not None: 

1563 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1564 

1565 return result 

1566 

1567 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1568 """!Retrieve a calibration dataset for removing instrument signature. 

1569 

1570 Parameters 

1571 ---------- 

1572 

1573 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1574 DataRef of the detector data to find calibration datasets 

1575 for. 

1576 datasetType : `str` 

1577 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1578 dateObs : `str`, optional 

1579 Date of the observation. Used to correct butler failures 

1580 when using fallback filters. 

1581 immediate : `Bool` 

1582 If True, disable butler proxies to enable error handling 

1583 within this routine. 

1584 

1585 Returns 

1586 ------- 

1587 exposure : `lsst.afw.image.Exposure` 

1588 Requested calibration frame. 

1589 

1590 Raises 

1591 ------ 

1592 RuntimeError 

1593 Raised if no matching calibration frame can be found. 

1594 """ 

1595 try: 

1596 exp = dataRef.get(datasetType, immediate=immediate) 

1597 except Exception as exc1: 

1598 if not self.config.fallbackFilterName: 

1599 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1600 try: 

1601 if self.config.useFallbackDate and dateObs: 

1602 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1603 dateObs=dateObs, immediate=immediate) 

1604 else: 

1605 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1606 except Exception as exc2: 

1607 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1608 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1609 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1610 

1611 if self.config.doAssembleIsrExposures: 

1612 exp = self.assembleCcd.assembleCcd(exp) 

1613 return exp 

1614 

1615 def ensureExposure(self, inputExp, camera, detectorNum): 

1616 """Ensure that the data returned by Butler is a fully constructed exposure. 

1617 

1618 ISR requires exposure-level image data for historical reasons, so if we did 

1619 not recieve that from Butler, construct it from what we have, modifying the 

1620 input in place. 

1621 

1622 Parameters 

1623 ---------- 

1624 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1625 `lsst.afw.image.ImageF` 

1626 The input data structure obtained from Butler. 

1627 camera : `lsst.afw.cameraGeom.camera` 

1628 The camera associated with the image. Used to find the appropriate 

1629 detector. 

1630 detectorNum : `int` 

1631 The detector this exposure should match. 

1632 

1633 Returns 

1634 ------- 

1635 inputExp : `lsst.afw.image.Exposure` 

1636 The re-constructed exposure, with appropriate detector parameters. 

1637 

1638 Raises 

1639 ------ 

1640 TypeError 

1641 Raised if the input data cannot be used to construct an exposure. 

1642 """ 

1643 if isinstance(inputExp, afwImage.DecoratedImageU): 

1644 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1645 elif isinstance(inputExp, afwImage.ImageF): 

1646 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1647 elif isinstance(inputExp, afwImage.MaskedImageF): 

1648 inputExp = afwImage.makeExposure(inputExp) 

1649 elif isinstance(inputExp, afwImage.Exposure): 

1650 pass 

1651 elif inputExp is None: 

1652 # Assume this will be caught by the setup if it is a problem. 

1653 return inputExp 

1654 else: 

1655 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1656 (type(inputExp), )) 

1657 

1658 if inputExp.getDetector() is None: 

1659 inputExp.setDetector(camera[detectorNum]) 

1660 

1661 return inputExp 

1662 

1663 def convertIntToFloat(self, exposure): 

1664 """Convert exposure image from uint16 to float. 

1665 

1666 If the exposure does not need to be converted, the input is 

1667 immediately returned. For exposures that are converted to use 

1668 floating point pixels, the variance is set to unity and the 

1669 mask to zero. 

1670 

1671 Parameters 

1672 ---------- 

1673 exposure : `lsst.afw.image.Exposure` 

1674 The raw exposure to be converted. 

1675 

1676 Returns 

1677 ------- 

1678 newexposure : `lsst.afw.image.Exposure` 

1679 The input ``exposure``, converted to floating point pixels. 

1680 

1681 Raises 

1682 ------ 

1683 RuntimeError 

1684 Raised if the exposure type cannot be converted to float. 

1685 

1686 """ 

1687 if isinstance(exposure, afwImage.ExposureF): 

1688 # Nothing to be done 

1689 self.log.debug("Exposure already of type float.") 

1690 return exposure 

1691 if not hasattr(exposure, "convertF"): 

1692 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1693 

1694 newexposure = exposure.convertF() 

1695 newexposure.variance[:] = 1 

1696 newexposure.mask[:] = 0x0 

1697 

1698 return newexposure 

1699 

1700 def maskAmplifier(self, ccdExposure, amp, defects): 

1701 """Identify bad amplifiers, saturated and suspect pixels. 

1702 

1703 Parameters 

1704 ---------- 

1705 ccdExposure : `lsst.afw.image.Exposure` 

1706 Input exposure to be masked. 

1707 amp : `lsst.afw.table.AmpInfoCatalog` 

1708 Catalog of parameters defining the amplifier on this 

1709 exposure to mask. 

1710 defects : `lsst.meas.algorithms.Defects` 

1711 List of defects. Used to determine if the entire 

1712 amplifier is bad. 

1713 

1714 Returns 

1715 ------- 

1716 badAmp : `Bool` 

1717 If this is true, the entire amplifier area is covered by 

1718 defects and unusable. 

1719 

1720 """ 

1721 maskedImage = ccdExposure.getMaskedImage() 

1722 

1723 badAmp = False 

1724 

1725 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1726 # comparison with current defects definition. 

1727 if defects is not None: 

1728 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1729 

1730 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1731 # association with pixels in current ccdExposure). 

1732 if badAmp: 

1733 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1734 afwImage.PARENT) 

1735 maskView = dataView.getMask() 

1736 maskView |= maskView.getPlaneBitMask("BAD") 

1737 del maskView 

1738 return badAmp 

1739 

1740 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1741 # Saturation and suspect pixels can be masked now, though. 

1742 limits = dict() 

1743 if self.config.doSaturation and not badAmp: 

1744 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1745 if self.config.doSuspect and not badAmp: 

1746 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1747 if math.isfinite(self.config.saturation): 

1748 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1749 

1750 for maskName, maskThreshold in limits.items(): 

1751 if not math.isnan(maskThreshold): 

1752 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1753 isrFunctions.makeThresholdMask( 

1754 maskedImage=dataView, 

1755 threshold=maskThreshold, 

1756 growFootprints=0, 

1757 maskName=maskName 

1758 ) 

1759 

1760 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1761 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1762 afwImage.PARENT) 

1763 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1764 self.config.suspectMaskName]) 

1765 if numpy.all(maskView.getArray() & maskVal > 0): 

1766 badAmp = True 

1767 maskView |= maskView.getPlaneBitMask("BAD") 

1768 

1769 return badAmp 

1770 

1771 def overscanCorrection(self, ccdExposure, amp): 

1772 """Apply overscan correction in place. 

1773 

1774 This method does initial pixel rejection of the overscan 

1775 region. The overscan can also be optionally segmented to 

1776 allow for discontinuous overscan responses to be fit 

1777 separately. The actual overscan subtraction is performed by 

1778 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1779 which is called here after the amplifier is preprocessed. 

1780 

1781 Parameters 

1782 ---------- 

1783 ccdExposure : `lsst.afw.image.Exposure` 

1784 Exposure to have overscan correction performed. 

1785 amp : `lsst.afw.table.AmpInfoCatalog` 

1786 The amplifier to consider while correcting the overscan. 

1787 

1788 Returns 

1789 ------- 

1790 overscanResults : `lsst.pipe.base.Struct` 

1791 Result struct with components: 

1792 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1793 Value or fit subtracted from the amplifier image data. 

1794 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1795 Value or fit subtracted from the overscan image data. 

1796 - ``overscanImage`` : `lsst.afw.image.Image` 

1797 Image of the overscan region with the overscan 

1798 correction applied. This quantity is used to estimate 

1799 the amplifier read noise empirically. 

1800 

1801 Raises 

1802 ------ 

1803 RuntimeError 

1804 Raised if the ``amp`` does not contain raw pixel information. 

1805 

1806 See Also 

1807 -------- 

1808 lsst.ip.isr.isrFunctions.overscanCorrection 

1809 """ 

1810 if not amp.getHasRawInfo(): 

1811 raise RuntimeError("This method must be executed on an amp with raw information.") 

1812 

1813 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1814 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1815 return None 

1816 

1817 statControl = afwMath.StatisticsControl() 

1818 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1819 

1820 # Determine the bounding boxes 

1821 dataBBox = amp.getRawDataBBox() 

1822 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1823 dx0 = 0 

1824 dx1 = 0 

1825 

1826 prescanBBox = amp.getRawPrescanBBox() 

1827 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1828 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1829 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1830 else: 

1831 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1832 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1833 

1834 # Determine if we need to work on subregions of the amplifier and overscan. 

1835 imageBBoxes = [] 

1836 overscanBBoxes = [] 

1837 

1838 if ((self.config.overscanBiasJump and 

1839 self.config.overscanBiasJumpLocation) and 

1840 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and 

1841 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1842 self.config.overscanBiasJumpDevices)): 

1843 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1844 yLower = self.config.overscanBiasJumpLocation 

1845 yUpper = dataBBox.getHeight() - yLower 

1846 else: 

1847 yUpper = self.config.overscanBiasJumpLocation 

1848 yLower = dataBBox.getHeight() - yUpper 

1849 

1850 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1851 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1852 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + 

1853 lsst.geom.Extent2I(dx0, 0), 

1854 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1855 yLower))) 

1856 

1857 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1858 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1859 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1860 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1861 yUpper))) 

1862 else: 

1863 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1864 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1865 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1866 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1867 oscanBBox.getHeight()))) 

1868 

1869 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

1870 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1871 ampImage = ccdExposure.maskedImage[imageBBox] 

1872 overscanImage = ccdExposure.maskedImage[overscanBBox] 

1873 

1874 overscanArray = overscanImage.image.array 

1875 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

1876 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

1877 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

1878 

1879 statControl = afwMath.StatisticsControl() 

1880 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1881 

1882 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage, 

1883 overscanImage=overscanImage, 

1884 fitType=self.config.overscanFitType, 

1885 order=self.config.overscanOrder, 

1886 collapseRej=self.config.overscanNumSigmaClip, 

1887 statControl=statControl, 

1888 overscanIsInt=self.config.overscanIsInt 

1889 ) 

1890 

1891 # Measure average overscan levels and record them in the metadata. 

1892 levelStat = afwMath.MEDIAN 

1893 sigmaStat = afwMath.STDEVCLIP 

1894 

1895 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

1896 self.config.qa.flatness.nIter) 

1897 metadata = ccdExposure.getMetadata() 

1898 ampNum = amp.getName() 

1899 if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

1900 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

1901 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

1902 else: 

1903 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

1904 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

1905 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

1906 

1907 return overscanResults 

1908 

1909 def updateVariance(self, ampExposure, amp, overscanImage=None): 

1910 """Set the variance plane using the amplifier gain and read noise 

1911 

1912 The read noise is calculated from the ``overscanImage`` if the 

1913 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1914 the value from the amplifier data is used. 

1915 

1916 Parameters 

1917 ---------- 

1918 ampExposure : `lsst.afw.image.Exposure` 

1919 Exposure to process. 

1920 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

1921 Amplifier detector data. 

1922 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

1923 Image of overscan, required only for empirical read noise. 

1924 

1925 See also 

1926 -------- 

1927 lsst.ip.isr.isrFunctions.updateVariance 

1928 """ 

1929 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

1930 gain = amp.getGain() 

1931 

1932 if math.isnan(gain): 

1933 gain = 1.0 

1934 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

1935 elif gain <= 0: 

1936 patchedGain = 1.0 

1937 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

1938 amp.getName(), gain, patchedGain) 

1939 gain = patchedGain 

1940 

1941 if self.config.doEmpiricalReadNoise and overscanImage is None: 

1942 self.log.info("Overscan is none for EmpiricalReadNoise.") 

1943 

1944 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

1945 stats = afwMath.StatisticsControl() 

1946 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

1947 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

1948 self.log.info("Calculated empirical read noise for amp %s: %f.", 

1949 amp.getName(), readNoise) 

1950 else: 

1951 readNoise = amp.getReadNoise() 

1952 

1953 isrFunctions.updateVariance( 

1954 maskedImage=ampExposure.getMaskedImage(), 

1955 gain=gain, 

1956 readNoise=readNoise, 

1957 ) 

1958 

1959 def darkCorrection(self, exposure, darkExposure, invert=False): 

1960 """!Apply dark correction in place. 

1961 

1962 Parameters 

1963 ---------- 

1964 exposure : `lsst.afw.image.Exposure` 

1965 Exposure to process. 

1966 darkExposure : `lsst.afw.image.Exposure` 

1967 Dark exposure of the same size as ``exposure``. 

1968 invert : `Bool`, optional 

1969 If True, re-add the dark to an already corrected image. 

1970 

1971 Raises 

1972 ------ 

1973 RuntimeError 

1974 Raised if either ``exposure`` or ``darkExposure`` do not 

1975 have their dark time defined. 

1976 

1977 See Also 

1978 -------- 

1979 lsst.ip.isr.isrFunctions.darkCorrection 

1980 """ 

1981 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

1982 if math.isnan(expScale): 

1983 raise RuntimeError("Exposure darktime is NAN.") 

1984 if darkExposure.getInfo().getVisitInfo() is not None: 

1985 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

1986 else: 

1987 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

1988 # so getDarkTime() does not exist. 

1989 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

1990 darkScale = 1.0 

1991 

1992 if math.isnan(darkScale): 

1993 raise RuntimeError("Dark calib darktime is NAN.") 

1994 isrFunctions.darkCorrection( 

1995 maskedImage=exposure.getMaskedImage(), 

1996 darkMaskedImage=darkExposure.getMaskedImage(), 

1997 expScale=expScale, 

1998 darkScale=darkScale, 

1999 invert=invert, 

2000 trimToFit=self.config.doTrimToMatchCalib 

2001 ) 

2002 

2003 def doLinearize(self, detector): 

2004 """!Check if linearization is needed for the detector cameraGeom. 

2005 

2006 Checks config.doLinearize and the linearity type of the first 

2007 amplifier. 

2008 

2009 Parameters 

2010 ---------- 

2011 detector : `lsst.afw.cameraGeom.Detector` 

2012 Detector to get linearity type from. 

2013 

2014 Returns 

2015 ------- 

2016 doLinearize : `Bool` 

2017 If True, linearization should be performed. 

2018 """ 

2019 return self.config.doLinearize and \ 

2020 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2021 

2022 def flatCorrection(self, exposure, flatExposure, invert=False): 

2023 """!Apply flat correction in place. 

2024 

2025 Parameters 

2026 ---------- 

2027 exposure : `lsst.afw.image.Exposure` 

2028 Exposure to process. 

2029 flatExposure : `lsst.afw.image.Exposure` 

2030 Flat exposure of the same size as ``exposure``. 

2031 invert : `Bool`, optional 

2032 If True, unflatten an already flattened image. 

2033 

2034 See Also 

2035 -------- 

2036 lsst.ip.isr.isrFunctions.flatCorrection 

2037 """ 

2038 isrFunctions.flatCorrection( 

2039 maskedImage=exposure.getMaskedImage(), 

2040 flatMaskedImage=flatExposure.getMaskedImage(), 

2041 scalingType=self.config.flatScalingType, 

2042 userScale=self.config.flatUserScale, 

2043 invert=invert, 

2044 trimToFit=self.config.doTrimToMatchCalib 

2045 ) 

2046 

2047 def saturationDetection(self, exposure, amp): 

2048 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2049 

2050 Parameters 

2051 ---------- 

2052 exposure : `lsst.afw.image.Exposure` 

2053 Exposure to process. Only the amplifier DataSec is processed. 

2054 amp : `lsst.afw.table.AmpInfoCatalog` 

2055 Amplifier detector data. 

2056 

2057 See Also 

2058 -------- 

2059 lsst.ip.isr.isrFunctions.makeThresholdMask 

2060 """ 

2061 if not math.isnan(amp.getSaturation()): 

2062 maskedImage = exposure.getMaskedImage() 

2063 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2064 isrFunctions.makeThresholdMask( 

2065 maskedImage=dataView, 

2066 threshold=amp.getSaturation(), 

2067 growFootprints=0, 

2068 maskName=self.config.saturatedMaskName, 

2069 ) 

2070 

2071 def saturationInterpolation(self, exposure): 

2072 """!Interpolate over saturated pixels, in place. 

2073 

2074 This method should be called after `saturationDetection`, to 

2075 ensure that the saturated pixels have been identified in the 

2076 SAT mask. It should also be called after `assembleCcd`, since 

2077 saturated regions may cross amplifier boundaries. 

2078 

2079 Parameters 

2080 ---------- 

2081 exposure : `lsst.afw.image.Exposure` 

2082 Exposure to process. 

2083 

2084 See Also 

2085 -------- 

2086 lsst.ip.isr.isrTask.saturationDetection 

2087 lsst.ip.isr.isrFunctions.interpolateFromMask 

2088 """ 

2089 isrFunctions.interpolateFromMask( 

2090 maskedImage=exposure.getMaskedImage(), 

2091 fwhm=self.config.fwhm, 

2092 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2093 maskNameList=list(self.config.saturatedMaskName), 

2094 ) 

2095 

2096 def suspectDetection(self, exposure, amp): 

2097 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2098 

2099 Parameters 

2100 ---------- 

2101 exposure : `lsst.afw.image.Exposure` 

2102 Exposure to process. Only the amplifier DataSec is processed. 

2103 amp : `lsst.afw.table.AmpInfoCatalog` 

2104 Amplifier detector data. 

2105 

2106 See Also 

2107 -------- 

2108 lsst.ip.isr.isrFunctions.makeThresholdMask 

2109 

2110 Notes 

2111 ----- 

2112 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2113 This is intended to indicate pixels that may be affected by unknown systematics; 

2114 for example if non-linearity corrections above a certain level are unstable 

2115 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2116 that no such level exists and no pixels are to be masked as suspicious. 

2117 """ 

2118 suspectLevel = amp.getSuspectLevel() 

2119 if math.isnan(suspectLevel): 

2120 return 

2121 

2122 maskedImage = exposure.getMaskedImage() 

2123 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2124 isrFunctions.makeThresholdMask( 

2125 maskedImage=dataView, 

2126 threshold=suspectLevel, 

2127 growFootprints=0, 

2128 maskName=self.config.suspectMaskName, 

2129 ) 

2130 

2131 def maskDefect(self, exposure, defectBaseList): 

2132 """!Mask defects using mask plane "BAD", in place. 

2133 

2134 Parameters 

2135 ---------- 

2136 exposure : `lsst.afw.image.Exposure` 

2137 Exposure to process. 

2138 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2139 `lsst.afw.image.DefectBase`. 

2140 List of defects to mask. 

2141 

2142 Notes 

2143 ----- 

2144 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2145 """ 

2146 maskedImage = exposure.getMaskedImage() 

2147 if not isinstance(defectBaseList, Defects): 

2148 # Promotes DefectBase to Defect 

2149 defectList = Defects(defectBaseList) 

2150 else: 

2151 defectList = defectBaseList 

2152 defectList.maskPixels(maskedImage, maskName="BAD") 

2153 

2154 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"): 

2155 """!Mask edge pixels with applicable mask plane. 

2156 

2157 Parameters 

2158 ---------- 

2159 exposure : `lsst.afw.image.Exposure` 

2160 Exposure to process. 

2161 numEdgePixels : `int`, optional 

2162 Number of edge pixels to mask. 

2163 maskPlane : `str`, optional 

2164 Mask plane name to use. 

2165 """ 

2166 maskedImage = exposure.getMaskedImage() 

2167 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2168 

2169 if numEdgePixels > 0: 

2170 goodBBox = maskedImage.getBBox() 

2171 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2172 goodBBox.grow(-numEdgePixels) 

2173 # Mask pixels outside goodBBox 

2174 SourceDetectionTask.setEdgeBits( 

2175 maskedImage, 

2176 goodBBox, 

2177 maskBitMask 

2178 ) 

2179 

2180 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2181 """Mask and interpolate defects using mask plane "BAD", in place. 

2182 

2183 Parameters 

2184 ---------- 

2185 exposure : `lsst.afw.image.Exposure` 

2186 Exposure to process. 

2187 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2188 `lsst.afw.image.DefectBase`. 

2189 List of defects to mask and interpolate. 

2190 

2191 See Also 

2192 -------- 

2193 lsst.ip.isr.isrTask.maskDefect() 

2194 """ 

2195 self.maskDefect(exposure, defectBaseList) 

2196 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2197 maskPlane="SUSPECT") 

2198 isrFunctions.interpolateFromMask( 

2199 maskedImage=exposure.getMaskedImage(), 

2200 fwhm=self.config.fwhm, 

2201 growSaturatedFootprints=0, 

2202 maskNameList=["BAD"], 

2203 ) 

2204 

2205 def maskNan(self, exposure): 

2206 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2207 

2208 Parameters 

2209 ---------- 

2210 exposure : `lsst.afw.image.Exposure` 

2211 Exposure to process. 

2212 

2213 Notes 

2214 ----- 

2215 We mask over all NaNs, including those that are masked with 

2216 other bits (because those may or may not be interpolated over 

2217 later, and we want to remove all NaNs). Despite this 

2218 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2219 the historical name. 

2220 """ 

2221 maskedImage = exposure.getMaskedImage() 

2222 

2223 # Find and mask NaNs 

2224 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2225 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2226 numNans = maskNans(maskedImage, maskVal) 

2227 self.metadata.set("NUMNANS", numNans) 

2228 if numNans > 0: 

2229 self.log.warn("There were %d unmasked NaNs.", numNans) 

2230 

2231 def maskAndInterpolateNan(self, exposure): 

2232 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2233 

2234 Parameters 

2235 ---------- 

2236 exposure : `lsst.afw.image.Exposure` 

2237 Exposure to process. 

2238 

2239 See Also 

2240 -------- 

2241 lsst.ip.isr.isrTask.maskNan() 

2242 """ 

2243 self.maskNan(exposure) 

2244 isrFunctions.interpolateFromMask( 

2245 maskedImage=exposure.getMaskedImage(), 

2246 fwhm=self.config.fwhm, 

2247 growSaturatedFootprints=0, 

2248 maskNameList=["UNMASKEDNAN"], 

2249 ) 

2250 

2251 def measureBackground(self, exposure, IsrQaConfig=None): 

2252 """Measure the image background in subgrids, for quality control purposes. 

2253 

2254 Parameters 

2255 ---------- 

2256 exposure : `lsst.afw.image.Exposure` 

2257 Exposure to process. 

2258 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2259 Configuration object containing parameters on which background 

2260 statistics and subgrids to use. 

2261 """ 

2262 if IsrQaConfig is not None: 

2263 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2264 IsrQaConfig.flatness.nIter) 

2265 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2266 statsControl.setAndMask(maskVal) 

2267 maskedImage = exposure.getMaskedImage() 

2268 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2269 skyLevel = stats.getValue(afwMath.MEDIAN) 

2270 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2271 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2272 metadata = exposure.getMetadata() 

2273 metadata.set('SKYLEVEL', skyLevel) 

2274 metadata.set('SKYSIGMA', skySigma) 

2275 

2276 # calcluating flatlevel over the subgrids 

2277 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2278 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2279 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2280 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2281 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2282 skyLevels = numpy.zeros((nX, nY)) 

2283 

2284 for j in range(nY): 

2285 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2286 for i in range(nX): 

2287 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2288 

2289 xLLC = xc - meshXHalf 

2290 yLLC = yc - meshYHalf 

2291 xURC = xc + meshXHalf - 1 

2292 yURC = yc + meshYHalf - 1 

2293 

2294 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2295 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2296 

2297 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2298 

2299 good = numpy.where(numpy.isfinite(skyLevels)) 

2300 skyMedian = numpy.median(skyLevels[good]) 

2301 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2302 flatness_rms = numpy.std(flatness) 

2303 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2304 

2305 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2306 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2307 nX, nY, flatness_pp, flatness_rms) 

2308 

2309 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2310 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2311 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2312 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2313 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2314 

2315 def roughZeroPoint(self, exposure): 

2316 """Set an approximate magnitude zero point for the exposure. 

2317 

2318 Parameters 

2319 ---------- 

2320 exposure : `lsst.afw.image.Exposure` 

2321 Exposure to process. 

2322 """ 

2323 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2324 if filterName in self.config.fluxMag0T1: 

2325 fluxMag0 = self.config.fluxMag0T1[filterName] 

2326 else: 

2327 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2328 fluxMag0 = self.config.defaultFluxMag0T1 

2329 

2330 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2331 if not expTime > 0: # handle NaN as well as <= 0 

2332 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2333 return 

2334 

2335 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2336 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2337 

2338 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2339 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2340 

2341 Parameters 

2342 ---------- 

2343 ccdExposure : `lsst.afw.image.Exposure` 

2344 Exposure to process. 

2345 fpPolygon : `lsst.afw.geom.Polygon` 

2346 Polygon in focal plane coordinates. 

2347 """ 

2348 # Get ccd corners in focal plane coordinates 

2349 ccd = ccdExposure.getDetector() 

2350 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2351 ccdPolygon = Polygon(fpCorners) 

2352 

2353 # Get intersection of ccd corners with fpPolygon 

2354 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2355 

2356 # Transform back to pixel positions and build new polygon 

2357 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2358 validPolygon = Polygon(ccdPoints) 

2359 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2360 

2361 @contextmanager 

2362 def flatContext(self, exp, flat, dark=None): 

2363 """Context manager that applies and removes flats and darks, 

2364 if the task is configured to apply them. 

2365 

2366 Parameters 

2367 ---------- 

2368 exp : `lsst.afw.image.Exposure` 

2369 Exposure to process. 

2370 flat : `lsst.afw.image.Exposure` 

2371 Flat exposure the same size as ``exp``. 

2372 dark : `lsst.afw.image.Exposure`, optional 

2373 Dark exposure the same size as ``exp``. 

2374 

2375 Yields 

2376 ------ 

2377 exp : `lsst.afw.image.Exposure` 

2378 The flat and dark corrected exposure. 

2379 """ 

2380 if self.config.doDark and dark is not None: 

2381 self.darkCorrection(exp, dark) 

2382 if self.config.doFlat: 

2383 self.flatCorrection(exp, flat) 

2384 try: 

2385 yield exp 

2386 finally: 

2387 if self.config.doFlat: 

2388 self.flatCorrection(exp, flat, invert=True) 

2389 if self.config.doDark and dark is not None: 

2390 self.darkCorrection(exp, dark, invert=True) 

2391 

2392 def debugView(self, exposure, stepname): 

2393 """Utility function to examine ISR exposure at different stages. 

2394 

2395 Parameters 

2396 ---------- 

2397 exposure : `lsst.afw.image.Exposure` 

2398 Exposure to view. 

2399 stepname : `str` 

2400 State of processing to view. 

2401 """ 

2402 frame = getDebugFrame(self._display, stepname) 

2403 if frame: 

2404 display = getDisplay(frame) 

2405 display.scale('asinh', 'zscale') 

2406 display.mtv(exposure) 

2407 prompt = "Press Enter to continue [c]... " 

2408 while True: 

2409 ans = input(prompt).lower() 

2410 if ans in ("", "c",): 

2411 break 

2412 

2413 

2414class FakeAmp(object): 

2415 """A Detector-like object that supports returning gain and saturation level 

2416 

2417 This is used when the input exposure does not have a detector. 

2418 

2419 Parameters 

2420 ---------- 

2421 exposure : `lsst.afw.image.Exposure` 

2422 Exposure to generate a fake amplifier for. 

2423 config : `lsst.ip.isr.isrTaskConfig` 

2424 Configuration to apply to the fake amplifier. 

2425 """ 

2426 

2427 def __init__(self, exposure, config): 

2428 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2429 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2430 self._gain = config.gain 

2431 self._readNoise = config.readNoise 

2432 self._saturation = config.saturation 

2433 

2434 def getBBox(self): 

2435 return self._bbox 

2436 

2437 def getRawBBox(self): 

2438 return self._bbox 

2439 

2440 def getHasRawInfo(self): 

2441 return True # but see getRawHorizontalOverscanBBox() 

2442 

2443 def getRawHorizontalOverscanBBox(self): 

2444 return self._RawHorizontalOverscanBBox 

2445 

2446 def getGain(self): 

2447 return self._gain 

2448 

2449 def getReadNoise(self): 

2450 return self._readNoise 

2451 

2452 def getSaturation(self): 

2453 return self._saturation 

2454 

2455 def getSuspectLevel(self): 

2456 return float("NaN") 

2457 

2458 

2459class RunIsrConfig(pexConfig.Config): 

2460 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2461 

2462 

2463class RunIsrTask(pipeBase.CmdLineTask): 

2464 """Task to wrap the default IsrTask to allow it to be retargeted. 

2465 

2466 The standard IsrTask can be called directly from a command line 

2467 program, but doing so removes the ability of the task to be 

2468 retargeted. As most cameras override some set of the IsrTask 

2469 methods, this would remove those data-specific methods in the 

2470 output post-ISR images. This wrapping class fixes the issue, 

2471 allowing identical post-ISR images to be generated by both the 

2472 processCcd and isrTask code. 

2473 """ 

2474 ConfigClass = RunIsrConfig 

2475 _DefaultName = "runIsr" 

2476 

2477 def __init__(self, *args, **kwargs): 

2478 super().__init__(*args, **kwargs) 

2479 self.makeSubtask("isr") 

2480 

2481 def runDataRef(self, dataRef): 

2482 """ 

2483 Parameters 

2484 ---------- 

2485 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2486 data reference of the detector data to be processed 

2487 

2488 Returns 

2489 ------- 

2490 result : `pipeBase.Struct` 

2491 Result struct with component: 

2492 

2493 - exposure : `lsst.afw.image.Exposure` 

2494 Post-ISR processed exposure. 

2495 """ 

2496 return self.isr.runDataRef(dataRef)