Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42from lsst.meas.algorithms import Defects 

43 

44from . import isrFunctions 

45from . import isrQa 

46from . import linearize 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .straylight import StrayLightTask 

54from .vignette import VignetteTask 

55 

56 

57__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

58 

59 

60class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

61 dimensions={"instrument", "visit", "detector"}, 

62 defaultTemplates={}): 

63 ccdExposure = cT.PrerequisiteInput( 

64 name="raw", 

65 doc="Input exposure to process.", 

66 storageClass="Exposure", 

67 dimensions=["instrument", "visit", "detector"], 

68 ) 

69 camera = cT.PrerequisiteInput( 

70 name="camera", 

71 storageClass="Camera", 

72 doc="Input camera to construct complete exposures.", 

73 dimensions=["instrument", "calibration_label"], 

74 ) 

75 bias = cT.PrerequisiteInput( 

76 name="bias", 

77 doc="Input bias calibration.", 

78 storageClass="ImageF", 

79 dimensions=["instrument", "calibration_label", "detector"], 

80 ) 

81 dark = cT.PrerequisiteInput( 

82 name='dark', 

83 doc="Input dark calibration.", 

84 storageClass="ImageF", 

85 dimensions=["instrument", "calibration_label", "detector"], 

86 ) 

87 flat = cT.PrerequisiteInput( 

88 name="flat", 

89 doc="Input flat calibration.", 

90 storageClass="MaskedImageF", 

91 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

92 ) 

93 fringes = cT.PrerequisiteInput( 

94 name="fringe", 

95 doc="Input fringe calibration.", 

96 storageClass="ExposureF", 

97 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

98 ) 

99 strayLightData = cT.PrerequisiteInput( 

100 name='yBackground', 

101 doc="Input stray light calibration.", 

102 storageClass="StrayLightData", 

103 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

104 ) 

105 bfKernel = cT.PrerequisiteInput( 

106 name='bfKernel', 

107 doc="Input brighter-fatter kernel.", 

108 storageClass="NumpyArray", 

109 dimensions=["instrument", "calibration_label"], 

110 ) 

111 newBFKernel = cT.PrerequisiteInput( 

112 name='brighterFatterKernel', 

113 doc="Newer complete kernel + gain solutions.", 

114 storageClass="BrighterFatterKernel", 

115 dimensions=["instrument", "calibration_label", "detector"], 

116 ) 

117 defects = cT.PrerequisiteInput( 

118 name='defects', 

119 doc="Input defect tables.", 

120 storageClass="DefectsList", 

121 dimensions=["instrument", "calibration_label", "detector"], 

122 ) 

123 opticsTransmission = cT.PrerequisiteInput( 

124 name="transmission_optics", 

125 storageClass="TransmissionCurve", 

126 doc="Transmission curve due to the optics.", 

127 dimensions=["instrument", "calibration_label"], 

128 ) 

129 filterTransmission = cT.PrerequisiteInput( 

130 name="transmission_filter", 

131 storageClass="TransmissionCurve", 

132 doc="Transmission curve due to the filter.", 

133 dimensions=["instrument", "physical_filter", "calibration_label"], 

134 ) 

135 sensorTransmission = cT.PrerequisiteInput( 

136 name="transmission_sensor", 

137 storageClass="TransmissionCurve", 

138 doc="Transmission curve due to the sensor.", 

139 dimensions=["instrument", "calibration_label", "detector"], 

140 ) 

141 atmosphereTransmission = cT.PrerequisiteInput( 

142 name="transmission_atmosphere", 

143 storageClass="TransmissionCurve", 

144 doc="Transmission curve due to the atmosphere.", 

145 dimensions=["instrument"], 

146 ) 

147 illumMaskedImage = cT.PrerequisiteInput( 

148 name="illum", 

149 doc="Input illumination correction.", 

150 storageClass="MaskedImageF", 

151 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

152 ) 

153 

154 outputExposure = cT.Output( 

155 name='postISRCCD', 

156 doc="Output ISR processed exposure.", 

157 storageClass="ExposureF", 

158 dimensions=["instrument", "visit", "detector"], 

159 ) 

160 preInterpExposure = cT.Output( 

161 name='preInterpISRCCD', 

162 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

163 storageClass="ExposureF", 

164 dimensions=["instrument", "visit", "detector"], 

165 ) 

166 outputOssThumbnail = cT.Output( 

167 name="OssThumb", 

168 doc="Output Overscan-subtracted thumbnail image.", 

169 storageClass="Thumbnail", 

170 dimensions=["instrument", "visit", "detector"], 

171 ) 

172 outputFlattenedThumbnail = cT.Output( 

173 name="FlattenedThumb", 

174 doc="Output flat-corrected thumbnail image.", 

175 storageClass="Thumbnail", 

176 dimensions=["instrument", "visit", "detector"], 

177 ) 

178 

179 def __init__(self, *, config=None): 

180 super().__init__(config=config) 

181 

182 if config.doBias is not True: 

183 self.prerequisiteInputs.discard("bias") 

184 if config.doLinearize is not True: 

185 self.prerequisiteInputs.discard("linearizer") 

186 if config.doCrosstalk is not True: 

187 self.prerequisiteInputs.discard("crosstalkSources") 

188 if config.doBrighterFatter is not True: 

189 self.prerequisiteInputs.discard("bfKernel") 

190 self.prerequisiteInputs.discard("newBFKernel") 

191 if config.doDefect is not True: 

192 self.prerequisiteInputs.discard("defects") 

193 if config.doDark is not True: 

194 self.prerequisiteInputs.discard("dark") 

195 if config.doFlat is not True: 

196 self.prerequisiteInputs.discard("flat") 

197 if config.doAttachTransmissionCurve is not True: 

198 self.prerequisiteInputs.discard("opticsTransmission") 

199 self.prerequisiteInputs.discard("filterTransmission") 

200 self.prerequisiteInputs.discard("sensorTransmission") 

201 self.prerequisiteInputs.discard("atmosphereTransmission") 

202 if config.doUseOpticsTransmission is not True: 

203 self.prerequisiteInputs.discard("opticsTransmission") 

204 if config.doUseFilterTransmission is not True: 

205 self.prerequisiteInputs.discard("filterTransmission") 

206 if config.doUseSensorTransmission is not True: 

207 self.prerequisiteInputs.discard("sensorTransmission") 

208 if config.doUseAtmosphereTransmission is not True: 

209 self.prerequisiteInputs.discard("atmosphereTransmission") 

210 if config.doIlluminationCorrection is not True: 

211 self.prerequisiteInputs.discard("illumMaskedImage") 

212 

213 if config.doWrite is not True: 

214 self.outputs.discard("outputExposure") 

215 self.outputs.discard("preInterpExposure") 

216 self.outputs.discard("outputFlattenedThumbnail") 

217 self.outputs.discard("outputOssThumbnail") 

218 if config.doSaveInterpPixels is not True: 

219 self.outputs.discard("preInterpExposure") 

220 if config.qa.doThumbnailOss is not True: 

221 self.outputs.discard("outputOssThumbnail") 

222 if config.qa.doThumbnailFlattened is not True: 

223 self.outputs.discard("outputFlattenedThumbnail") 

224 

225 

226class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

227 pipelineConnections=IsrTaskConnections): 

228 """Configuration parameters for IsrTask. 

229 

230 Items are grouped in the order in which they are executed by the task. 

231 """ 

232 datasetType = pexConfig.Field( 

233 dtype=str, 

234 doc="Dataset type for input data; users will typically leave this alone, " 

235 "but camera-specific ISR tasks will override it", 

236 default="raw", 

237 ) 

238 

239 fallbackFilterName = pexConfig.Field( 

240 dtype=str, 

241 doc="Fallback default filter name for calibrations.", 

242 optional=True 

243 ) 

244 useFallbackDate = pexConfig.Field( 

245 dtype=bool, 

246 doc="Pass observation date when using fallback filter.", 

247 default=False, 

248 ) 

249 expectWcs = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

253 ) 

254 fwhm = pexConfig.Field( 

255 dtype=float, 

256 doc="FWHM of PSF in arcseconds.", 

257 default=1.0, 

258 ) 

259 qa = pexConfig.ConfigField( 

260 dtype=isrQa.IsrQaConfig, 

261 doc="QA related configuration options.", 

262 ) 

263 

264 # Image conversion configuration 

265 doConvertIntToFloat = pexConfig.Field( 

266 dtype=bool, 

267 doc="Convert integer raw images to floating point values?", 

268 default=True, 

269 ) 

270 

271 # Saturated pixel handling. 

272 doSaturation = pexConfig.Field( 

273 dtype=bool, 

274 doc="Mask saturated pixels? NB: this is totally independent of the" 

275 " interpolation option - this is ONLY setting the bits in the mask." 

276 " To have them interpolated make sure doSaturationInterpolation=True", 

277 default=True, 

278 ) 

279 saturatedMaskName = pexConfig.Field( 

280 dtype=str, 

281 doc="Name of mask plane to use in saturation detection and interpolation", 

282 default="SAT", 

283 ) 

284 saturation = pexConfig.Field( 

285 dtype=float, 

286 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

287 default=float("NaN"), 

288 ) 

289 growSaturationFootprintSize = pexConfig.Field( 

290 dtype=int, 

291 doc="Number of pixels by which to grow the saturation footprints", 

292 default=1, 

293 ) 

294 

295 # Suspect pixel handling. 

296 doSuspect = pexConfig.Field( 

297 dtype=bool, 

298 doc="Mask suspect pixels?", 

299 default=False, 

300 ) 

301 suspectMaskName = pexConfig.Field( 

302 dtype=str, 

303 doc="Name of mask plane to use for suspect pixels", 

304 default="SUSPECT", 

305 ) 

306 numEdgeSuspect = pexConfig.Field( 

307 dtype=int, 

308 doc="Number of edge pixels to be flagged as untrustworthy.", 

309 default=0, 

310 ) 

311 

312 # Initial masking options. 

313 doSetBadRegions = pexConfig.Field( 

314 dtype=bool, 

315 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

316 default=True, 

317 ) 

318 badStatistic = pexConfig.ChoiceField( 

319 dtype=str, 

320 doc="How to estimate the average value for BAD regions.", 

321 default='MEANCLIP', 

322 allowed={ 

323 "MEANCLIP": "Correct using the (clipped) mean of good data", 

324 "MEDIAN": "Correct using the median of the good data", 

325 }, 

326 ) 

327 

328 # Overscan subtraction configuration. 

329 doOverscan = pexConfig.Field( 

330 dtype=bool, 

331 doc="Do overscan subtraction?", 

332 default=True, 

333 ) 

334 overscanFitType = pexConfig.ChoiceField( 

335 dtype=str, 

336 doc="The method for fitting the overscan bias level.", 

337 default='MEDIAN', 

338 allowed={ 

339 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

340 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

341 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

342 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

343 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

344 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

345 "MEAN": "Correct using the mean of the overscan region", 

346 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

347 "MEDIAN": "Correct using the median of the overscan region", 

348 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

349 }, 

350 ) 

351 overscanOrder = pexConfig.Field( 

352 dtype=int, 

353 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " + 

354 "or number of spline knots if overscan fit type is a spline."), 

355 default=1, 

356 ) 

357 overscanNumSigmaClip = pexConfig.Field( 

358 dtype=float, 

359 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

360 default=3.0, 

361 ) 

362 overscanIsInt = pexConfig.Field( 

363 dtype=bool, 

364 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" + 

365 " and overscan.FitType=MEDIAN_PER_ROW.", 

366 default=True, 

367 ) 

368 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

369 dtype=int, 

370 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

371 default=0, 

372 ) 

373 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

374 dtype=int, 

375 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

376 default=0, 

377 ) 

378 overscanMaxDev = pexConfig.Field( 378 ↛ exitline 378 didn't jump to the function exit

379 dtype=float, 

380 doc="Maximum deviation from the median for overscan", 

381 default=1000.0, check=lambda x: x > 0 

382 ) 

383 overscanBiasJump = pexConfig.Field( 

384 dtype=bool, 

385 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

386 default=False, 

387 ) 

388 overscanBiasJumpKeyword = pexConfig.Field( 

389 dtype=str, 

390 doc="Header keyword containing information about devices.", 

391 default="NO_SUCH_KEY", 

392 ) 

393 overscanBiasJumpDevices = pexConfig.ListField( 

394 dtype=str, 

395 doc="List of devices that need piecewise overscan correction.", 

396 default=(), 

397 ) 

398 overscanBiasJumpLocation = pexConfig.Field( 

399 dtype=int, 

400 doc="Location of bias jump along y-axis.", 

401 default=0, 

402 ) 

403 

404 # Amplifier to CCD assembly configuration 

405 doAssembleCcd = pexConfig.Field( 

406 dtype=bool, 

407 default=True, 

408 doc="Assemble amp-level exposures into a ccd-level exposure?" 

409 ) 

410 assembleCcd = pexConfig.ConfigurableField( 

411 target=AssembleCcdTask, 

412 doc="CCD assembly task", 

413 ) 

414 

415 # General calibration configuration. 

416 doAssembleIsrExposures = pexConfig.Field( 

417 dtype=bool, 

418 default=False, 

419 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

420 ) 

421 doTrimToMatchCalib = pexConfig.Field( 

422 dtype=bool, 

423 default=False, 

424 doc="Trim raw data to match calibration bounding boxes?" 

425 ) 

426 

427 # Bias subtraction. 

428 doBias = pexConfig.Field( 

429 dtype=bool, 

430 doc="Apply bias frame correction?", 

431 default=True, 

432 ) 

433 biasDataProductName = pexConfig.Field( 

434 dtype=str, 

435 doc="Name of the bias data product", 

436 default="bias", 

437 ) 

438 

439 # Variance construction 

440 doVariance = pexConfig.Field( 

441 dtype=bool, 

442 doc="Calculate variance?", 

443 default=True 

444 ) 

445 gain = pexConfig.Field( 

446 dtype=float, 

447 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

448 default=float("NaN"), 

449 ) 

450 readNoise = pexConfig.Field( 

451 dtype=float, 

452 doc="The read noise to use if no Detector is present in the Exposure", 

453 default=0.0, 

454 ) 

455 doEmpiricalReadNoise = pexConfig.Field( 

456 dtype=bool, 

457 default=False, 

458 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

459 ) 

460 

461 # Linearization. 

462 doLinearize = pexConfig.Field( 

463 dtype=bool, 

464 doc="Correct for nonlinearity of the detector's response?", 

465 default=True, 

466 ) 

467 

468 # Crosstalk. 

469 doCrosstalk = pexConfig.Field( 

470 dtype=bool, 

471 doc="Apply intra-CCD crosstalk correction?", 

472 default=False, 

473 ) 

474 doCrosstalkBeforeAssemble = pexConfig.Field( 

475 dtype=bool, 

476 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

477 default=False, 

478 ) 

479 crosstalk = pexConfig.ConfigurableField( 

480 target=CrosstalkTask, 

481 doc="Intra-CCD crosstalk correction", 

482 ) 

483 

484 # Masking options. 

485 doDefect = pexConfig.Field( 

486 dtype=bool, 

487 doc="Apply correction for CCD defects, e.g. hot pixels?", 

488 default=True, 

489 ) 

490 doNanMasking = pexConfig.Field( 

491 dtype=bool, 

492 doc="Mask NAN pixels?", 

493 default=True, 

494 ) 

495 doWidenSaturationTrails = pexConfig.Field( 

496 dtype=bool, 

497 doc="Widen bleed trails based on their width?", 

498 default=True 

499 ) 

500 

501 # Brighter-Fatter correction. 

502 doBrighterFatter = pexConfig.Field( 

503 dtype=bool, 

504 default=False, 

505 doc="Apply the brighter fatter correction" 

506 ) 

507 brighterFatterLevel = pexConfig.ChoiceField( 

508 dtype=str, 

509 default="DETECTOR", 

510 doc="The level at which to correct for brighter-fatter.", 

511 allowed={ 

512 "AMP": "Every amplifier treated separately.", 

513 "DETECTOR": "One kernel per detector", 

514 } 

515 ) 

516 brighterFatterMaxIter = pexConfig.Field( 

517 dtype=int, 

518 default=10, 

519 doc="Maximum number of iterations for the brighter fatter correction" 

520 ) 

521 brighterFatterThreshold = pexConfig.Field( 

522 dtype=float, 

523 default=1000, 

524 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

525 " absolute value of the difference between the current corrected image and the one" 

526 " from the previous iteration summed over all the pixels." 

527 ) 

528 brighterFatterApplyGain = pexConfig.Field( 

529 dtype=bool, 

530 default=True, 

531 doc="Should the gain be applied when applying the brighter fatter correction?" 

532 ) 

533 brighterFatterMaskGrowSize = pexConfig.Field( 

534 dtype=int, 

535 default=0, 

536 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

537 " when brighter-fatter correction is applied." 

538 ) 

539 

540 # Dark subtraction. 

541 doDark = pexConfig.Field( 

542 dtype=bool, 

543 doc="Apply dark frame correction?", 

544 default=True, 

545 ) 

546 darkDataProductName = pexConfig.Field( 

547 dtype=str, 

548 doc="Name of the dark data product", 

549 default="dark", 

550 ) 

551 

552 # Camera-specific stray light removal. 

553 doStrayLight = pexConfig.Field( 

554 dtype=bool, 

555 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

556 default=False, 

557 ) 

558 strayLight = pexConfig.ConfigurableField( 

559 target=StrayLightTask, 

560 doc="y-band stray light correction" 

561 ) 

562 

563 # Flat correction. 

564 doFlat = pexConfig.Field( 

565 dtype=bool, 

566 doc="Apply flat field correction?", 

567 default=True, 

568 ) 

569 flatDataProductName = pexConfig.Field( 

570 dtype=str, 

571 doc="Name of the flat data product", 

572 default="flat", 

573 ) 

574 flatScalingType = pexConfig.ChoiceField( 

575 dtype=str, 

576 doc="The method for scaling the flat on the fly.", 

577 default='USER', 

578 allowed={ 

579 "USER": "Scale by flatUserScale", 

580 "MEAN": "Scale by the inverse of the mean", 

581 "MEDIAN": "Scale by the inverse of the median", 

582 }, 

583 ) 

584 flatUserScale = pexConfig.Field( 

585 dtype=float, 

586 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

587 default=1.0, 

588 ) 

589 doTweakFlat = pexConfig.Field( 

590 dtype=bool, 

591 doc="Tweak flats to match observed amplifier ratios?", 

592 default=False 

593 ) 

594 

595 # Amplifier normalization based on gains instead of using flats configuration. 

596 doApplyGains = pexConfig.Field( 

597 dtype=bool, 

598 doc="Correct the amplifiers for their gains instead of applying flat correction", 

599 default=False, 

600 ) 

601 normalizeGains = pexConfig.Field( 

602 dtype=bool, 

603 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

604 default=False, 

605 ) 

606 

607 # Fringe correction. 

608 doFringe = pexConfig.Field( 

609 dtype=bool, 

610 doc="Apply fringe correction?", 

611 default=True, 

612 ) 

613 fringe = pexConfig.ConfigurableField( 

614 target=FringeTask, 

615 doc="Fringe subtraction task", 

616 ) 

617 fringeAfterFlat = pexConfig.Field( 

618 dtype=bool, 

619 doc="Do fringe subtraction after flat-fielding?", 

620 default=True, 

621 ) 

622 

623 # Distortion model application. 

624 doAddDistortionModel = pexConfig.Field( 

625 dtype=bool, 

626 doc="Apply a distortion model based on camera geometry to the WCS?", 

627 default=True, 

628 deprecated=("Camera geometry is incorporated when reading the raw files." 

629 " This option no longer is used, and will be removed after v19.") 

630 ) 

631 

632 # Initial CCD-level background statistics options. 

633 doMeasureBackground = pexConfig.Field( 

634 dtype=bool, 

635 doc="Measure the background level on the reduced image?", 

636 default=False, 

637 ) 

638 

639 # Camera-specific masking configuration. 

640 doCameraSpecificMasking = pexConfig.Field( 

641 dtype=bool, 

642 doc="Mask camera-specific bad regions?", 

643 default=False, 

644 ) 

645 masking = pexConfig.ConfigurableField( 

646 target=MaskingTask, 

647 doc="Masking task." 

648 ) 

649 

650 # Interpolation options. 

651 

652 doInterpolate = pexConfig.Field( 

653 dtype=bool, 

654 doc="Interpolate masked pixels?", 

655 default=True, 

656 ) 

657 doSaturationInterpolation = pexConfig.Field( 

658 dtype=bool, 

659 doc="Perform interpolation over pixels masked as saturated?" 

660 " NB: This is independent of doSaturation; if that is False this plane" 

661 " will likely be blank, resulting in a no-op here.", 

662 default=True, 

663 ) 

664 doNanInterpolation = pexConfig.Field( 

665 dtype=bool, 

666 doc="Perform interpolation over pixels masked as NaN?" 

667 " NB: This is independent of doNanMasking; if that is False this plane" 

668 " will likely be blank, resulting in a no-op here.", 

669 default=True, 

670 ) 

671 doNanInterpAfterFlat = pexConfig.Field( 

672 dtype=bool, 

673 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

674 "also have to interpolate them before flat-fielding."), 

675 default=False, 

676 ) 

677 maskListToInterpolate = pexConfig.ListField( 

678 dtype=str, 

679 doc="List of mask planes that should be interpolated.", 

680 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

681 ) 

682 doSaveInterpPixels = pexConfig.Field( 

683 dtype=bool, 

684 doc="Save a copy of the pre-interpolated pixel values?", 

685 default=False, 

686 ) 

687 

688 # Default photometric calibration options. 

689 fluxMag0T1 = pexConfig.DictField( 

690 keytype=str, 

691 itemtype=float, 

692 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

693 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

694 )) 

695 ) 

696 defaultFluxMag0T1 = pexConfig.Field( 

697 dtype=float, 

698 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

699 default=pow(10.0, 0.4*28.0) 

700 ) 

701 

702 # Vignette correction configuration. 

703 doVignette = pexConfig.Field( 

704 dtype=bool, 

705 doc="Apply vignetting parameters?", 

706 default=False, 

707 ) 

708 vignette = pexConfig.ConfigurableField( 

709 target=VignetteTask, 

710 doc="Vignetting task.", 

711 ) 

712 

713 # Transmission curve configuration. 

714 doAttachTransmissionCurve = pexConfig.Field( 

715 dtype=bool, 

716 default=False, 

717 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

718 ) 

719 doUseOpticsTransmission = pexConfig.Field( 

720 dtype=bool, 

721 default=True, 

722 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

723 ) 

724 doUseFilterTransmission = pexConfig.Field( 

725 dtype=bool, 

726 default=True, 

727 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

728 ) 

729 doUseSensorTransmission = pexConfig.Field( 

730 dtype=bool, 

731 default=True, 

732 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

733 ) 

734 doUseAtmosphereTransmission = pexConfig.Field( 

735 dtype=bool, 

736 default=True, 

737 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

738 ) 

739 

740 # Illumination correction. 

741 doIlluminationCorrection = pexConfig.Field( 

742 dtype=bool, 

743 default=False, 

744 doc="Perform illumination correction?" 

745 ) 

746 illuminationCorrectionDataProductName = pexConfig.Field( 

747 dtype=str, 

748 doc="Name of the illumination correction data product.", 

749 default="illumcor", 

750 ) 

751 illumScale = pexConfig.Field( 

752 dtype=float, 

753 doc="Scale factor for the illumination correction.", 

754 default=1.0, 

755 ) 

756 illumFilters = pexConfig.ListField( 

757 dtype=str, 

758 default=[], 

759 doc="Only perform illumination correction for these filters." 

760 ) 

761 

762 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

763 doWrite = pexConfig.Field( 

764 dtype=bool, 

765 doc="Persist postISRCCD?", 

766 default=True, 

767 ) 

768 

769 def validate(self): 

770 super().validate() 

771 if self.doFlat and self.doApplyGains: 

772 raise ValueError("You may not specify both doFlat and doApplyGains") 

773 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate: 

774 self.config.maskListToInterpolate.append("SAT") 

775 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

776 self.config.maskListToInterpolate.append("UNMASKEDNAN") 

777 

778 

779class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

780 """Apply common instrument signature correction algorithms to a raw frame. 

781 

782 The process for correcting imaging data is very similar from 

783 camera to camera. This task provides a vanilla implementation of 

784 doing these corrections, including the ability to turn certain 

785 corrections off if they are not needed. The inputs to the primary 

786 method, `run()`, are a raw exposure to be corrected and the 

787 calibration data products. The raw input is a single chip sized 

788 mosaic of all amps including overscans and other non-science 

789 pixels. The method `runDataRef()` identifies and defines the 

790 calibration data products, and is intended for use by a 

791 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

792 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

793 subclassed for different camera, although the most camera specific 

794 methods have been split into subtasks that can be redirected 

795 appropriately. 

796 

797 The __init__ method sets up the subtasks for ISR processing, using 

798 the defaults from `lsst.ip.isr`. 

799 

800 Parameters 

801 ---------- 

802 args : `list` 

803 Positional arguments passed to the Task constructor. None used at this time. 

804 kwargs : `dict`, optional 

805 Keyword arguments passed on to the Task constructor. None used at this time. 

806 """ 

807 ConfigClass = IsrTaskConfig 

808 _DefaultName = "isr" 

809 

810 def __init__(self, **kwargs): 

811 super().__init__(**kwargs) 

812 self.makeSubtask("assembleCcd") 

813 self.makeSubtask("crosstalk") 

814 self.makeSubtask("strayLight") 

815 self.makeSubtask("fringe") 

816 self.makeSubtask("masking") 

817 self.makeSubtask("vignette") 

818 

819 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

820 inputs = butlerQC.get(inputRefs) 

821 

822 try: 

823 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

824 except Exception as e: 

825 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

826 (inputRefs, e)) 

827 

828 inputs['isGen3'] = True 

829 

830 detector = inputs['ccdExposure'].getDetector() 

831 if self.doLinearize(detector) is True: 

832 if 'linearizer' not in inputs: 

833 linearityName = detector.getAmplifiers()[0].getLinearityType() 

834 inputs['linearizer'] = linearize.getLinearityTypeByName(linearityName)() 

835 

836 if self.config.doDefect is True: 

837 if "defects" in inputs and inputs['defects'] is not None: 

838 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

839 # masking expects a list of defects defined by their bounding box 

840 if not isinstance(inputs["defects"], Defects): 

841 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

842 

843 # Load the correct style of brighter fatter kernel, and repack 

844 # the information as a numpy array. 

845 if self.config.doBrighterFatter: 

846 brighterFatterKernel = inputs.pop('newBFKernel', None) 

847 if brighterFatterKernel is None: 

848 brighterFatterKernel = inputs.get('bfKernel', None) 

849 

850 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

851 detId = detector.getId() 

852 inputs['bfGains'] = brighterFatterKernel.gain 

853 # If the kernel is not an ndarray, it's the cp_pipe version 

854 # so extract the kernel for this detector, or raise an error 

855 if self.config.brighterFatterLevel == 'DETECTOR': 

856 if brighterFatterKernel.detectorKernel: 

857 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

858 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

859 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

860 else: 

861 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

862 else: 

863 # TODO DM-15631 for implementing this 

864 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

865 

866 # Broken: DM-17169 

867 # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be 

868 # fixed for non-HSC cameras in the future. 

869 # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure']) 

870 # if self.config.doCrosstalk else None) 

871 

872 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

873 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

874 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

875 expId=expId, 

876 assembler=self.assembleCcd 

877 if self.config.doAssembleIsrExposures else None) 

878 else: 

879 inputs['fringes'] = pipeBase.Struct(fringes=None) 

880 

881 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

882 if 'strayLightData' not in inputs: 

883 inputs['strayLightData'] = None 

884 

885 outputs = self.run(**inputs) 

886 butlerQC.put(outputs, outputRefs) 

887 

888 def readIsrData(self, dataRef, rawExposure): 

889 """!Retrieve necessary frames for instrument signature removal. 

890 

891 Pre-fetching all required ISR data products limits the IO 

892 required by the ISR. Any conflict between the calibration data 

893 available and that needed for ISR is also detected prior to 

894 doing processing, allowing it to fail quickly. 

895 

896 Parameters 

897 ---------- 

898 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

899 Butler reference of the detector data to be processed 

900 rawExposure : `afw.image.Exposure` 

901 The raw exposure that will later be corrected with the 

902 retrieved calibration data; should not be modified in this 

903 method. 

904 

905 Returns 

906 ------- 

907 result : `lsst.pipe.base.Struct` 

908 Result struct with components (which may be `None`): 

909 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

910 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

911 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

912 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

913 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

914 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

915 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 

916 - ``fringes``: `lsst.pipe.base.Struct` with components: 

917 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

918 - ``seed``: random seed derived from the ccdExposureId for random 

919 number generator (`uint32`). 

920 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

921 A ``TransmissionCurve`` that represents the throughput of the optics, 

922 to be evaluated in focal-plane coordinates. 

923 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

924 A ``TransmissionCurve`` that represents the throughput of the filter 

925 itself, to be evaluated in focal-plane coordinates. 

926 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

927 A ``TransmissionCurve`` that represents the throughput of the sensor 

928 itself, to be evaluated in post-assembly trimmed detector coordinates. 

929 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

930 A ``TransmissionCurve`` that represents the throughput of the 

931 atmosphere, assumed to be spatially constant. 

932 - ``strayLightData`` : `object` 

933 An opaque object containing calibration information for 

934 stray-light correction. If `None`, no correction will be 

935 performed. 

936 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

937 

938 Raises 

939 ------ 

940 NotImplementedError : 

941 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

942 """ 

943 try: 

944 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

945 dateObs = dateObs.toPython().isoformat() 

946 except RuntimeError: 

947 self.log.warn("Unable to identify dateObs for rawExposure.") 

948 dateObs = None 

949 

950 ccd = rawExposure.getDetector() 

951 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

952 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

953 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

954 if self.config.doBias else None) 

955 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

956 linearizer = (dataRef.get("linearizer", immediate=True) 

957 if self.doLinearize(ccd) else None) 

958 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef) 

959 if self.config.doCrosstalk else None) 

960 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

961 if self.config.doDark else None) 

962 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

963 dateObs=dateObs) 

964 if self.config.doFlat else None) 

965 

966 brighterFatterKernel = None 

967 brighterFatterGains = None 

968 if self.config.doBrighterFatter is True: 

969 try: 

970 # Use the new-style cp_pipe version of the kernel if it exists 

971 # If using a new-style kernel, always use the self-consistent 

972 # gains, i.e. the ones inside the kernel object itself 

973 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

974 brighterFatterGains = brighterFatterKernel.gain 

975 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

976 except NoResults: 

977 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

978 brighterFatterKernel = dataRef.get("bfKernel") 

979 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

980 except NoResults: 

981 brighterFatterKernel = None 

982 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

983 # If the kernel is not an ndarray, it's the cp_pipe version 

984 # so extract the kernel for this detector, or raise an error 

985 if self.config.brighterFatterLevel == 'DETECTOR': 

986 if brighterFatterKernel.detectorKernel: 

987 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

988 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

989 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

990 else: 

991 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

992 else: 

993 # TODO DM-15631 for implementing this 

994 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

995 

996 defectList = (dataRef.get("defects") 

997 if self.config.doDefect else None) 

998 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

999 if self.config.doAssembleIsrExposures else None) 

1000 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1001 else pipeBase.Struct(fringes=None)) 

1002 

1003 if self.config.doAttachTransmissionCurve: 

1004 opticsTransmission = (dataRef.get("transmission_optics") 

1005 if self.config.doUseOpticsTransmission else None) 

1006 filterTransmission = (dataRef.get("transmission_filter") 

1007 if self.config.doUseFilterTransmission else None) 

1008 sensorTransmission = (dataRef.get("transmission_sensor") 

1009 if self.config.doUseSensorTransmission else None) 

1010 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1011 if self.config.doUseAtmosphereTransmission else None) 

1012 else: 

1013 opticsTransmission = None 

1014 filterTransmission = None 

1015 sensorTransmission = None 

1016 atmosphereTransmission = None 

1017 

1018 if self.config.doStrayLight: 

1019 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1020 else: 

1021 strayLightData = None 

1022 

1023 illumMaskedImage = (self.getIsrExposure(dataRef, 

1024 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1025 if (self.config.doIlluminationCorrection and 

1026 filterName in self.config.illumFilters) 

1027 else None) 

1028 

1029 # Struct should include only kwargs to run() 

1030 return pipeBase.Struct(bias=biasExposure, 

1031 linearizer=linearizer, 

1032 crosstalkSources=crosstalkSources, 

1033 dark=darkExposure, 

1034 flat=flatExposure, 

1035 bfKernel=brighterFatterKernel, 

1036 bfGains=brighterFatterGains, 

1037 defects=defectList, 

1038 fringes=fringeStruct, 

1039 opticsTransmission=opticsTransmission, 

1040 filterTransmission=filterTransmission, 

1041 sensorTransmission=sensorTransmission, 

1042 atmosphereTransmission=atmosphereTransmission, 

1043 strayLightData=strayLightData, 

1044 illumMaskedImage=illumMaskedImage 

1045 ) 

1046 

1047 @pipeBase.timeMethod 

1048 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, 

1049 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1050 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1051 sensorTransmission=None, atmosphereTransmission=None, 

1052 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1053 isGen3=False, 

1054 ): 

1055 """!Perform instrument signature removal on an exposure. 

1056 

1057 Steps included in the ISR processing, in order performed, are: 

1058 - saturation and suspect pixel masking 

1059 - overscan subtraction 

1060 - CCD assembly of individual amplifiers 

1061 - bias subtraction 

1062 - variance image construction 

1063 - linearization of non-linear response 

1064 - crosstalk masking 

1065 - brighter-fatter correction 

1066 - dark subtraction 

1067 - fringe correction 

1068 - stray light subtraction 

1069 - flat correction 

1070 - masking of known defects and camera specific features 

1071 - vignette calculation 

1072 - appending transmission curve and distortion model 

1073 

1074 Parameters 

1075 ---------- 

1076 ccdExposure : `lsst.afw.image.Exposure` 

1077 The raw exposure that is to be run through ISR. The 

1078 exposure is modified by this method. 

1079 camera : `lsst.afw.cameraGeom.Camera`, optional 

1080 The camera geometry for this exposure. Used to select the 

1081 distortion model appropriate for this data. 

1082 bias : `lsst.afw.image.Exposure`, optional 

1083 Bias calibration frame. 

1084 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1085 Functor for linearization. 

1086 crosstalkSources : `list`, optional 

1087 List of possible crosstalk sources. 

1088 dark : `lsst.afw.image.Exposure`, optional 

1089 Dark calibration frame. 

1090 flat : `lsst.afw.image.Exposure`, optional 

1091 Flat calibration frame. 

1092 bfKernel : `numpy.ndarray`, optional 

1093 Brighter-fatter kernel. 

1094 bfGains : `dict` of `float`, optional 

1095 Gains used to override the detector's nominal gains for the 

1096 brighter-fatter correction. A dict keyed by amplifier name for 

1097 the detector in question. 

1098 defects : `lsst.meas.algorithms.Defects`, optional 

1099 List of defects. 

1100 fringes : `lsst.pipe.base.Struct`, optional 

1101 Struct containing the fringe correction data, with 

1102 elements: 

1103 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1104 - ``seed``: random seed derived from the ccdExposureId for random 

1105 number generator (`uint32`) 

1106 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1107 A ``TransmissionCurve`` that represents the throughput of the optics, 

1108 to be evaluated in focal-plane coordinates. 

1109 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1110 A ``TransmissionCurve`` that represents the throughput of the filter 

1111 itself, to be evaluated in focal-plane coordinates. 

1112 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1113 A ``TransmissionCurve`` that represents the throughput of the sensor 

1114 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1115 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1116 A ``TransmissionCurve`` that represents the throughput of the 

1117 atmosphere, assumed to be spatially constant. 

1118 detectorNum : `int`, optional 

1119 The integer number for the detector to process. 

1120 isGen3 : bool, optional 

1121 Flag this call to run() as using the Gen3 butler environment. 

1122 strayLightData : `object`, optional 

1123 Opaque object containing calibration information for stray-light 

1124 correction. If `None`, no correction will be performed. 

1125 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1126 Illumination correction image. 

1127 

1128 Returns 

1129 ------- 

1130 result : `lsst.pipe.base.Struct` 

1131 Result struct with component: 

1132 - ``exposure`` : `afw.image.Exposure` 

1133 The fully ISR corrected exposure. 

1134 - ``outputExposure`` : `afw.image.Exposure` 

1135 An alias for `exposure` 

1136 - ``ossThumb`` : `numpy.ndarray` 

1137 Thumbnail image of the exposure after overscan subtraction. 

1138 - ``flattenedThumb`` : `numpy.ndarray` 

1139 Thumbnail image of the exposure after flat-field correction. 

1140 

1141 Raises 

1142 ------ 

1143 RuntimeError 

1144 Raised if a configuration option is set to True, but the 

1145 required calibration data has not been specified. 

1146 

1147 Notes 

1148 ----- 

1149 The current processed exposure can be viewed by setting the 

1150 appropriate lsstDebug entries in the `debug.display` 

1151 dictionary. The names of these entries correspond to some of 

1152 the IsrTaskConfig Boolean options, with the value denoting the 

1153 frame to use. The exposure is shown inside the matching 

1154 option check and after the processing of that step has 

1155 finished. The steps with debug points are: 

1156 

1157 doAssembleCcd 

1158 doBias 

1159 doCrosstalk 

1160 doBrighterFatter 

1161 doDark 

1162 doFringe 

1163 doStrayLight 

1164 doFlat 

1165 

1166 In addition, setting the "postISRCCD" entry displays the 

1167 exposure after all ISR processing has finished. 

1168 

1169 """ 

1170 

1171 if isGen3 is True: 

1172 # Gen3 currently cannot automatically do configuration overrides. 

1173 # DM-15257 looks to discuss this issue. 

1174 # Configure input exposures; 

1175 if detectorNum is None: 

1176 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1177 

1178 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1179 bias = self.ensureExposure(bias, camera, detectorNum) 

1180 dark = self.ensureExposure(dark, camera, detectorNum) 

1181 flat = self.ensureExposure(flat, camera, detectorNum) 

1182 else: 

1183 if isinstance(ccdExposure, ButlerDataRef): 

1184 return self.runDataRef(ccdExposure) 

1185 

1186 ccd = ccdExposure.getDetector() 

1187 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1188 

1189 if not ccd: 

1190 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1191 ccd = [FakeAmp(ccdExposure, self.config)] 

1192 

1193 # Validate Input 

1194 if self.config.doBias and bias is None: 

1195 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1196 if self.doLinearize(ccd) and linearizer is None: 

1197 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1198 if self.config.doBrighterFatter and bfKernel is None: 

1199 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1200 if self.config.doDark and dark is None: 

1201 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1202 if self.config.doFlat and flat is None: 

1203 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1204 if self.config.doDefect and defects is None: 

1205 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1206 if (self.config.doFringe and filterName in self.fringe.config.filters and 

1207 fringes.fringes is None): 

1208 # The `fringes` object needs to be a pipeBase.Struct, as 

1209 # we use it as a `dict` for the parameters of 

1210 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1211 # not be `None` if `doFringe=True`. Otherwise, raise. 

1212 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1213 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and 

1214 illumMaskedImage is None): 

1215 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1216 

1217 # Begin ISR processing. 

1218 if self.config.doConvertIntToFloat: 

1219 self.log.info("Converting exposure to floating point values.") 

1220 ccdExposure = self.convertIntToFloat(ccdExposure) 

1221 

1222 # Amplifier level processing. 

1223 overscans = [] 

1224 for amp in ccd: 

1225 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1226 if ccdExposure.getBBox().contains(amp.getBBox()): 

1227 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1228 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1229 

1230 if self.config.doOverscan and not badAmp: 

1231 # Overscan correction on amp-by-amp basis. 

1232 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1233 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1234 if overscanResults is not None and \ 

1235 self.config.qa is not None and self.config.qa.saveStats is True: 

1236 if isinstance(overscanResults.overscanFit, float): 

1237 qaMedian = overscanResults.overscanFit 

1238 qaStdev = float("NaN") 

1239 else: 

1240 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1241 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1242 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1243 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1244 

1245 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian) 

1246 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev) 

1247 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1248 amp.getName(), qaMedian, qaStdev) 

1249 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1250 else: 

1251 if badAmp: 

1252 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1253 overscanResults = None 

1254 

1255 overscans.append(overscanResults if overscanResults is not None else None) 

1256 else: 

1257 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1258 

1259 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1260 self.log.info("Applying crosstalk correction.") 

1261 self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources) 

1262 self.debugView(ccdExposure, "doCrosstalk") 

1263 

1264 if self.config.doAssembleCcd: 

1265 self.log.info("Assembling CCD from amplifiers.") 

1266 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1267 

1268 if self.config.expectWcs and not ccdExposure.getWcs(): 

1269 self.log.warn("No WCS found in input exposure.") 

1270 self.debugView(ccdExposure, "doAssembleCcd") 

1271 

1272 ossThumb = None 

1273 if self.config.qa.doThumbnailOss: 

1274 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1275 

1276 if self.config.doBias: 

1277 self.log.info("Applying bias correction.") 

1278 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1279 trimToFit=self.config.doTrimToMatchCalib) 

1280 self.debugView(ccdExposure, "doBias") 

1281 

1282 if self.config.doVariance: 

1283 for amp, overscanResults in zip(ccd, overscans): 

1284 if ccdExposure.getBBox().contains(amp.getBBox()): 

1285 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1286 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1287 if overscanResults is not None: 

1288 self.updateVariance(ampExposure, amp, 

1289 overscanImage=overscanResults.overscanImage) 

1290 else: 

1291 self.updateVariance(ampExposure, amp, 

1292 overscanImage=None) 

1293 if self.config.qa is not None and self.config.qa.saveStats is True: 

1294 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1295 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1296 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1297 qaStats.getValue(afwMath.MEDIAN)) 

1298 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1299 qaStats.getValue(afwMath.STDEVCLIP)) 

1300 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1301 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1302 qaStats.getValue(afwMath.STDEVCLIP)) 

1303 

1304 if self.doLinearize(ccd): 

1305 self.log.info("Applying linearizer.") 

1306 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log) 

1307 

1308 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1309 self.log.info("Applying crosstalk correction.") 

1310 self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True) 

1311 self.debugView(ccdExposure, "doCrosstalk") 

1312 

1313 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1314 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1315 if self.config.doDefect: 

1316 self.log.info("Masking defects.") 

1317 self.maskDefect(ccdExposure, defects) 

1318 

1319 if self.config.numEdgeSuspect > 0: 

1320 self.log.info("Masking edges as SUSPECT.") 

1321 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1322 maskPlane="SUSPECT") 

1323 

1324 if self.config.doNanMasking: 

1325 self.log.info("Masking NAN value pixels.") 

1326 self.maskNan(ccdExposure) 

1327 

1328 if self.config.doWidenSaturationTrails: 

1329 self.log.info("Widening saturation trails.") 

1330 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1331 

1332 if self.config.doCameraSpecificMasking: 

1333 self.log.info("Masking regions for camera specific reasons.") 

1334 self.masking.run(ccdExposure) 

1335 

1336 if self.config.doBrighterFatter: 

1337 # We need to apply flats and darks before we can interpolate, and we 

1338 # need to interpolate before we do B-F, but we do B-F without the 

1339 # flats and darks applied so we can work in units of electrons or holes. 

1340 # This context manager applies and then removes the darks and flats. 

1341 # 

1342 # We also do not want to interpolate values here, so operate on temporary 

1343 # images so we can apply only the BF-correction and roll back the 

1344 # interpolation. 

1345 interpExp = ccdExposure.clone() 

1346 with self.flatContext(interpExp, flat, dark): 

1347 isrFunctions.interpolateFromMask( 

1348 maskedImage=interpExp.getMaskedImage(), 

1349 fwhm=self.config.fwhm, 

1350 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1351 maskNameList=self.config.maskListToInterpolate 

1352 ) 

1353 bfExp = interpExp.clone() 

1354 

1355 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1356 type(bfKernel), type(bfGains)) 

1357 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1358 self.config.brighterFatterMaxIter, 

1359 self.config.brighterFatterThreshold, 

1360 self.config.brighterFatterApplyGain, 

1361 bfGains) 

1362 if bfResults[1] == self.config.brighterFatterMaxIter: 

1363 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1364 bfResults[0]) 

1365 else: 

1366 self.log.info("Finished brighter fatter correction in %d iterations.", 

1367 bfResults[1]) 

1368 image = ccdExposure.getMaskedImage().getImage() 

1369 bfCorr = bfExp.getMaskedImage().getImage() 

1370 bfCorr -= interpExp.getMaskedImage().getImage() 

1371 image += bfCorr 

1372 

1373 # Applying the brighter-fatter correction applies a 

1374 # convolution to the science image. At the edges this 

1375 # convolution may not have sufficient valid pixels to 

1376 # produce a valid correction. Mark pixels within the size 

1377 # of the brighter-fatter kernel as EDGE to warn of this 

1378 # fact. 

1379 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1380 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1381 maskPlane="EDGE") 

1382 

1383 if self.config.brighterFatterMaskGrowSize > 0: 

1384 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1385 for maskPlane in self.config.maskListToInterpolate: 

1386 isrFunctions.growMasks(ccdExposure.getMask(), 

1387 radius=self.config.brighterFatterMaskGrowSize, 

1388 maskNameList=maskPlane, 

1389 maskValue=maskPlane) 

1390 

1391 self.debugView(ccdExposure, "doBrighterFatter") 

1392 

1393 if self.config.doDark: 

1394 self.log.info("Applying dark correction.") 

1395 self.darkCorrection(ccdExposure, dark) 

1396 self.debugView(ccdExposure, "doDark") 

1397 

1398 if self.config.doFringe and not self.config.fringeAfterFlat: 

1399 self.log.info("Applying fringe correction before flat.") 

1400 self.fringe.run(ccdExposure, **fringes.getDict()) 

1401 self.debugView(ccdExposure, "doFringe") 

1402 

1403 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1404 self.log.info("Checking strayLight correction.") 

1405 self.strayLight.run(ccdExposure, strayLightData) 

1406 self.debugView(ccdExposure, "doStrayLight") 

1407 

1408 if self.config.doFlat: 

1409 self.log.info("Applying flat correction.") 

1410 self.flatCorrection(ccdExposure, flat) 

1411 self.debugView(ccdExposure, "doFlat") 

1412 

1413 if self.config.doApplyGains: 

1414 self.log.info("Applying gain correction instead of flat.") 

1415 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1416 

1417 if self.config.doFringe and self.config.fringeAfterFlat: 

1418 self.log.info("Applying fringe correction after flat.") 

1419 self.fringe.run(ccdExposure, **fringes.getDict()) 

1420 

1421 if self.config.doVignette: 

1422 self.log.info("Constructing Vignette polygon.") 

1423 self.vignettePolygon = self.vignette.run(ccdExposure) 

1424 

1425 if self.config.vignette.doWriteVignettePolygon: 

1426 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1427 

1428 if self.config.doAttachTransmissionCurve: 

1429 self.log.info("Adding transmission curves.") 

1430 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1431 filterTransmission=filterTransmission, 

1432 sensorTransmission=sensorTransmission, 

1433 atmosphereTransmission=atmosphereTransmission) 

1434 

1435 flattenedThumb = None 

1436 if self.config.qa.doThumbnailFlattened: 

1437 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1438 

1439 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1440 self.log.info("Performing illumination correction.") 

1441 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1442 illumMaskedImage, illumScale=self.config.illumScale, 

1443 trimToFit=self.config.doTrimToMatchCalib) 

1444 

1445 preInterpExp = None 

1446 if self.config.doSaveInterpPixels: 

1447 preInterpExp = ccdExposure.clone() 

1448 

1449 # Reset and interpolate bad pixels. 

1450 # 

1451 # Large contiguous bad regions (which should have the BAD mask 

1452 # bit set) should have their values set to the image median. 

1453 # This group should include defects and bad amplifiers. As the 

1454 # area covered by these defects are large, there's little 

1455 # reason to expect that interpolation would provide a more 

1456 # useful value. 

1457 # 

1458 # Smaller defects can be safely interpolated after the larger 

1459 # regions have had their pixel values reset. This ensures 

1460 # that the remaining defects adjacent to bad amplifiers (as an 

1461 # example) do not attempt to interpolate extreme values. 

1462 if self.config.doSetBadRegions: 

1463 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1464 if badPixelCount > 0: 

1465 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1466 

1467 if self.config.doInterpolate: 

1468 self.log.info("Interpolating masked pixels.") 

1469 isrFunctions.interpolateFromMask( 

1470 maskedImage=ccdExposure.getMaskedImage(), 

1471 fwhm=self.config.fwhm, 

1472 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1473 maskNameList=list(self.config.maskListToInterpolate) 

1474 ) 

1475 

1476 self.roughZeroPoint(ccdExposure) 

1477 

1478 if self.config.doMeasureBackground: 

1479 self.log.info("Measuring background level.") 

1480 self.measureBackground(ccdExposure, self.config.qa) 

1481 

1482 if self.config.qa is not None and self.config.qa.saveStats is True: 

1483 for amp in ccd: 

1484 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1485 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1486 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1487 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1488 qaStats.getValue(afwMath.MEDIAN)) 

1489 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1490 qaStats.getValue(afwMath.STDEVCLIP)) 

1491 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1492 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1493 qaStats.getValue(afwMath.STDEVCLIP)) 

1494 

1495 self.debugView(ccdExposure, "postISRCCD") 

1496 

1497 return pipeBase.Struct( 

1498 exposure=ccdExposure, 

1499 ossThumb=ossThumb, 

1500 flattenedThumb=flattenedThumb, 

1501 

1502 preInterpolatedExposure=preInterpExp, 

1503 outputExposure=ccdExposure, 

1504 outputOssThumbnail=ossThumb, 

1505 outputFlattenedThumbnail=flattenedThumb, 

1506 ) 

1507 

1508 @pipeBase.timeMethod 

1509 def runDataRef(self, sensorRef): 

1510 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1511 

1512 This method contains the `CmdLineTask` interface to the ISR 

1513 processing. All IO is handled here, freeing the `run()` method 

1514 to manage only pixel-level calculations. The steps performed 

1515 are: 

1516 - Read in necessary detrending/isr/calibration data. 

1517 - Process raw exposure in `run()`. 

1518 - Persist the ISR-corrected exposure as "postISRCCD" if 

1519 config.doWrite=True. 

1520 

1521 Parameters 

1522 ---------- 

1523 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1524 DataRef of the detector data to be processed 

1525 

1526 Returns 

1527 ------- 

1528 result : `lsst.pipe.base.Struct` 

1529 Result struct with component: 

1530 - ``exposure`` : `afw.image.Exposure` 

1531 The fully ISR corrected exposure. 

1532 

1533 Raises 

1534 ------ 

1535 RuntimeError 

1536 Raised if a configuration option is set to True, but the 

1537 required calibration data does not exist. 

1538 

1539 """ 

1540 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1541 

1542 ccdExposure = sensorRef.get(self.config.datasetType) 

1543 

1544 camera = sensorRef.get("camera") 

1545 isrData = self.readIsrData(sensorRef, ccdExposure) 

1546 

1547 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1548 

1549 if self.config.doWrite: 

1550 sensorRef.put(result.exposure, "postISRCCD") 

1551 if result.preInterpolatedExposure is not None: 

1552 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1553 if result.ossThumb is not None: 

1554 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1555 if result.flattenedThumb is not None: 

1556 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1557 

1558 return result 

1559 

1560 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1561 """!Retrieve a calibration dataset for removing instrument signature. 

1562 

1563 Parameters 

1564 ---------- 

1565 

1566 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1567 DataRef of the detector data to find calibration datasets 

1568 for. 

1569 datasetType : `str` 

1570 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1571 dateObs : `str`, optional 

1572 Date of the observation. Used to correct butler failures 

1573 when using fallback filters. 

1574 immediate : `Bool` 

1575 If True, disable butler proxies to enable error handling 

1576 within this routine. 

1577 

1578 Returns 

1579 ------- 

1580 exposure : `lsst.afw.image.Exposure` 

1581 Requested calibration frame. 

1582 

1583 Raises 

1584 ------ 

1585 RuntimeError 

1586 Raised if no matching calibration frame can be found. 

1587 """ 

1588 try: 

1589 exp = dataRef.get(datasetType, immediate=immediate) 

1590 except Exception as exc1: 

1591 if not self.config.fallbackFilterName: 

1592 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1593 try: 

1594 if self.config.useFallbackDate and dateObs: 

1595 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1596 dateObs=dateObs, immediate=immediate) 

1597 else: 

1598 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1599 except Exception as exc2: 

1600 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1601 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1602 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1603 

1604 if self.config.doAssembleIsrExposures: 

1605 exp = self.assembleCcd.assembleCcd(exp) 

1606 return exp 

1607 

1608 def ensureExposure(self, inputExp, camera, detectorNum): 

1609 """Ensure that the data returned by Butler is a fully constructed exposure. 

1610 

1611 ISR requires exposure-level image data for historical reasons, so if we did 

1612 not recieve that from Butler, construct it from what we have, modifying the 

1613 input in place. 

1614 

1615 Parameters 

1616 ---------- 

1617 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1618 `lsst.afw.image.ImageF` 

1619 The input data structure obtained from Butler. 

1620 camera : `lsst.afw.cameraGeom.camera` 

1621 The camera associated with the image. Used to find the appropriate 

1622 detector. 

1623 detectorNum : `int` 

1624 The detector this exposure should match. 

1625 

1626 Returns 

1627 ------- 

1628 inputExp : `lsst.afw.image.Exposure` 

1629 The re-constructed exposure, with appropriate detector parameters. 

1630 

1631 Raises 

1632 ------ 

1633 TypeError 

1634 Raised if the input data cannot be used to construct an exposure. 

1635 """ 

1636 if isinstance(inputExp, afwImage.DecoratedImageU): 

1637 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1638 elif isinstance(inputExp, afwImage.ImageF): 

1639 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1640 elif isinstance(inputExp, afwImage.MaskedImageF): 

1641 inputExp = afwImage.makeExposure(inputExp) 

1642 elif isinstance(inputExp, afwImage.Exposure): 

1643 pass 

1644 elif inputExp is None: 

1645 # Assume this will be caught by the setup if it is a problem. 

1646 return inputExp 

1647 else: 

1648 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1649 (type(inputExp), )) 

1650 

1651 if inputExp.getDetector() is None: 

1652 inputExp.setDetector(camera[detectorNum]) 

1653 

1654 return inputExp 

1655 

1656 def convertIntToFloat(self, exposure): 

1657 """Convert exposure image from uint16 to float. 

1658 

1659 If the exposure does not need to be converted, the input is 

1660 immediately returned. For exposures that are converted to use 

1661 floating point pixels, the variance is set to unity and the 

1662 mask to zero. 

1663 

1664 Parameters 

1665 ---------- 

1666 exposure : `lsst.afw.image.Exposure` 

1667 The raw exposure to be converted. 

1668 

1669 Returns 

1670 ------- 

1671 newexposure : `lsst.afw.image.Exposure` 

1672 The input ``exposure``, converted to floating point pixels. 

1673 

1674 Raises 

1675 ------ 

1676 RuntimeError 

1677 Raised if the exposure type cannot be converted to float. 

1678 

1679 """ 

1680 if isinstance(exposure, afwImage.ExposureF): 

1681 # Nothing to be done 

1682 self.log.debug("Exposure already of type float.") 

1683 return exposure 

1684 if not hasattr(exposure, "convertF"): 

1685 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1686 

1687 newexposure = exposure.convertF() 

1688 newexposure.variance[:] = 1 

1689 newexposure.mask[:] = 0x0 

1690 

1691 return newexposure 

1692 

1693 def maskAmplifier(self, ccdExposure, amp, defects): 

1694 """Identify bad amplifiers, saturated and suspect pixels. 

1695 

1696 Parameters 

1697 ---------- 

1698 ccdExposure : `lsst.afw.image.Exposure` 

1699 Input exposure to be masked. 

1700 amp : `lsst.afw.table.AmpInfoCatalog` 

1701 Catalog of parameters defining the amplifier on this 

1702 exposure to mask. 

1703 defects : `lsst.meas.algorithms.Defects` 

1704 List of defects. Used to determine if the entire 

1705 amplifier is bad. 

1706 

1707 Returns 

1708 ------- 

1709 badAmp : `Bool` 

1710 If this is true, the entire amplifier area is covered by 

1711 defects and unusable. 

1712 

1713 """ 

1714 maskedImage = ccdExposure.getMaskedImage() 

1715 

1716 badAmp = False 

1717 

1718 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1719 # comparison with current defects definition. 

1720 if defects is not None: 

1721 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1722 

1723 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1724 # association with pixels in current ccdExposure). 

1725 if badAmp: 

1726 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1727 afwImage.PARENT) 

1728 maskView = dataView.getMask() 

1729 maskView |= maskView.getPlaneBitMask("BAD") 

1730 del maskView 

1731 return badAmp 

1732 

1733 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1734 # Saturation and suspect pixels can be masked now, though. 

1735 limits = dict() 

1736 if self.config.doSaturation and not badAmp: 

1737 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1738 if self.config.doSuspect and not badAmp: 

1739 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1740 if math.isfinite(self.config.saturation): 

1741 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1742 

1743 for maskName, maskThreshold in limits.items(): 

1744 if not math.isnan(maskThreshold): 

1745 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1746 isrFunctions.makeThresholdMask( 

1747 maskedImage=dataView, 

1748 threshold=maskThreshold, 

1749 growFootprints=0, 

1750 maskName=maskName 

1751 ) 

1752 

1753 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1754 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1755 afwImage.PARENT) 

1756 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1757 self.config.suspectMaskName]) 

1758 if numpy.all(maskView.getArray() & maskVal > 0): 

1759 badAmp = True 

1760 maskView |= maskView.getPlaneBitMask("BAD") 

1761 

1762 return badAmp 

1763 

1764 def overscanCorrection(self, ccdExposure, amp): 

1765 """Apply overscan correction in place. 

1766 

1767 This method does initial pixel rejection of the overscan 

1768 region. The overscan can also be optionally segmented to 

1769 allow for discontinuous overscan responses to be fit 

1770 separately. The actual overscan subtraction is performed by 

1771 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1772 which is called here after the amplifier is preprocessed. 

1773 

1774 Parameters 

1775 ---------- 

1776 ccdExposure : `lsst.afw.image.Exposure` 

1777 Exposure to have overscan correction performed. 

1778 amp : `lsst.afw.table.AmpInfoCatalog` 

1779 The amplifier to consider while correcting the overscan. 

1780 

1781 Returns 

1782 ------- 

1783 overscanResults : `lsst.pipe.base.Struct` 

1784 Result struct with components: 

1785 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1786 Value or fit subtracted from the amplifier image data. 

1787 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1788 Value or fit subtracted from the overscan image data. 

1789 - ``overscanImage`` : `lsst.afw.image.Image` 

1790 Image of the overscan region with the overscan 

1791 correction applied. This quantity is used to estimate 

1792 the amplifier read noise empirically. 

1793 

1794 Raises 

1795 ------ 

1796 RuntimeError 

1797 Raised if the ``amp`` does not contain raw pixel information. 

1798 

1799 See Also 

1800 -------- 

1801 lsst.ip.isr.isrFunctions.overscanCorrection 

1802 """ 

1803 if not amp.getHasRawInfo(): 

1804 raise RuntimeError("This method must be executed on an amp with raw information.") 

1805 

1806 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1807 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1808 return None 

1809 

1810 statControl = afwMath.StatisticsControl() 

1811 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1812 

1813 # Determine the bounding boxes 

1814 dataBBox = amp.getRawDataBBox() 

1815 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1816 dx0 = 0 

1817 dx1 = 0 

1818 

1819 prescanBBox = amp.getRawPrescanBBox() 

1820 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1821 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1822 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1823 else: 

1824 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1825 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1826 

1827 # Determine if we need to work on subregions of the amplifier and overscan. 

1828 imageBBoxes = [] 

1829 overscanBBoxes = [] 

1830 

1831 if ((self.config.overscanBiasJump and 

1832 self.config.overscanBiasJumpLocation) and 

1833 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and 

1834 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1835 self.config.overscanBiasJumpDevices)): 

1836 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1837 yLower = self.config.overscanBiasJumpLocation 

1838 yUpper = dataBBox.getHeight() - yLower 

1839 else: 

1840 yUpper = self.config.overscanBiasJumpLocation 

1841 yLower = dataBBox.getHeight() - yUpper 

1842 

1843 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1844 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1845 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + 

1846 lsst.geom.Extent2I(dx0, 0), 

1847 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1848 yLower))) 

1849 

1850 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1851 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1852 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1853 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1854 yUpper))) 

1855 else: 

1856 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1857 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1858 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1859 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1860 oscanBBox.getHeight()))) 

1861 

1862 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

1863 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1864 ampImage = ccdExposure.maskedImage[imageBBox] 

1865 overscanImage = ccdExposure.maskedImage[overscanBBox] 

1866 

1867 overscanArray = overscanImage.image.array 

1868 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

1869 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

1870 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

1871 

1872 statControl = afwMath.StatisticsControl() 

1873 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1874 

1875 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage, 

1876 overscanImage=overscanImage, 

1877 fitType=self.config.overscanFitType, 

1878 order=self.config.overscanOrder, 

1879 collapseRej=self.config.overscanNumSigmaClip, 

1880 statControl=statControl, 

1881 overscanIsInt=self.config.overscanIsInt 

1882 ) 

1883 

1884 # Measure average overscan levels and record them in the metadata. 

1885 levelStat = afwMath.MEDIAN 

1886 sigmaStat = afwMath.STDEVCLIP 

1887 

1888 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

1889 self.config.qa.flatness.nIter) 

1890 metadata = ccdExposure.getMetadata() 

1891 ampNum = amp.getName() 

1892 if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

1893 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

1894 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

1895 else: 

1896 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

1897 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

1898 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

1899 

1900 return overscanResults 

1901 

1902 def updateVariance(self, ampExposure, amp, overscanImage=None): 

1903 """Set the variance plane using the amplifier gain and read noise 

1904 

1905 The read noise is calculated from the ``overscanImage`` if the 

1906 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1907 the value from the amplifier data is used. 

1908 

1909 Parameters 

1910 ---------- 

1911 ampExposure : `lsst.afw.image.Exposure` 

1912 Exposure to process. 

1913 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

1914 Amplifier detector data. 

1915 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

1916 Image of overscan, required only for empirical read noise. 

1917 

1918 See also 

1919 -------- 

1920 lsst.ip.isr.isrFunctions.updateVariance 

1921 """ 

1922 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

1923 gain = amp.getGain() 

1924 

1925 if math.isnan(gain): 

1926 gain = 1.0 

1927 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

1928 elif gain <= 0: 

1929 patchedGain = 1.0 

1930 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

1931 amp.getName(), gain, patchedGain) 

1932 gain = patchedGain 

1933 

1934 if self.config.doEmpiricalReadNoise and overscanImage is None: 

1935 self.log.info("Overscan is none for EmpiricalReadNoise.") 

1936 

1937 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

1938 stats = afwMath.StatisticsControl() 

1939 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

1940 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

1941 self.log.info("Calculated empirical read noise for amp %s: %f.", 

1942 amp.getName(), readNoise) 

1943 else: 

1944 readNoise = amp.getReadNoise() 

1945 

1946 isrFunctions.updateVariance( 

1947 maskedImage=ampExposure.getMaskedImage(), 

1948 gain=gain, 

1949 readNoise=readNoise, 

1950 ) 

1951 

1952 def darkCorrection(self, exposure, darkExposure, invert=False): 

1953 """!Apply dark correction in place. 

1954 

1955 Parameters 

1956 ---------- 

1957 exposure : `lsst.afw.image.Exposure` 

1958 Exposure to process. 

1959 darkExposure : `lsst.afw.image.Exposure` 

1960 Dark exposure of the same size as ``exposure``. 

1961 invert : `Bool`, optional 

1962 If True, re-add the dark to an already corrected image. 

1963 

1964 Raises 

1965 ------ 

1966 RuntimeError 

1967 Raised if either ``exposure`` or ``darkExposure`` do not 

1968 have their dark time defined. 

1969 

1970 See Also 

1971 -------- 

1972 lsst.ip.isr.isrFunctions.darkCorrection 

1973 """ 

1974 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

1975 if math.isnan(expScale): 

1976 raise RuntimeError("Exposure darktime is NAN.") 

1977 if darkExposure.getInfo().getVisitInfo() is not None: 

1978 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

1979 else: 

1980 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

1981 # so getDarkTime() does not exist. 

1982 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

1983 darkScale = 1.0 

1984 

1985 if math.isnan(darkScale): 

1986 raise RuntimeError("Dark calib darktime is NAN.") 

1987 isrFunctions.darkCorrection( 

1988 maskedImage=exposure.getMaskedImage(), 

1989 darkMaskedImage=darkExposure.getMaskedImage(), 

1990 expScale=expScale, 

1991 darkScale=darkScale, 

1992 invert=invert, 

1993 trimToFit=self.config.doTrimToMatchCalib 

1994 ) 

1995 

1996 def doLinearize(self, detector): 

1997 """!Check if linearization is needed for the detector cameraGeom. 

1998 

1999 Checks config.doLinearize and the linearity type of the first 

2000 amplifier. 

2001 

2002 Parameters 

2003 ---------- 

2004 detector : `lsst.afw.cameraGeom.Detector` 

2005 Detector to get linearity type from. 

2006 

2007 Returns 

2008 ------- 

2009 doLinearize : `Bool` 

2010 If True, linearization should be performed. 

2011 """ 

2012 return self.config.doLinearize and \ 

2013 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2014 

2015 def flatCorrection(self, exposure, flatExposure, invert=False): 

2016 """!Apply flat correction in place. 

2017 

2018 Parameters 

2019 ---------- 

2020 exposure : `lsst.afw.image.Exposure` 

2021 Exposure to process. 

2022 flatExposure : `lsst.afw.image.Exposure` 

2023 Flat exposure of the same size as ``exposure``. 

2024 invert : `Bool`, optional 

2025 If True, unflatten an already flattened image. 

2026 

2027 See Also 

2028 -------- 

2029 lsst.ip.isr.isrFunctions.flatCorrection 

2030 """ 

2031 isrFunctions.flatCorrection( 

2032 maskedImage=exposure.getMaskedImage(), 

2033 flatMaskedImage=flatExposure.getMaskedImage(), 

2034 scalingType=self.config.flatScalingType, 

2035 userScale=self.config.flatUserScale, 

2036 invert=invert, 

2037 trimToFit=self.config.doTrimToMatchCalib 

2038 ) 

2039 

2040 def saturationDetection(self, exposure, amp): 

2041 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2042 

2043 Parameters 

2044 ---------- 

2045 exposure : `lsst.afw.image.Exposure` 

2046 Exposure to process. Only the amplifier DataSec is processed. 

2047 amp : `lsst.afw.table.AmpInfoCatalog` 

2048 Amplifier detector data. 

2049 

2050 See Also 

2051 -------- 

2052 lsst.ip.isr.isrFunctions.makeThresholdMask 

2053 """ 

2054 if not math.isnan(amp.getSaturation()): 

2055 maskedImage = exposure.getMaskedImage() 

2056 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2057 isrFunctions.makeThresholdMask( 

2058 maskedImage=dataView, 

2059 threshold=amp.getSaturation(), 

2060 growFootprints=0, 

2061 maskName=self.config.saturatedMaskName, 

2062 ) 

2063 

2064 def saturationInterpolation(self, exposure): 

2065 """!Interpolate over saturated pixels, in place. 

2066 

2067 This method should be called after `saturationDetection`, to 

2068 ensure that the saturated pixels have been identified in the 

2069 SAT mask. It should also be called after `assembleCcd`, since 

2070 saturated regions may cross amplifier boundaries. 

2071 

2072 Parameters 

2073 ---------- 

2074 exposure : `lsst.afw.image.Exposure` 

2075 Exposure to process. 

2076 

2077 See Also 

2078 -------- 

2079 lsst.ip.isr.isrTask.saturationDetection 

2080 lsst.ip.isr.isrFunctions.interpolateFromMask 

2081 """ 

2082 isrFunctions.interpolateFromMask( 

2083 maskedImage=exposure.getMaskedImage(), 

2084 fwhm=self.config.fwhm, 

2085 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2086 maskNameList=list(self.config.saturatedMaskName), 

2087 ) 

2088 

2089 def suspectDetection(self, exposure, amp): 

2090 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2091 

2092 Parameters 

2093 ---------- 

2094 exposure : `lsst.afw.image.Exposure` 

2095 Exposure to process. Only the amplifier DataSec is processed. 

2096 amp : `lsst.afw.table.AmpInfoCatalog` 

2097 Amplifier detector data. 

2098 

2099 See Also 

2100 -------- 

2101 lsst.ip.isr.isrFunctions.makeThresholdMask 

2102 

2103 Notes 

2104 ----- 

2105 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2106 This is intended to indicate pixels that may be affected by unknown systematics; 

2107 for example if non-linearity corrections above a certain level are unstable 

2108 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2109 that no such level exists and no pixels are to be masked as suspicious. 

2110 """ 

2111 suspectLevel = amp.getSuspectLevel() 

2112 if math.isnan(suspectLevel): 

2113 return 

2114 

2115 maskedImage = exposure.getMaskedImage() 

2116 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2117 isrFunctions.makeThresholdMask( 

2118 maskedImage=dataView, 

2119 threshold=suspectLevel, 

2120 growFootprints=0, 

2121 maskName=self.config.suspectMaskName, 

2122 ) 

2123 

2124 def maskDefect(self, exposure, defectBaseList): 

2125 """!Mask defects using mask plane "BAD", in place. 

2126 

2127 Parameters 

2128 ---------- 

2129 exposure : `lsst.afw.image.Exposure` 

2130 Exposure to process. 

2131 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2132 `lsst.afw.image.DefectBase`. 

2133 List of defects to mask. 

2134 

2135 Notes 

2136 ----- 

2137 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2138 """ 

2139 maskedImage = exposure.getMaskedImage() 

2140 if not isinstance(defectBaseList, Defects): 

2141 # Promotes DefectBase to Defect 

2142 defectList = Defects(defectBaseList) 

2143 else: 

2144 defectList = defectBaseList 

2145 defectList.maskPixels(maskedImage, maskName="BAD") 

2146 

2147 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"): 

2148 """!Mask edge pixels with applicable mask plane. 

2149 

2150 Parameters 

2151 ---------- 

2152 exposure : `lsst.afw.image.Exposure` 

2153 Exposure to process. 

2154 numEdgePixels : `int`, optional 

2155 Number of edge pixels to mask. 

2156 maskPlane : `str`, optional 

2157 Mask plane name to use. 

2158 """ 

2159 maskedImage = exposure.getMaskedImage() 

2160 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2161 

2162 if numEdgePixels > 0: 

2163 goodBBox = maskedImage.getBBox() 

2164 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2165 goodBBox.grow(-numEdgePixels) 

2166 # Mask pixels outside goodBBox 

2167 SourceDetectionTask.setEdgeBits( 

2168 maskedImage, 

2169 goodBBox, 

2170 maskBitMask 

2171 ) 

2172 

2173 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2174 """Mask and interpolate defects using mask plane "BAD", in place. 

2175 

2176 Parameters 

2177 ---------- 

2178 exposure : `lsst.afw.image.Exposure` 

2179 Exposure to process. 

2180 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2181 `lsst.afw.image.DefectBase`. 

2182 List of defects to mask and interpolate. 

2183 

2184 See Also 

2185 -------- 

2186 lsst.ip.isr.isrTask.maskDefect() 

2187 """ 

2188 self.maskDefect(exposure, defectBaseList) 

2189 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2190 maskPlane="SUSPECT") 

2191 isrFunctions.interpolateFromMask( 

2192 maskedImage=exposure.getMaskedImage(), 

2193 fwhm=self.config.fwhm, 

2194 growSaturatedFootprints=0, 

2195 maskNameList=["BAD"], 

2196 ) 

2197 

2198 def maskNan(self, exposure): 

2199 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2200 

2201 Parameters 

2202 ---------- 

2203 exposure : `lsst.afw.image.Exposure` 

2204 Exposure to process. 

2205 

2206 Notes 

2207 ----- 

2208 We mask over all NaNs, including those that are masked with 

2209 other bits (because those may or may not be interpolated over 

2210 later, and we want to remove all NaNs). Despite this 

2211 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2212 the historical name. 

2213 """ 

2214 maskedImage = exposure.getMaskedImage() 

2215 

2216 # Find and mask NaNs 

2217 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2218 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2219 numNans = maskNans(maskedImage, maskVal) 

2220 self.metadata.set("NUMNANS", numNans) 

2221 if numNans > 0: 

2222 self.log.warn("There were %d unmasked NaNs.", numNans) 

2223 

2224 def maskAndInterpolateNan(self, exposure): 

2225 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2226 

2227 Parameters 

2228 ---------- 

2229 exposure : `lsst.afw.image.Exposure` 

2230 Exposure to process. 

2231 

2232 See Also 

2233 -------- 

2234 lsst.ip.isr.isrTask.maskNan() 

2235 """ 

2236 self.maskNan(exposure) 

2237 isrFunctions.interpolateFromMask( 

2238 maskedImage=exposure.getMaskedImage(), 

2239 fwhm=self.config.fwhm, 

2240 growSaturatedFootprints=0, 

2241 maskNameList=["UNMASKEDNAN"], 

2242 ) 

2243 

2244 def measureBackground(self, exposure, IsrQaConfig=None): 

2245 """Measure the image background in subgrids, for quality control purposes. 

2246 

2247 Parameters 

2248 ---------- 

2249 exposure : `lsst.afw.image.Exposure` 

2250 Exposure to process. 

2251 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2252 Configuration object containing parameters on which background 

2253 statistics and subgrids to use. 

2254 """ 

2255 if IsrQaConfig is not None: 

2256 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2257 IsrQaConfig.flatness.nIter) 

2258 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2259 statsControl.setAndMask(maskVal) 

2260 maskedImage = exposure.getMaskedImage() 

2261 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2262 skyLevel = stats.getValue(afwMath.MEDIAN) 

2263 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2264 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2265 metadata = exposure.getMetadata() 

2266 metadata.set('SKYLEVEL', skyLevel) 

2267 metadata.set('SKYSIGMA', skySigma) 

2268 

2269 # calcluating flatlevel over the subgrids 

2270 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2271 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2272 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2273 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2274 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2275 skyLevels = numpy.zeros((nX, nY)) 

2276 

2277 for j in range(nY): 

2278 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2279 for i in range(nX): 

2280 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2281 

2282 xLLC = xc - meshXHalf 

2283 yLLC = yc - meshYHalf 

2284 xURC = xc + meshXHalf - 1 

2285 yURC = yc + meshYHalf - 1 

2286 

2287 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2288 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2289 

2290 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2291 

2292 good = numpy.where(numpy.isfinite(skyLevels)) 

2293 skyMedian = numpy.median(skyLevels[good]) 

2294 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2295 flatness_rms = numpy.std(flatness) 

2296 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2297 

2298 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2299 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2300 nX, nY, flatness_pp, flatness_rms) 

2301 

2302 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2303 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2304 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2305 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2306 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2307 

2308 def roughZeroPoint(self, exposure): 

2309 """Set an approximate magnitude zero point for the exposure. 

2310 

2311 Parameters 

2312 ---------- 

2313 exposure : `lsst.afw.image.Exposure` 

2314 Exposure to process. 

2315 """ 

2316 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2317 if filterName in self.config.fluxMag0T1: 

2318 fluxMag0 = self.config.fluxMag0T1[filterName] 

2319 else: 

2320 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2321 fluxMag0 = self.config.defaultFluxMag0T1 

2322 

2323 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2324 if not expTime > 0: # handle NaN as well as <= 0 

2325 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2326 return 

2327 

2328 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2329 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2330 

2331 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2332 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2333 

2334 Parameters 

2335 ---------- 

2336 ccdExposure : `lsst.afw.image.Exposure` 

2337 Exposure to process. 

2338 fpPolygon : `lsst.afw.geom.Polygon` 

2339 Polygon in focal plane coordinates. 

2340 """ 

2341 # Get ccd corners in focal plane coordinates 

2342 ccd = ccdExposure.getDetector() 

2343 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2344 ccdPolygon = Polygon(fpCorners) 

2345 

2346 # Get intersection of ccd corners with fpPolygon 

2347 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2348 

2349 # Transform back to pixel positions and build new polygon 

2350 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2351 validPolygon = Polygon(ccdPoints) 

2352 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2353 

2354 @contextmanager 

2355 def flatContext(self, exp, flat, dark=None): 

2356 """Context manager that applies and removes flats and darks, 

2357 if the task is configured to apply them. 

2358 

2359 Parameters 

2360 ---------- 

2361 exp : `lsst.afw.image.Exposure` 

2362 Exposure to process. 

2363 flat : `lsst.afw.image.Exposure` 

2364 Flat exposure the same size as ``exp``. 

2365 dark : `lsst.afw.image.Exposure`, optional 

2366 Dark exposure the same size as ``exp``. 

2367 

2368 Yields 

2369 ------ 

2370 exp : `lsst.afw.image.Exposure` 

2371 The flat and dark corrected exposure. 

2372 """ 

2373 if self.config.doDark and dark is not None: 

2374 self.darkCorrection(exp, dark) 

2375 if self.config.doFlat: 

2376 self.flatCorrection(exp, flat) 

2377 try: 

2378 yield exp 

2379 finally: 

2380 if self.config.doFlat: 

2381 self.flatCorrection(exp, flat, invert=True) 

2382 if self.config.doDark and dark is not None: 

2383 self.darkCorrection(exp, dark, invert=True) 

2384 

2385 def debugView(self, exposure, stepname): 

2386 """Utility function to examine ISR exposure at different stages. 

2387 

2388 Parameters 

2389 ---------- 

2390 exposure : `lsst.afw.image.Exposure` 

2391 Exposure to view. 

2392 stepname : `str` 

2393 State of processing to view. 

2394 """ 

2395 frame = getDebugFrame(self._display, stepname) 

2396 if frame: 

2397 display = getDisplay(frame) 

2398 display.scale('asinh', 'zscale') 

2399 display.mtv(exposure) 

2400 prompt = "Press Enter to continue [c]... " 

2401 while True: 

2402 ans = input(prompt).lower() 

2403 if ans in ("", "c",): 

2404 break 

2405 

2406 

2407class FakeAmp(object): 

2408 """A Detector-like object that supports returning gain and saturation level 

2409 

2410 This is used when the input exposure does not have a detector. 

2411 

2412 Parameters 

2413 ---------- 

2414 exposure : `lsst.afw.image.Exposure` 

2415 Exposure to generate a fake amplifier for. 

2416 config : `lsst.ip.isr.isrTaskConfig` 

2417 Configuration to apply to the fake amplifier. 

2418 """ 

2419 

2420 def __init__(self, exposure, config): 

2421 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2422 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2423 self._gain = config.gain 

2424 self._readNoise = config.readNoise 

2425 self._saturation = config.saturation 

2426 

2427 def getBBox(self): 

2428 return self._bbox 

2429 

2430 def getRawBBox(self): 

2431 return self._bbox 

2432 

2433 def getHasRawInfo(self): 

2434 return True # but see getRawHorizontalOverscanBBox() 

2435 

2436 def getRawHorizontalOverscanBBox(self): 

2437 return self._RawHorizontalOverscanBBox 

2438 

2439 def getGain(self): 

2440 return self._gain 

2441 

2442 def getReadNoise(self): 

2443 return self._readNoise 

2444 

2445 def getSaturation(self): 

2446 return self._saturation 

2447 

2448 def getSuspectLevel(self): 

2449 return float("NaN") 

2450 

2451 

2452class RunIsrConfig(pexConfig.Config): 

2453 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2454 

2455 

2456class RunIsrTask(pipeBase.CmdLineTask): 

2457 """Task to wrap the default IsrTask to allow it to be retargeted. 

2458 

2459 The standard IsrTask can be called directly from a command line 

2460 program, but doing so removes the ability of the task to be 

2461 retargeted. As most cameras override some set of the IsrTask 

2462 methods, this would remove those data-specific methods in the 

2463 output post-ISR images. This wrapping class fixes the issue, 

2464 allowing identical post-ISR images to be generated by both the 

2465 processCcd and isrTask code. 

2466 """ 

2467 ConfigClass = RunIsrConfig 

2468 _DefaultName = "runIsr" 

2469 

2470 def __init__(self, *args, **kwargs): 

2471 super().__init__(*args, **kwargs) 

2472 self.makeSubtask("isr") 

2473 

2474 def runDataRef(self, dataRef): 

2475 """ 

2476 Parameters 

2477 ---------- 

2478 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2479 data reference of the detector data to be processed 

2480 

2481 Returns 

2482 ------- 

2483 result : `pipeBase.Struct` 

2484 Result struct with component: 

2485 

2486 - exposure : `lsst.afw.image.Exposure` 

2487 Post-ISR processed exposure. 

2488 """ 

2489 return self.isr.runDataRef(dataRef)