Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42from lsst.meas.algorithms import Defects 

43 

44from . import isrFunctions 

45from . import isrQa 

46from . import linearize 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DataCoordinate, DimensionGraph, DimensionUniverse 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.afw.image.Exposure`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = DataCoordinate(DimensionGraph(DimensionUniverse(), 

91 names=('instrument', 'exposure')), 

92 (quantumDataId['instrument'], quantumDataId['exposure'])) 

93 results = list(registry.queryDatasets(datasetType, 

94 collections=collections, 

95 dataId=newDataId, 

96 deduplicate=True, 

97 expand=True)) 

98 return(results) 

99 

100 

101class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

102 dimensions={"instrument", "exposure", "detector"}, 

103 defaultTemplates={}): 

104 ccdExposure = cT.Input( 

105 name="raw", 

106 doc="Input exposure to process.", 

107 storageClass="Exposure", 

108 dimensions=["instrument", "exposure", "detector"], 

109 ) 

110 camera = cT.PrerequisiteInput( 

111 name="camera", 

112 storageClass="Camera", 

113 doc="Input camera to construct complete exposures.", 

114 dimensions=["instrument", "calibration_label"], 

115 ) 

116 

117 crosstalk = cT.PrerequisiteInput( 

118 name="crosstalk", 

119 doc="Input crosstalk object", 

120 storageClass="CrosstalkCalib", 

121 dimensions=["instrument", "calibration_label", "detector"], 

122 ) 

123 # TODO: DM-25348. This does not work yet to correctly load 

124 # possible crosstalk sources. 

125 crosstalkSources = cT.PrerequisiteInput( 

126 name="isrOverscanCorrected", 

127 doc="Overscan corrected input images.", 

128 storageClass="Exposure", 

129 dimensions=["instrument", "exposure", "detector"], 

130 deferLoad=True, 

131 multiple=True, 

132 lookupFunction=crosstalkSourceLookup, 

133 ) 

134 bias = cT.PrerequisiteInput( 

135 name="bias", 

136 doc="Input bias calibration.", 

137 storageClass="ExposureF", 

138 dimensions=["instrument", "calibration_label", "detector"], 

139 ) 

140 dark = cT.PrerequisiteInput( 

141 name='dark', 

142 doc="Input dark calibration.", 

143 storageClass="ExposureF", 

144 dimensions=["instrument", "calibration_label", "detector"], 

145 ) 

146 flat = cT.PrerequisiteInput( 

147 name="flat", 

148 doc="Input flat calibration.", 

149 storageClass="ExposureF", 

150 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

151 ) 

152 fringes = cT.PrerequisiteInput( 

153 name="fringe", 

154 doc="Input fringe calibration.", 

155 storageClass="ExposureF", 

156 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

157 ) 

158 strayLightData = cT.PrerequisiteInput( 

159 name='yBackground', 

160 doc="Input stray light calibration.", 

161 storageClass="StrayLightData", 

162 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

163 ) 

164 bfKernel = cT.PrerequisiteInput( 

165 name='bfKernel', 

166 doc="Input brighter-fatter kernel.", 

167 storageClass="NumpyArray", 

168 dimensions=["instrument", "calibration_label"], 

169 ) 

170 newBFKernel = cT.PrerequisiteInput( 

171 name='brighterFatterKernel', 

172 doc="Newer complete kernel + gain solutions.", 

173 storageClass="BrighterFatterKernel", 

174 dimensions=["instrument", "calibration_label", "detector"], 

175 ) 

176 defects = cT.PrerequisiteInput( 

177 name='defects', 

178 doc="Input defect tables.", 

179 storageClass="Defects", 

180 dimensions=["instrument", "calibration_label", "detector"], 

181 ) 

182 opticsTransmission = cT.PrerequisiteInput( 

183 name="transmission_optics", 

184 storageClass="TransmissionCurve", 

185 doc="Transmission curve due to the optics.", 

186 dimensions=["instrument", "calibration_label"], 

187 ) 

188 filterTransmission = cT.PrerequisiteInput( 

189 name="transmission_filter", 

190 storageClass="TransmissionCurve", 

191 doc="Transmission curve due to the filter.", 

192 dimensions=["instrument", "physical_filter", "calibration_label"], 

193 ) 

194 sensorTransmission = cT.PrerequisiteInput( 

195 name="transmission_sensor", 

196 storageClass="TransmissionCurve", 

197 doc="Transmission curve due to the sensor.", 

198 dimensions=["instrument", "calibration_label", "detector"], 

199 ) 

200 atmosphereTransmission = cT.PrerequisiteInput( 

201 name="transmission_atmosphere", 

202 storageClass="TransmissionCurve", 

203 doc="Transmission curve due to the atmosphere.", 

204 dimensions=["instrument"], 

205 ) 

206 illumMaskedImage = cT.PrerequisiteInput( 

207 name="illum", 

208 doc="Input illumination correction.", 

209 storageClass="MaskedImageF", 

210 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

211 ) 

212 

213 outputExposure = cT.Output( 

214 name='postISRCCD', 

215 doc="Output ISR processed exposure.", 

216 storageClass="Exposure", 

217 dimensions=["instrument", "exposure", "detector"], 

218 ) 

219 preInterpExposure = cT.Output( 

220 name='preInterpISRCCD', 

221 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

222 storageClass="ExposureF", 

223 dimensions=["instrument", "exposure", "detector"], 

224 ) 

225 outputOssThumbnail = cT.Output( 

226 name="OssThumb", 

227 doc="Output Overscan-subtracted thumbnail image.", 

228 storageClass="Thumbnail", 

229 dimensions=["instrument", "exposure", "detector"], 

230 ) 

231 outputFlattenedThumbnail = cT.Output( 

232 name="FlattenedThumb", 

233 doc="Output flat-corrected thumbnail image.", 

234 storageClass="Thumbnail", 

235 dimensions=["instrument", "exposure", "detector"], 

236 ) 

237 

238 def __init__(self, *, config=None): 

239 super().__init__(config=config) 

240 

241 if config.doBias is not True: 

242 self.prerequisiteInputs.discard("bias") 

243 if config.doLinearize is not True: 

244 self.prerequisiteInputs.discard("linearizer") 

245 if config.doCrosstalk is not True: 

246 self.inputs.discard("crosstalkSources") 

247 self.prerequisiteInputs.discard("crosstalk") 

248 if config.doBrighterFatter is not True: 

249 self.prerequisiteInputs.discard("bfKernel") 

250 self.prerequisiteInputs.discard("newBFKernel") 

251 if config.doDefect is not True: 

252 self.prerequisiteInputs.discard("defects") 

253 if config.doDark is not True: 

254 self.prerequisiteInputs.discard("dark") 

255 if config.doFlat is not True: 

256 self.prerequisiteInputs.discard("flat") 

257 if config.doAttachTransmissionCurve is not True: 

258 self.prerequisiteInputs.discard("opticsTransmission") 

259 self.prerequisiteInputs.discard("filterTransmission") 

260 self.prerequisiteInputs.discard("sensorTransmission") 

261 self.prerequisiteInputs.discard("atmosphereTransmission") 

262 if config.doUseOpticsTransmission is not True: 

263 self.prerequisiteInputs.discard("opticsTransmission") 

264 if config.doUseFilterTransmission is not True: 

265 self.prerequisiteInputs.discard("filterTransmission") 

266 if config.doUseSensorTransmission is not True: 

267 self.prerequisiteInputs.discard("sensorTransmission") 

268 if config.doUseAtmosphereTransmission is not True: 

269 self.prerequisiteInputs.discard("atmosphereTransmission") 

270 if config.doIlluminationCorrection is not True: 

271 self.prerequisiteInputs.discard("illumMaskedImage") 

272 

273 if config.doWrite is not True: 

274 self.outputs.discard("outputExposure") 

275 self.outputs.discard("preInterpExposure") 

276 self.outputs.discard("outputFlattenedThumbnail") 

277 self.outputs.discard("outputOssThumbnail") 

278 if config.doSaveInterpPixels is not True: 

279 self.outputs.discard("preInterpExposure") 

280 if config.qa.doThumbnailOss is not True: 

281 self.outputs.discard("outputOssThumbnail") 

282 if config.qa.doThumbnailFlattened is not True: 

283 self.outputs.discard("outputFlattenedThumbnail") 

284 

285 

286class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

287 pipelineConnections=IsrTaskConnections): 

288 """Configuration parameters for IsrTask. 

289 

290 Items are grouped in the order in which they are executed by the task. 

291 """ 

292 datasetType = pexConfig.Field( 

293 dtype=str, 

294 doc="Dataset type for input data; users will typically leave this alone, " 

295 "but camera-specific ISR tasks will override it", 

296 default="raw", 

297 ) 

298 

299 fallbackFilterName = pexConfig.Field( 

300 dtype=str, 

301 doc="Fallback default filter name for calibrations.", 

302 optional=True 

303 ) 

304 useFallbackDate = pexConfig.Field( 

305 dtype=bool, 

306 doc="Pass observation date when using fallback filter.", 

307 default=False, 

308 ) 

309 expectWcs = pexConfig.Field( 

310 dtype=bool, 

311 default=True, 

312 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

313 ) 

314 fwhm = pexConfig.Field( 

315 dtype=float, 

316 doc="FWHM of PSF in arcseconds.", 

317 default=1.0, 

318 ) 

319 qa = pexConfig.ConfigField( 

320 dtype=isrQa.IsrQaConfig, 

321 doc="QA related configuration options.", 

322 ) 

323 

324 # Image conversion configuration 

325 doConvertIntToFloat = pexConfig.Field( 

326 dtype=bool, 

327 doc="Convert integer raw images to floating point values?", 

328 default=True, 

329 ) 

330 

331 # Saturated pixel handling. 

332 doSaturation = pexConfig.Field( 

333 dtype=bool, 

334 doc="Mask saturated pixels? NB: this is totally independent of the" 

335 " interpolation option - this is ONLY setting the bits in the mask." 

336 " To have them interpolated make sure doSaturationInterpolation=True", 

337 default=True, 

338 ) 

339 saturatedMaskName = pexConfig.Field( 

340 dtype=str, 

341 doc="Name of mask plane to use in saturation detection and interpolation", 

342 default="SAT", 

343 ) 

344 saturation = pexConfig.Field( 

345 dtype=float, 

346 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

347 default=float("NaN"), 

348 ) 

349 growSaturationFootprintSize = pexConfig.Field( 

350 dtype=int, 

351 doc="Number of pixels by which to grow the saturation footprints", 

352 default=1, 

353 ) 

354 

355 # Suspect pixel handling. 

356 doSuspect = pexConfig.Field( 

357 dtype=bool, 

358 doc="Mask suspect pixels?", 

359 default=False, 

360 ) 

361 suspectMaskName = pexConfig.Field( 

362 dtype=str, 

363 doc="Name of mask plane to use for suspect pixels", 

364 default="SUSPECT", 

365 ) 

366 numEdgeSuspect = pexConfig.Field( 

367 dtype=int, 

368 doc="Number of edge pixels to be flagged as untrustworthy.", 

369 default=0, 

370 ) 

371 

372 # Initial masking options. 

373 doSetBadRegions = pexConfig.Field( 

374 dtype=bool, 

375 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

376 default=True, 

377 ) 

378 badStatistic = pexConfig.ChoiceField( 

379 dtype=str, 

380 doc="How to estimate the average value for BAD regions.", 

381 default='MEANCLIP', 

382 allowed={ 

383 "MEANCLIP": "Correct using the (clipped) mean of good data", 

384 "MEDIAN": "Correct using the median of the good data", 

385 }, 

386 ) 

387 

388 # Overscan subtraction configuration. 

389 doOverscan = pexConfig.Field( 

390 dtype=bool, 

391 doc="Do overscan subtraction?", 

392 default=True, 

393 ) 

394 overscan = pexConfig.ConfigurableField( 

395 target=OverscanCorrectionTask, 

396 doc="Overscan subtraction task for image segments.", 

397 ) 

398 

399 overscanFitType = pexConfig.ChoiceField( 

400 dtype=str, 

401 doc="The method for fitting the overscan bias level.", 

402 default='MEDIAN', 

403 allowed={ 

404 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

405 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

406 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

407 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

408 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

409 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

410 "MEAN": "Correct using the mean of the overscan region", 

411 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

412 "MEDIAN": "Correct using the median of the overscan region", 

413 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

414 }, 

415 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

416 " This option will no longer be used, and will be removed after v20.") 

417 ) 

418 overscanOrder = pexConfig.Field( 

419 dtype=int, 

420 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

421 "or number of spline knots if overscan fit type is a spline."), 

422 default=1, 

423 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

424 " This option will no longer be used, and will be removed after v20.") 

425 ) 

426 overscanNumSigmaClip = pexConfig.Field( 

427 dtype=float, 

428 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

429 default=3.0, 

430 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

431 " This option will no longer be used, and will be removed after v20.") 

432 ) 

433 overscanIsInt = pexConfig.Field( 

434 dtype=bool, 

435 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

436 " and overscan.FitType=MEDIAN_PER_ROW.", 

437 default=True, 

438 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

439 " This option will no longer be used, and will be removed after v20.") 

440 ) 

441 # These options do not get deprecated, as they define how we slice up the image data. 

442 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

443 dtype=int, 

444 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

445 default=0, 

446 ) 

447 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

448 dtype=int, 

449 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

450 default=0, 

451 ) 

452 overscanMaxDev = pexConfig.Field( 452 ↛ exitline 452 didn't jump to the function exit

453 dtype=float, 

454 doc="Maximum deviation from the median for overscan", 

455 default=1000.0, check=lambda x: x > 0 

456 ) 

457 overscanBiasJump = pexConfig.Field( 

458 dtype=bool, 

459 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

460 default=False, 

461 ) 

462 overscanBiasJumpKeyword = pexConfig.Field( 

463 dtype=str, 

464 doc="Header keyword containing information about devices.", 

465 default="NO_SUCH_KEY", 

466 ) 

467 overscanBiasJumpDevices = pexConfig.ListField( 

468 dtype=str, 

469 doc="List of devices that need piecewise overscan correction.", 

470 default=(), 

471 ) 

472 overscanBiasJumpLocation = pexConfig.Field( 

473 dtype=int, 

474 doc="Location of bias jump along y-axis.", 

475 default=0, 

476 ) 

477 

478 # Amplifier to CCD assembly configuration 

479 doAssembleCcd = pexConfig.Field( 

480 dtype=bool, 

481 default=True, 

482 doc="Assemble amp-level exposures into a ccd-level exposure?" 

483 ) 

484 assembleCcd = pexConfig.ConfigurableField( 

485 target=AssembleCcdTask, 

486 doc="CCD assembly task", 

487 ) 

488 

489 # General calibration configuration. 

490 doAssembleIsrExposures = pexConfig.Field( 

491 dtype=bool, 

492 default=False, 

493 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

494 ) 

495 doTrimToMatchCalib = pexConfig.Field( 

496 dtype=bool, 

497 default=False, 

498 doc="Trim raw data to match calibration bounding boxes?" 

499 ) 

500 

501 # Bias subtraction. 

502 doBias = pexConfig.Field( 

503 dtype=bool, 

504 doc="Apply bias frame correction?", 

505 default=True, 

506 ) 

507 biasDataProductName = pexConfig.Field( 

508 dtype=str, 

509 doc="Name of the bias data product", 

510 default="bias", 

511 ) 

512 

513 # Variance construction 

514 doVariance = pexConfig.Field( 

515 dtype=bool, 

516 doc="Calculate variance?", 

517 default=True 

518 ) 

519 gain = pexConfig.Field( 

520 dtype=float, 

521 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

522 default=float("NaN"), 

523 ) 

524 readNoise = pexConfig.Field( 

525 dtype=float, 

526 doc="The read noise to use if no Detector is present in the Exposure", 

527 default=0.0, 

528 ) 

529 doEmpiricalReadNoise = pexConfig.Field( 

530 dtype=bool, 

531 default=False, 

532 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

533 ) 

534 

535 # Linearization. 

536 doLinearize = pexConfig.Field( 

537 dtype=bool, 

538 doc="Correct for nonlinearity of the detector's response?", 

539 default=True, 

540 ) 

541 

542 # Crosstalk. 

543 doCrosstalk = pexConfig.Field( 

544 dtype=bool, 

545 doc="Apply intra-CCD crosstalk correction?", 

546 default=False, 

547 ) 

548 doCrosstalkBeforeAssemble = pexConfig.Field( 

549 dtype=bool, 

550 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

551 default=False, 

552 ) 

553 crosstalk = pexConfig.ConfigurableField( 

554 target=CrosstalkTask, 

555 doc="Intra-CCD crosstalk correction", 

556 ) 

557 

558 # Masking options. 

559 doDefect = pexConfig.Field( 

560 dtype=bool, 

561 doc="Apply correction for CCD defects, e.g. hot pixels?", 

562 default=True, 

563 ) 

564 doNanMasking = pexConfig.Field( 

565 dtype=bool, 

566 doc="Mask NAN pixels?", 

567 default=True, 

568 ) 

569 doWidenSaturationTrails = pexConfig.Field( 

570 dtype=bool, 

571 doc="Widen bleed trails based on their width?", 

572 default=True 

573 ) 

574 

575 # Brighter-Fatter correction. 

576 doBrighterFatter = pexConfig.Field( 

577 dtype=bool, 

578 default=False, 

579 doc="Apply the brighter fatter correction" 

580 ) 

581 brighterFatterLevel = pexConfig.ChoiceField( 

582 dtype=str, 

583 default="DETECTOR", 

584 doc="The level at which to correct for brighter-fatter.", 

585 allowed={ 

586 "AMP": "Every amplifier treated separately.", 

587 "DETECTOR": "One kernel per detector", 

588 } 

589 ) 

590 brighterFatterMaxIter = pexConfig.Field( 

591 dtype=int, 

592 default=10, 

593 doc="Maximum number of iterations for the brighter fatter correction" 

594 ) 

595 brighterFatterThreshold = pexConfig.Field( 

596 dtype=float, 

597 default=1000, 

598 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

599 " absolute value of the difference between the current corrected image and the one" 

600 " from the previous iteration summed over all the pixels." 

601 ) 

602 brighterFatterApplyGain = pexConfig.Field( 

603 dtype=bool, 

604 default=True, 

605 doc="Should the gain be applied when applying the brighter fatter correction?" 

606 ) 

607 brighterFatterMaskGrowSize = pexConfig.Field( 

608 dtype=int, 

609 default=0, 

610 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

611 " when brighter-fatter correction is applied." 

612 ) 

613 

614 # Dark subtraction. 

615 doDark = pexConfig.Field( 

616 dtype=bool, 

617 doc="Apply dark frame correction?", 

618 default=True, 

619 ) 

620 darkDataProductName = pexConfig.Field( 

621 dtype=str, 

622 doc="Name of the dark data product", 

623 default="dark", 

624 ) 

625 

626 # Camera-specific stray light removal. 

627 doStrayLight = pexConfig.Field( 

628 dtype=bool, 

629 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

630 default=False, 

631 ) 

632 strayLight = pexConfig.ConfigurableField( 

633 target=StrayLightTask, 

634 doc="y-band stray light correction" 

635 ) 

636 

637 # Flat correction. 

638 doFlat = pexConfig.Field( 

639 dtype=bool, 

640 doc="Apply flat field correction?", 

641 default=True, 

642 ) 

643 flatDataProductName = pexConfig.Field( 

644 dtype=str, 

645 doc="Name of the flat data product", 

646 default="flat", 

647 ) 

648 flatScalingType = pexConfig.ChoiceField( 

649 dtype=str, 

650 doc="The method for scaling the flat on the fly.", 

651 default='USER', 

652 allowed={ 

653 "USER": "Scale by flatUserScale", 

654 "MEAN": "Scale by the inverse of the mean", 

655 "MEDIAN": "Scale by the inverse of the median", 

656 }, 

657 ) 

658 flatUserScale = pexConfig.Field( 

659 dtype=float, 

660 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

661 default=1.0, 

662 ) 

663 doTweakFlat = pexConfig.Field( 

664 dtype=bool, 

665 doc="Tweak flats to match observed amplifier ratios?", 

666 default=False 

667 ) 

668 

669 # Amplifier normalization based on gains instead of using flats configuration. 

670 doApplyGains = pexConfig.Field( 

671 dtype=bool, 

672 doc="Correct the amplifiers for their gains instead of applying flat correction", 

673 default=False, 

674 ) 

675 normalizeGains = pexConfig.Field( 

676 dtype=bool, 

677 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

678 default=False, 

679 ) 

680 

681 # Fringe correction. 

682 doFringe = pexConfig.Field( 

683 dtype=bool, 

684 doc="Apply fringe correction?", 

685 default=True, 

686 ) 

687 fringe = pexConfig.ConfigurableField( 

688 target=FringeTask, 

689 doc="Fringe subtraction task", 

690 ) 

691 fringeAfterFlat = pexConfig.Field( 

692 dtype=bool, 

693 doc="Do fringe subtraction after flat-fielding?", 

694 default=True, 

695 ) 

696 

697 # Initial CCD-level background statistics options. 

698 doMeasureBackground = pexConfig.Field( 

699 dtype=bool, 

700 doc="Measure the background level on the reduced image?", 

701 default=False, 

702 ) 

703 

704 # Camera-specific masking configuration. 

705 doCameraSpecificMasking = pexConfig.Field( 

706 dtype=bool, 

707 doc="Mask camera-specific bad regions?", 

708 default=False, 

709 ) 

710 masking = pexConfig.ConfigurableField( 

711 target=MaskingTask, 

712 doc="Masking task." 

713 ) 

714 

715 # Interpolation options. 

716 

717 doInterpolate = pexConfig.Field( 

718 dtype=bool, 

719 doc="Interpolate masked pixels?", 

720 default=True, 

721 ) 

722 doSaturationInterpolation = pexConfig.Field( 

723 dtype=bool, 

724 doc="Perform interpolation over pixels masked as saturated?" 

725 " NB: This is independent of doSaturation; if that is False this plane" 

726 " will likely be blank, resulting in a no-op here.", 

727 default=True, 

728 ) 

729 doNanInterpolation = pexConfig.Field( 

730 dtype=bool, 

731 doc="Perform interpolation over pixels masked as NaN?" 

732 " NB: This is independent of doNanMasking; if that is False this plane" 

733 " will likely be blank, resulting in a no-op here.", 

734 default=True, 

735 ) 

736 doNanInterpAfterFlat = pexConfig.Field( 

737 dtype=bool, 

738 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

739 "also have to interpolate them before flat-fielding."), 

740 default=False, 

741 ) 

742 maskListToInterpolate = pexConfig.ListField( 

743 dtype=str, 

744 doc="List of mask planes that should be interpolated.", 

745 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

746 ) 

747 doSaveInterpPixels = pexConfig.Field( 

748 dtype=bool, 

749 doc="Save a copy of the pre-interpolated pixel values?", 

750 default=False, 

751 ) 

752 

753 # Default photometric calibration options. 

754 fluxMag0T1 = pexConfig.DictField( 

755 keytype=str, 

756 itemtype=float, 

757 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

758 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

759 )) 

760 ) 

761 defaultFluxMag0T1 = pexConfig.Field( 

762 dtype=float, 

763 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

764 default=pow(10.0, 0.4*28.0) 

765 ) 

766 

767 # Vignette correction configuration. 

768 doVignette = pexConfig.Field( 

769 dtype=bool, 

770 doc="Apply vignetting parameters?", 

771 default=False, 

772 ) 

773 vignette = pexConfig.ConfigurableField( 

774 target=VignetteTask, 

775 doc="Vignetting task.", 

776 ) 

777 

778 # Transmission curve configuration. 

779 doAttachTransmissionCurve = pexConfig.Field( 

780 dtype=bool, 

781 default=False, 

782 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

783 ) 

784 doUseOpticsTransmission = pexConfig.Field( 

785 dtype=bool, 

786 default=True, 

787 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

788 ) 

789 doUseFilterTransmission = pexConfig.Field( 

790 dtype=bool, 

791 default=True, 

792 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

793 ) 

794 doUseSensorTransmission = pexConfig.Field( 

795 dtype=bool, 

796 default=True, 

797 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

798 ) 

799 doUseAtmosphereTransmission = pexConfig.Field( 

800 dtype=bool, 

801 default=True, 

802 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

803 ) 

804 

805 # Illumination correction. 

806 doIlluminationCorrection = pexConfig.Field( 

807 dtype=bool, 

808 default=False, 

809 doc="Perform illumination correction?" 

810 ) 

811 illuminationCorrectionDataProductName = pexConfig.Field( 

812 dtype=str, 

813 doc="Name of the illumination correction data product.", 

814 default="illumcor", 

815 ) 

816 illumScale = pexConfig.Field( 

817 dtype=float, 

818 doc="Scale factor for the illumination correction.", 

819 default=1.0, 

820 ) 

821 illumFilters = pexConfig.ListField( 

822 dtype=str, 

823 default=[], 

824 doc="Only perform illumination correction for these filters." 

825 ) 

826 

827 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

828 doWrite = pexConfig.Field( 

829 dtype=bool, 

830 doc="Persist postISRCCD?", 

831 default=True, 

832 ) 

833 

834 def validate(self): 

835 super().validate() 

836 if self.doFlat and self.doApplyGains: 

837 raise ValueError("You may not specify both doFlat and doApplyGains") 

838 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate: 

839 self.config.maskListToInterpolate.append("SAT") 

840 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

841 self.config.maskListToInterpolate.append("UNMASKEDNAN") 

842 

843 

844class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

845 """Apply common instrument signature correction algorithms to a raw frame. 

846 

847 The process for correcting imaging data is very similar from 

848 camera to camera. This task provides a vanilla implementation of 

849 doing these corrections, including the ability to turn certain 

850 corrections off if they are not needed. The inputs to the primary 

851 method, `run()`, are a raw exposure to be corrected and the 

852 calibration data products. The raw input is a single chip sized 

853 mosaic of all amps including overscans and other non-science 

854 pixels. The method `runDataRef()` identifies and defines the 

855 calibration data products, and is intended for use by a 

856 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

857 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

858 subclassed for different camera, although the most camera specific 

859 methods have been split into subtasks that can be redirected 

860 appropriately. 

861 

862 The __init__ method sets up the subtasks for ISR processing, using 

863 the defaults from `lsst.ip.isr`. 

864 

865 Parameters 

866 ---------- 

867 args : `list` 

868 Positional arguments passed to the Task constructor. None used at this time. 

869 kwargs : `dict`, optional 

870 Keyword arguments passed on to the Task constructor. None used at this time. 

871 """ 

872 ConfigClass = IsrTaskConfig 

873 _DefaultName = "isr" 

874 

875 def __init__(self, **kwargs): 

876 super().__init__(**kwargs) 

877 self.makeSubtask("assembleCcd") 

878 self.makeSubtask("crosstalk") 

879 self.makeSubtask("strayLight") 

880 self.makeSubtask("fringe") 

881 self.makeSubtask("masking") 

882 self.makeSubtask("overscan") 

883 self.makeSubtask("vignette") 

884 

885 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

886 inputs = butlerQC.get(inputRefs) 

887 

888 try: 

889 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

890 except Exception as e: 

891 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

892 (inputRefs, e)) 

893 

894 inputs['isGen3'] = True 

895 

896 detector = inputs['ccdExposure'].getDetector() 

897 

898 if self.config.doCrosstalk is True: 

899 # Crosstalk sources need to be defined by the pipeline 

900 # yaml if they exist. 

901 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

902 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

903 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

904 else: 

905 coeffVector = (self.config.crosstalk.crosstalkValues 

906 if self.config.crosstalk.useConfigCoefficients else None) 

907 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

908 inputs['crosstalk'] = crosstalkCalib 

909 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

910 if 'crosstalkSources' not in inputs: 

911 self.log.warn("No crosstalkSources found for chip with interChip terms!") 

912 

913 if self.doLinearize(detector) is True: 

914 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict): 

915 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

916 linearizer.fromYaml(inputs['linearizer']) 

917 else: 

918 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector, 

919 log=self.log) 

920 inputs['linearizer'] = linearizer 

921 

922 if self.config.doDefect is True: 

923 if "defects" in inputs and inputs['defects'] is not None: 

924 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

925 # masking expects a list of defects defined by their bounding box 

926 if not isinstance(inputs["defects"], Defects): 

927 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

928 

929 # Load the correct style of brighter fatter kernel, and repack 

930 # the information as a numpy array. 

931 if self.config.doBrighterFatter: 

932 brighterFatterKernel = inputs.pop('newBFKernel', None) 

933 if brighterFatterKernel is None: 

934 brighterFatterKernel = inputs.get('bfKernel', None) 

935 

936 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

937 detId = detector.getId() 

938 inputs['bfGains'] = brighterFatterKernel.gain 

939 # If the kernel is not an ndarray, it's the cp_pipe version 

940 # so extract the kernel for this detector, or raise an error 

941 if self.config.brighterFatterLevel == 'DETECTOR': 

942 if brighterFatterKernel.detectorKernel: 

943 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

944 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

945 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

946 else: 

947 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

948 else: 

949 # TODO DM-15631 for implementing this 

950 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

951 

952 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

953 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

954 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

955 expId=expId, 

956 assembler=self.assembleCcd 

957 if self.config.doAssembleIsrExposures else None) 

958 else: 

959 inputs['fringes'] = pipeBase.Struct(fringes=None) 

960 

961 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

962 if 'strayLightData' not in inputs: 

963 inputs['strayLightData'] = None 

964 

965 outputs = self.run(**inputs) 

966 butlerQC.put(outputs, outputRefs) 

967 

968 def readIsrData(self, dataRef, rawExposure): 

969 """!Retrieve necessary frames for instrument signature removal. 

970 

971 Pre-fetching all required ISR data products limits the IO 

972 required by the ISR. Any conflict between the calibration data 

973 available and that needed for ISR is also detected prior to 

974 doing processing, allowing it to fail quickly. 

975 

976 Parameters 

977 ---------- 

978 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

979 Butler reference of the detector data to be processed 

980 rawExposure : `afw.image.Exposure` 

981 The raw exposure that will later be corrected with the 

982 retrieved calibration data; should not be modified in this 

983 method. 

984 

985 Returns 

986 ------- 

987 result : `lsst.pipe.base.Struct` 

988 Result struct with components (which may be `None`): 

989 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

990 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

991 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

992 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

993 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

994 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

995 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 

996 - ``fringes``: `lsst.pipe.base.Struct` with components: 

997 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

998 - ``seed``: random seed derived from the ccdExposureId for random 

999 number generator (`uint32`). 

1000 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1001 A ``TransmissionCurve`` that represents the throughput of the optics, 

1002 to be evaluated in focal-plane coordinates. 

1003 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1004 A ``TransmissionCurve`` that represents the throughput of the filter 

1005 itself, to be evaluated in focal-plane coordinates. 

1006 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1007 A ``TransmissionCurve`` that represents the throughput of the sensor 

1008 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1009 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1010 A ``TransmissionCurve`` that represents the throughput of the 

1011 atmosphere, assumed to be spatially constant. 

1012 - ``strayLightData`` : `object` 

1013 An opaque object containing calibration information for 

1014 stray-light correction. If `None`, no correction will be 

1015 performed. 

1016 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1017 

1018 Raises 

1019 ------ 

1020 NotImplementedError : 

1021 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1022 """ 

1023 try: 

1024 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1025 dateObs = dateObs.toPython().isoformat() 

1026 except RuntimeError: 

1027 self.log.warn("Unable to identify dateObs for rawExposure.") 

1028 dateObs = None 

1029 

1030 ccd = rawExposure.getDetector() 

1031 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

1032 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1033 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1034 if self.config.doBias else None) 

1035 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1036 linearizer = (dataRef.get("linearizer", immediate=True) 

1037 if self.doLinearize(ccd) else None) 

1038 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1039 linearizer.log = self.log 

1040 if isinstance(linearizer, numpy.ndarray): 

1041 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1042 

1043 crosstalkCalib = None 

1044 if self.config.doCrosstalk: 

1045 try: 

1046 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1047 except NoResults: 

1048 coeffVector = (self.config.crosstalk.crosstalkValues 

1049 if self.config.crosstalk.useConfigCoefficients else None) 

1050 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1051 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1052 if self.config.doCrosstalk else None) 

1053 

1054 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1055 if self.config.doDark else None) 

1056 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1057 dateObs=dateObs) 

1058 if self.config.doFlat else None) 

1059 

1060 brighterFatterKernel = None 

1061 brighterFatterGains = None 

1062 if self.config.doBrighterFatter is True: 

1063 try: 

1064 # Use the new-style cp_pipe version of the kernel if it exists 

1065 # If using a new-style kernel, always use the self-consistent 

1066 # gains, i.e. the ones inside the kernel object itself 

1067 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1068 brighterFatterGains = brighterFatterKernel.gain 

1069 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

1070 except NoResults: 

1071 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1072 brighterFatterKernel = dataRef.get("bfKernel") 

1073 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

1074 except NoResults: 

1075 brighterFatterKernel = None 

1076 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1077 # If the kernel is not an ndarray, it's the cp_pipe version 

1078 # so extract the kernel for this detector, or raise an error 

1079 if self.config.brighterFatterLevel == 'DETECTOR': 

1080 if brighterFatterKernel.detectorKernel: 

1081 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

1082 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

1083 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

1084 else: 

1085 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1086 else: 

1087 # TODO DM-15631 for implementing this 

1088 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1089 

1090 defectList = (dataRef.get("defects") 

1091 if self.config.doDefect else None) 

1092 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

1093 if self.config.doAssembleIsrExposures else None) 

1094 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1095 else pipeBase.Struct(fringes=None)) 

1096 

1097 if self.config.doAttachTransmissionCurve: 

1098 opticsTransmission = (dataRef.get("transmission_optics") 

1099 if self.config.doUseOpticsTransmission else None) 

1100 filterTransmission = (dataRef.get("transmission_filter") 

1101 if self.config.doUseFilterTransmission else None) 

1102 sensorTransmission = (dataRef.get("transmission_sensor") 

1103 if self.config.doUseSensorTransmission else None) 

1104 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1105 if self.config.doUseAtmosphereTransmission else None) 

1106 else: 

1107 opticsTransmission = None 

1108 filterTransmission = None 

1109 sensorTransmission = None 

1110 atmosphereTransmission = None 

1111 

1112 if self.config.doStrayLight: 

1113 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1114 else: 

1115 strayLightData = None 

1116 

1117 illumMaskedImage = (self.getIsrExposure(dataRef, 

1118 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1119 if (self.config.doIlluminationCorrection 

1120 and filterName in self.config.illumFilters) 

1121 else None) 

1122 

1123 # Struct should include only kwargs to run() 

1124 return pipeBase.Struct(bias=biasExposure, 

1125 linearizer=linearizer, 

1126 crosstalk=crosstalkCalib, 

1127 crosstalkSources=crosstalkSources, 

1128 dark=darkExposure, 

1129 flat=flatExposure, 

1130 bfKernel=brighterFatterKernel, 

1131 bfGains=brighterFatterGains, 

1132 defects=defectList, 

1133 fringes=fringeStruct, 

1134 opticsTransmission=opticsTransmission, 

1135 filterTransmission=filterTransmission, 

1136 sensorTransmission=sensorTransmission, 

1137 atmosphereTransmission=atmosphereTransmission, 

1138 strayLightData=strayLightData, 

1139 illumMaskedImage=illumMaskedImage 

1140 ) 

1141 

1142 @pipeBase.timeMethod 

1143 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, 

1144 crosstalk=None, crosstalkSources=None, 

1145 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1146 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1147 sensorTransmission=None, atmosphereTransmission=None, 

1148 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1149 isGen3=False, 

1150 ): 

1151 """!Perform instrument signature removal on an exposure. 

1152 

1153 Steps included in the ISR processing, in order performed, are: 

1154 - saturation and suspect pixel masking 

1155 - overscan subtraction 

1156 - CCD assembly of individual amplifiers 

1157 - bias subtraction 

1158 - variance image construction 

1159 - linearization of non-linear response 

1160 - crosstalk masking 

1161 - brighter-fatter correction 

1162 - dark subtraction 

1163 - fringe correction 

1164 - stray light subtraction 

1165 - flat correction 

1166 - masking of known defects and camera specific features 

1167 - vignette calculation 

1168 - appending transmission curve and distortion model 

1169 

1170 Parameters 

1171 ---------- 

1172 ccdExposure : `lsst.afw.image.Exposure` 

1173 The raw exposure that is to be run through ISR. The 

1174 exposure is modified by this method. 

1175 camera : `lsst.afw.cameraGeom.Camera`, optional 

1176 The camera geometry for this exposure. Required if ``isGen3`` is 

1177 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1178 ``flat`` does not have an associated detector. 

1179 bias : `lsst.afw.image.Exposure`, optional 

1180 Bias calibration frame. 

1181 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1182 Functor for linearization. 

1183 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1184 Calibration for crosstalk. 

1185 crosstalkSources : `list`, optional 

1186 List of possible crosstalk sources. 

1187 dark : `lsst.afw.image.Exposure`, optional 

1188 Dark calibration frame. 

1189 flat : `lsst.afw.image.Exposure`, optional 

1190 Flat calibration frame. 

1191 bfKernel : `numpy.ndarray`, optional 

1192 Brighter-fatter kernel. 

1193 bfGains : `dict` of `float`, optional 

1194 Gains used to override the detector's nominal gains for the 

1195 brighter-fatter correction. A dict keyed by amplifier name for 

1196 the detector in question. 

1197 defects : `lsst.meas.algorithms.Defects`, optional 

1198 List of defects. 

1199 fringes : `lsst.pipe.base.Struct`, optional 

1200 Struct containing the fringe correction data, with 

1201 elements: 

1202 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1203 - ``seed``: random seed derived from the ccdExposureId for random 

1204 number generator (`uint32`) 

1205 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1206 A ``TransmissionCurve`` that represents the throughput of the optics, 

1207 to be evaluated in focal-plane coordinates. 

1208 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1209 A ``TransmissionCurve`` that represents the throughput of the filter 

1210 itself, to be evaluated in focal-plane coordinates. 

1211 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1212 A ``TransmissionCurve`` that represents the throughput of the sensor 

1213 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1214 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1215 A ``TransmissionCurve`` that represents the throughput of the 

1216 atmosphere, assumed to be spatially constant. 

1217 detectorNum : `int`, optional 

1218 The integer number for the detector to process. 

1219 isGen3 : bool, optional 

1220 Flag this call to run() as using the Gen3 butler environment. 

1221 strayLightData : `object`, optional 

1222 Opaque object containing calibration information for stray-light 

1223 correction. If `None`, no correction will be performed. 

1224 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1225 Illumination correction image. 

1226 

1227 Returns 

1228 ------- 

1229 result : `lsst.pipe.base.Struct` 

1230 Result struct with component: 

1231 - ``exposure`` : `afw.image.Exposure` 

1232 The fully ISR corrected exposure. 

1233 - ``outputExposure`` : `afw.image.Exposure` 

1234 An alias for `exposure` 

1235 - ``ossThumb`` : `numpy.ndarray` 

1236 Thumbnail image of the exposure after overscan subtraction. 

1237 - ``flattenedThumb`` : `numpy.ndarray` 

1238 Thumbnail image of the exposure after flat-field correction. 

1239 

1240 Raises 

1241 ------ 

1242 RuntimeError 

1243 Raised if a configuration option is set to True, but the 

1244 required calibration data has not been specified. 

1245 

1246 Notes 

1247 ----- 

1248 The current processed exposure can be viewed by setting the 

1249 appropriate lsstDebug entries in the `debug.display` 

1250 dictionary. The names of these entries correspond to some of 

1251 the IsrTaskConfig Boolean options, with the value denoting the 

1252 frame to use. The exposure is shown inside the matching 

1253 option check and after the processing of that step has 

1254 finished. The steps with debug points are: 

1255 

1256 doAssembleCcd 

1257 doBias 

1258 doCrosstalk 

1259 doBrighterFatter 

1260 doDark 

1261 doFringe 

1262 doStrayLight 

1263 doFlat 

1264 

1265 In addition, setting the "postISRCCD" entry displays the 

1266 exposure after all ISR processing has finished. 

1267 

1268 """ 

1269 

1270 if isGen3 is True: 

1271 # Gen3 currently cannot automatically do configuration overrides. 

1272 # DM-15257 looks to discuss this issue. 

1273 # Configure input exposures; 

1274 if detectorNum is None: 

1275 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1276 

1277 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1278 bias = self.ensureExposure(bias, camera, detectorNum) 

1279 dark = self.ensureExposure(dark, camera, detectorNum) 

1280 flat = self.ensureExposure(flat, camera, detectorNum) 

1281 else: 

1282 if isinstance(ccdExposure, ButlerDataRef): 

1283 return self.runDataRef(ccdExposure) 

1284 

1285 ccd = ccdExposure.getDetector() 

1286 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1287 

1288 if not ccd: 

1289 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1290 ccd = [FakeAmp(ccdExposure, self.config)] 

1291 

1292 # Validate Input 

1293 if self.config.doBias and bias is None: 

1294 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1295 if self.doLinearize(ccd) and linearizer is None: 

1296 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1297 if self.config.doBrighterFatter and bfKernel is None: 

1298 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1299 if self.config.doDark and dark is None: 

1300 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1301 if self.config.doFlat and flat is None: 

1302 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1303 if self.config.doDefect and defects is None: 

1304 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1305 if (self.config.doFringe and filterName in self.fringe.config.filters 

1306 and fringes.fringes is None): 

1307 # The `fringes` object needs to be a pipeBase.Struct, as 

1308 # we use it as a `dict` for the parameters of 

1309 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1310 # not be `None` if `doFringe=True`. Otherwise, raise. 

1311 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1312 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters 

1313 and illumMaskedImage is None): 

1314 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1315 

1316 # Begin ISR processing. 

1317 if self.config.doConvertIntToFloat: 

1318 self.log.info("Converting exposure to floating point values.") 

1319 ccdExposure = self.convertIntToFloat(ccdExposure) 

1320 

1321 # Amplifier level processing. 

1322 overscans = [] 

1323 for amp in ccd: 

1324 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1325 if ccdExposure.getBBox().contains(amp.getBBox()): 

1326 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1327 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1328 

1329 if self.config.doOverscan and not badAmp: 

1330 # Overscan correction on amp-by-amp basis. 

1331 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1332 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1333 if overscanResults is not None and \ 

1334 self.config.qa is not None and self.config.qa.saveStats is True: 

1335 if isinstance(overscanResults.overscanFit, float): 

1336 qaMedian = overscanResults.overscanFit 

1337 qaStdev = float("NaN") 

1338 else: 

1339 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1340 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1341 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1342 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1343 

1344 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian) 

1345 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev) 

1346 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1347 amp.getName(), qaMedian, qaStdev) 

1348 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1349 else: 

1350 if badAmp: 

1351 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1352 overscanResults = None 

1353 

1354 overscans.append(overscanResults if overscanResults is not None else None) 

1355 else: 

1356 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1357 

1358 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1359 self.log.info("Applying crosstalk correction.") 

1360 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1361 crosstalkSources=crosstalkSources) 

1362 self.debugView(ccdExposure, "doCrosstalk") 

1363 

1364 if self.config.doAssembleCcd: 

1365 self.log.info("Assembling CCD from amplifiers.") 

1366 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1367 

1368 if self.config.expectWcs and not ccdExposure.getWcs(): 

1369 self.log.warn("No WCS found in input exposure.") 

1370 self.debugView(ccdExposure, "doAssembleCcd") 

1371 

1372 ossThumb = None 

1373 if self.config.qa.doThumbnailOss: 

1374 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1375 

1376 if self.config.doBias: 

1377 self.log.info("Applying bias correction.") 

1378 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1379 trimToFit=self.config.doTrimToMatchCalib) 

1380 self.debugView(ccdExposure, "doBias") 

1381 

1382 if self.config.doVariance: 

1383 for amp, overscanResults in zip(ccd, overscans): 

1384 if ccdExposure.getBBox().contains(amp.getBBox()): 

1385 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1386 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1387 if overscanResults is not None: 

1388 self.updateVariance(ampExposure, amp, 

1389 overscanImage=overscanResults.overscanImage) 

1390 else: 

1391 self.updateVariance(ampExposure, amp, 

1392 overscanImage=None) 

1393 if self.config.qa is not None and self.config.qa.saveStats is True: 

1394 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1395 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1396 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1397 qaStats.getValue(afwMath.MEDIAN)) 

1398 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1399 qaStats.getValue(afwMath.STDEVCLIP)) 

1400 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1401 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1402 qaStats.getValue(afwMath.STDEVCLIP)) 

1403 

1404 if self.doLinearize(ccd): 

1405 self.log.info("Applying linearizer.") 

1406 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1407 detector=ccd, log=self.log) 

1408 

1409 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1410 self.log.info("Applying crosstalk correction.") 

1411 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1412 crosstalkSources=crosstalkSources, isTrimmed=True) 

1413 self.debugView(ccdExposure, "doCrosstalk") 

1414 

1415 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1416 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1417 if self.config.doDefect: 

1418 self.log.info("Masking defects.") 

1419 self.maskDefect(ccdExposure, defects) 

1420 

1421 if self.config.numEdgeSuspect > 0: 

1422 self.log.info("Masking edges as SUSPECT.") 

1423 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1424 maskPlane="SUSPECT") 

1425 

1426 if self.config.doNanMasking: 

1427 self.log.info("Masking NAN value pixels.") 

1428 self.maskNan(ccdExposure) 

1429 

1430 if self.config.doWidenSaturationTrails: 

1431 self.log.info("Widening saturation trails.") 

1432 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1433 

1434 if self.config.doCameraSpecificMasking: 

1435 self.log.info("Masking regions for camera specific reasons.") 

1436 self.masking.run(ccdExposure) 

1437 

1438 if self.config.doBrighterFatter: 

1439 # We need to apply flats and darks before we can interpolate, and we 

1440 # need to interpolate before we do B-F, but we do B-F without the 

1441 # flats and darks applied so we can work in units of electrons or holes. 

1442 # This context manager applies and then removes the darks and flats. 

1443 # 

1444 # We also do not want to interpolate values here, so operate on temporary 

1445 # images so we can apply only the BF-correction and roll back the 

1446 # interpolation. 

1447 interpExp = ccdExposure.clone() 

1448 with self.flatContext(interpExp, flat, dark): 

1449 isrFunctions.interpolateFromMask( 

1450 maskedImage=interpExp.getMaskedImage(), 

1451 fwhm=self.config.fwhm, 

1452 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1453 maskNameList=self.config.maskListToInterpolate 

1454 ) 

1455 bfExp = interpExp.clone() 

1456 

1457 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1458 type(bfKernel), type(bfGains)) 

1459 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1460 self.config.brighterFatterMaxIter, 

1461 self.config.brighterFatterThreshold, 

1462 self.config.brighterFatterApplyGain, 

1463 bfGains) 

1464 if bfResults[1] == self.config.brighterFatterMaxIter: 

1465 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1466 bfResults[0]) 

1467 else: 

1468 self.log.info("Finished brighter fatter correction in %d iterations.", 

1469 bfResults[1]) 

1470 image = ccdExposure.getMaskedImage().getImage() 

1471 bfCorr = bfExp.getMaskedImage().getImage() 

1472 bfCorr -= interpExp.getMaskedImage().getImage() 

1473 image += bfCorr 

1474 

1475 # Applying the brighter-fatter correction applies a 

1476 # convolution to the science image. At the edges this 

1477 # convolution may not have sufficient valid pixels to 

1478 # produce a valid correction. Mark pixels within the size 

1479 # of the brighter-fatter kernel as EDGE to warn of this 

1480 # fact. 

1481 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1482 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1483 maskPlane="EDGE") 

1484 

1485 if self.config.brighterFatterMaskGrowSize > 0: 

1486 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1487 for maskPlane in self.config.maskListToInterpolate: 

1488 isrFunctions.growMasks(ccdExposure.getMask(), 

1489 radius=self.config.brighterFatterMaskGrowSize, 

1490 maskNameList=maskPlane, 

1491 maskValue=maskPlane) 

1492 

1493 self.debugView(ccdExposure, "doBrighterFatter") 

1494 

1495 if self.config.doDark: 

1496 self.log.info("Applying dark correction.") 

1497 self.darkCorrection(ccdExposure, dark) 

1498 self.debugView(ccdExposure, "doDark") 

1499 

1500 if self.config.doFringe and not self.config.fringeAfterFlat: 

1501 self.log.info("Applying fringe correction before flat.") 

1502 self.fringe.run(ccdExposure, **fringes.getDict()) 

1503 self.debugView(ccdExposure, "doFringe") 

1504 

1505 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1506 self.log.info("Checking strayLight correction.") 

1507 self.strayLight.run(ccdExposure, strayLightData) 

1508 self.debugView(ccdExposure, "doStrayLight") 

1509 

1510 if self.config.doFlat: 

1511 self.log.info("Applying flat correction.") 

1512 self.flatCorrection(ccdExposure, flat) 

1513 self.debugView(ccdExposure, "doFlat") 

1514 

1515 if self.config.doApplyGains: 

1516 self.log.info("Applying gain correction instead of flat.") 

1517 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1518 

1519 if self.config.doFringe and self.config.fringeAfterFlat: 

1520 self.log.info("Applying fringe correction after flat.") 

1521 self.fringe.run(ccdExposure, **fringes.getDict()) 

1522 

1523 if self.config.doVignette: 

1524 self.log.info("Constructing Vignette polygon.") 

1525 self.vignettePolygon = self.vignette.run(ccdExposure) 

1526 

1527 if self.config.vignette.doWriteVignettePolygon: 

1528 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1529 

1530 if self.config.doAttachTransmissionCurve: 

1531 self.log.info("Adding transmission curves.") 

1532 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1533 filterTransmission=filterTransmission, 

1534 sensorTransmission=sensorTransmission, 

1535 atmosphereTransmission=atmosphereTransmission) 

1536 

1537 flattenedThumb = None 

1538 if self.config.qa.doThumbnailFlattened: 

1539 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1540 

1541 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1542 self.log.info("Performing illumination correction.") 

1543 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1544 illumMaskedImage, illumScale=self.config.illumScale, 

1545 trimToFit=self.config.doTrimToMatchCalib) 

1546 

1547 preInterpExp = None 

1548 if self.config.doSaveInterpPixels: 

1549 preInterpExp = ccdExposure.clone() 

1550 

1551 # Reset and interpolate bad pixels. 

1552 # 

1553 # Large contiguous bad regions (which should have the BAD mask 

1554 # bit set) should have their values set to the image median. 

1555 # This group should include defects and bad amplifiers. As the 

1556 # area covered by these defects are large, there's little 

1557 # reason to expect that interpolation would provide a more 

1558 # useful value. 

1559 # 

1560 # Smaller defects can be safely interpolated after the larger 

1561 # regions have had their pixel values reset. This ensures 

1562 # that the remaining defects adjacent to bad amplifiers (as an 

1563 # example) do not attempt to interpolate extreme values. 

1564 if self.config.doSetBadRegions: 

1565 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1566 if badPixelCount > 0: 

1567 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1568 

1569 if self.config.doInterpolate: 

1570 self.log.info("Interpolating masked pixels.") 

1571 isrFunctions.interpolateFromMask( 

1572 maskedImage=ccdExposure.getMaskedImage(), 

1573 fwhm=self.config.fwhm, 

1574 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1575 maskNameList=list(self.config.maskListToInterpolate) 

1576 ) 

1577 

1578 self.roughZeroPoint(ccdExposure) 

1579 

1580 if self.config.doMeasureBackground: 

1581 self.log.info("Measuring background level.") 

1582 self.measureBackground(ccdExposure, self.config.qa) 

1583 

1584 if self.config.qa is not None and self.config.qa.saveStats is True: 

1585 for amp in ccd: 

1586 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1587 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1588 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1589 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1590 qaStats.getValue(afwMath.MEDIAN)) 

1591 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1592 qaStats.getValue(afwMath.STDEVCLIP)) 

1593 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1594 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1595 qaStats.getValue(afwMath.STDEVCLIP)) 

1596 

1597 self.debugView(ccdExposure, "postISRCCD") 

1598 

1599 return pipeBase.Struct( 

1600 exposure=ccdExposure, 

1601 ossThumb=ossThumb, 

1602 flattenedThumb=flattenedThumb, 

1603 

1604 preInterpolatedExposure=preInterpExp, 

1605 outputExposure=ccdExposure, 

1606 outputOssThumbnail=ossThumb, 

1607 outputFlattenedThumbnail=flattenedThumb, 

1608 ) 

1609 

1610 @pipeBase.timeMethod 

1611 def runDataRef(self, sensorRef): 

1612 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1613 

1614 This method contains the `CmdLineTask` interface to the ISR 

1615 processing. All IO is handled here, freeing the `run()` method 

1616 to manage only pixel-level calculations. The steps performed 

1617 are: 

1618 - Read in necessary detrending/isr/calibration data. 

1619 - Process raw exposure in `run()`. 

1620 - Persist the ISR-corrected exposure as "postISRCCD" if 

1621 config.doWrite=True. 

1622 

1623 Parameters 

1624 ---------- 

1625 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1626 DataRef of the detector data to be processed 

1627 

1628 Returns 

1629 ------- 

1630 result : `lsst.pipe.base.Struct` 

1631 Result struct with component: 

1632 - ``exposure`` : `afw.image.Exposure` 

1633 The fully ISR corrected exposure. 

1634 

1635 Raises 

1636 ------ 

1637 RuntimeError 

1638 Raised if a configuration option is set to True, but the 

1639 required calibration data does not exist. 

1640 

1641 """ 

1642 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1643 

1644 ccdExposure = sensorRef.get(self.config.datasetType) 

1645 

1646 camera = sensorRef.get("camera") 

1647 isrData = self.readIsrData(sensorRef, ccdExposure) 

1648 

1649 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1650 

1651 if self.config.doWrite: 

1652 sensorRef.put(result.exposure, "postISRCCD") 

1653 if result.preInterpolatedExposure is not None: 

1654 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1655 if result.ossThumb is not None: 

1656 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1657 if result.flattenedThumb is not None: 

1658 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1659 

1660 return result 

1661 

1662 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1663 """!Retrieve a calibration dataset for removing instrument signature. 

1664 

1665 Parameters 

1666 ---------- 

1667 

1668 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1669 DataRef of the detector data to find calibration datasets 

1670 for. 

1671 datasetType : `str` 

1672 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1673 dateObs : `str`, optional 

1674 Date of the observation. Used to correct butler failures 

1675 when using fallback filters. 

1676 immediate : `Bool` 

1677 If True, disable butler proxies to enable error handling 

1678 within this routine. 

1679 

1680 Returns 

1681 ------- 

1682 exposure : `lsst.afw.image.Exposure` 

1683 Requested calibration frame. 

1684 

1685 Raises 

1686 ------ 

1687 RuntimeError 

1688 Raised if no matching calibration frame can be found. 

1689 """ 

1690 try: 

1691 exp = dataRef.get(datasetType, immediate=immediate) 

1692 except Exception as exc1: 

1693 if not self.config.fallbackFilterName: 

1694 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1695 try: 

1696 if self.config.useFallbackDate and dateObs: 

1697 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1698 dateObs=dateObs, immediate=immediate) 

1699 else: 

1700 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1701 except Exception as exc2: 

1702 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1703 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1704 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1705 

1706 if self.config.doAssembleIsrExposures: 

1707 exp = self.assembleCcd.assembleCcd(exp) 

1708 return exp 

1709 

1710 def ensureExposure(self, inputExp, camera, detectorNum): 

1711 """Ensure that the data returned by Butler is a fully constructed exposure. 

1712 

1713 ISR requires exposure-level image data for historical reasons, so if we did 

1714 not recieve that from Butler, construct it from what we have, modifying the 

1715 input in place. 

1716 

1717 Parameters 

1718 ---------- 

1719 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1720 `lsst.afw.image.ImageF` 

1721 The input data structure obtained from Butler. 

1722 camera : `lsst.afw.cameraGeom.camera` 

1723 The camera associated with the image. Used to find the appropriate 

1724 detector. 

1725 detectorNum : `int` 

1726 The detector this exposure should match. 

1727 

1728 Returns 

1729 ------- 

1730 inputExp : `lsst.afw.image.Exposure` 

1731 The re-constructed exposure, with appropriate detector parameters. 

1732 

1733 Raises 

1734 ------ 

1735 TypeError 

1736 Raised if the input data cannot be used to construct an exposure. 

1737 """ 

1738 if isinstance(inputExp, afwImage.DecoratedImageU): 

1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1740 elif isinstance(inputExp, afwImage.ImageF): 

1741 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1742 elif isinstance(inputExp, afwImage.MaskedImageF): 

1743 inputExp = afwImage.makeExposure(inputExp) 

1744 elif isinstance(inputExp, afwImage.Exposure): 

1745 pass 

1746 elif inputExp is None: 

1747 # Assume this will be caught by the setup if it is a problem. 

1748 return inputExp 

1749 else: 

1750 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1751 (type(inputExp), )) 

1752 

1753 if inputExp.getDetector() is None: 

1754 inputExp.setDetector(camera[detectorNum]) 

1755 

1756 return inputExp 

1757 

1758 def convertIntToFloat(self, exposure): 

1759 """Convert exposure image from uint16 to float. 

1760 

1761 If the exposure does not need to be converted, the input is 

1762 immediately returned. For exposures that are converted to use 

1763 floating point pixels, the variance is set to unity and the 

1764 mask to zero. 

1765 

1766 Parameters 

1767 ---------- 

1768 exposure : `lsst.afw.image.Exposure` 

1769 The raw exposure to be converted. 

1770 

1771 Returns 

1772 ------- 

1773 newexposure : `lsst.afw.image.Exposure` 

1774 The input ``exposure``, converted to floating point pixels. 

1775 

1776 Raises 

1777 ------ 

1778 RuntimeError 

1779 Raised if the exposure type cannot be converted to float. 

1780 

1781 """ 

1782 if isinstance(exposure, afwImage.ExposureF): 

1783 # Nothing to be done 

1784 self.log.debug("Exposure already of type float.") 

1785 return exposure 

1786 if not hasattr(exposure, "convertF"): 

1787 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1788 

1789 newexposure = exposure.convertF() 

1790 newexposure.variance[:] = 1 

1791 newexposure.mask[:] = 0x0 

1792 

1793 return newexposure 

1794 

1795 def maskAmplifier(self, ccdExposure, amp, defects): 

1796 """Identify bad amplifiers, saturated and suspect pixels. 

1797 

1798 Parameters 

1799 ---------- 

1800 ccdExposure : `lsst.afw.image.Exposure` 

1801 Input exposure to be masked. 

1802 amp : `lsst.afw.table.AmpInfoCatalog` 

1803 Catalog of parameters defining the amplifier on this 

1804 exposure to mask. 

1805 defects : `lsst.meas.algorithms.Defects` 

1806 List of defects. Used to determine if the entire 

1807 amplifier is bad. 

1808 

1809 Returns 

1810 ------- 

1811 badAmp : `Bool` 

1812 If this is true, the entire amplifier area is covered by 

1813 defects and unusable. 

1814 

1815 """ 

1816 maskedImage = ccdExposure.getMaskedImage() 

1817 

1818 badAmp = False 

1819 

1820 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1821 # comparison with current defects definition. 

1822 if defects is not None: 

1823 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1824 

1825 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1826 # association with pixels in current ccdExposure). 

1827 if badAmp: 

1828 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1829 afwImage.PARENT) 

1830 maskView = dataView.getMask() 

1831 maskView |= maskView.getPlaneBitMask("BAD") 

1832 del maskView 

1833 return badAmp 

1834 

1835 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1836 # Saturation and suspect pixels can be masked now, though. 

1837 limits = dict() 

1838 if self.config.doSaturation and not badAmp: 

1839 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1840 if self.config.doSuspect and not badAmp: 

1841 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1842 if math.isfinite(self.config.saturation): 

1843 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1844 

1845 for maskName, maskThreshold in limits.items(): 

1846 if not math.isnan(maskThreshold): 

1847 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1848 isrFunctions.makeThresholdMask( 

1849 maskedImage=dataView, 

1850 threshold=maskThreshold, 

1851 growFootprints=0, 

1852 maskName=maskName 

1853 ) 

1854 

1855 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1856 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1857 afwImage.PARENT) 

1858 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1859 self.config.suspectMaskName]) 

1860 if numpy.all(maskView.getArray() & maskVal > 0): 

1861 badAmp = True 

1862 maskView |= maskView.getPlaneBitMask("BAD") 

1863 

1864 return badAmp 

1865 

1866 def overscanCorrection(self, ccdExposure, amp): 

1867 """Apply overscan correction in place. 

1868 

1869 This method does initial pixel rejection of the overscan 

1870 region. The overscan can also be optionally segmented to 

1871 allow for discontinuous overscan responses to be fit 

1872 separately. The actual overscan subtraction is performed by 

1873 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1874 which is called here after the amplifier is preprocessed. 

1875 

1876 Parameters 

1877 ---------- 

1878 ccdExposure : `lsst.afw.image.Exposure` 

1879 Exposure to have overscan correction performed. 

1880 amp : `lsst.afw.table.AmpInfoCatalog` 

1881 The amplifier to consider while correcting the overscan. 

1882 

1883 Returns 

1884 ------- 

1885 overscanResults : `lsst.pipe.base.Struct` 

1886 Result struct with components: 

1887 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1888 Value or fit subtracted from the amplifier image data. 

1889 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1890 Value or fit subtracted from the overscan image data. 

1891 - ``overscanImage`` : `lsst.afw.image.Image` 

1892 Image of the overscan region with the overscan 

1893 correction applied. This quantity is used to estimate 

1894 the amplifier read noise empirically. 

1895 

1896 Raises 

1897 ------ 

1898 RuntimeError 

1899 Raised if the ``amp`` does not contain raw pixel information. 

1900 

1901 See Also 

1902 -------- 

1903 lsst.ip.isr.isrFunctions.overscanCorrection 

1904 """ 

1905 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1906 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1907 return None 

1908 

1909 statControl = afwMath.StatisticsControl() 

1910 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1911 

1912 # Determine the bounding boxes 

1913 dataBBox = amp.getRawDataBBox() 

1914 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1915 dx0 = 0 

1916 dx1 = 0 

1917 

1918 prescanBBox = amp.getRawPrescanBBox() 

1919 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1920 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1921 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1922 else: 

1923 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1924 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1925 

1926 # Determine if we need to work on subregions of the amplifier and overscan. 

1927 imageBBoxes = [] 

1928 overscanBBoxes = [] 

1929 

1930 if ((self.config.overscanBiasJump 

1931 and self.config.overscanBiasJumpLocation) 

1932 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

1933 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1934 self.config.overscanBiasJumpDevices)): 

1935 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1936 yLower = self.config.overscanBiasJumpLocation 

1937 yUpper = dataBBox.getHeight() - yLower 

1938 else: 

1939 yUpper = self.config.overscanBiasJumpLocation 

1940 yLower = dataBBox.getHeight() - yUpper 

1941 

1942 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1943 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1944 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1945 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1946 yLower))) 

1947 

1948 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1949 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1950 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1951 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1952 yUpper))) 

1953 else: 

1954 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1955 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1956 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1957 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1958 oscanBBox.getHeight()))) 

1959 

1960 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

1961 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1962 ampImage = ccdExposure.maskedImage[imageBBox] 

1963 overscanImage = ccdExposure.maskedImage[overscanBBox] 

1964 

1965 overscanArray = overscanImage.image.array 

1966 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

1967 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

1968 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

1969 

1970 statControl = afwMath.StatisticsControl() 

1971 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1972 

1973 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage) 

1974 

1975 # Measure average overscan levels and record them in the metadata. 

1976 levelStat = afwMath.MEDIAN 

1977 sigmaStat = afwMath.STDEVCLIP 

1978 

1979 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

1980 self.config.qa.flatness.nIter) 

1981 metadata = ccdExposure.getMetadata() 

1982 ampNum = amp.getName() 

1983 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

1984 if isinstance(overscanResults.overscanFit, float): 

1985 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

1986 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

1987 else: 

1988 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

1989 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

1990 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

1991 

1992 return overscanResults 

1993 

1994 def updateVariance(self, ampExposure, amp, overscanImage=None): 

1995 """Set the variance plane using the amplifier gain and read noise 

1996 

1997 The read noise is calculated from the ``overscanImage`` if the 

1998 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1999 the value from the amplifier data is used. 

2000 

2001 Parameters 

2002 ---------- 

2003 ampExposure : `lsst.afw.image.Exposure` 

2004 Exposure to process. 

2005 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2006 Amplifier detector data. 

2007 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2008 Image of overscan, required only for empirical read noise. 

2009 

2010 See also 

2011 -------- 

2012 lsst.ip.isr.isrFunctions.updateVariance 

2013 """ 

2014 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2015 gain = amp.getGain() 

2016 

2017 if math.isnan(gain): 

2018 gain = 1.0 

2019 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2020 elif gain <= 0: 

2021 patchedGain = 1.0 

2022 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

2023 amp.getName(), gain, patchedGain) 

2024 gain = patchedGain 

2025 

2026 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2027 self.log.info("Overscan is none for EmpiricalReadNoise.") 

2028 

2029 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2030 stats = afwMath.StatisticsControl() 

2031 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2032 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2033 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2034 amp.getName(), readNoise) 

2035 else: 

2036 readNoise = amp.getReadNoise() 

2037 

2038 isrFunctions.updateVariance( 

2039 maskedImage=ampExposure.getMaskedImage(), 

2040 gain=gain, 

2041 readNoise=readNoise, 

2042 ) 

2043 

2044 def darkCorrection(self, exposure, darkExposure, invert=False): 

2045 """!Apply dark correction in place. 

2046 

2047 Parameters 

2048 ---------- 

2049 exposure : `lsst.afw.image.Exposure` 

2050 Exposure to process. 

2051 darkExposure : `lsst.afw.image.Exposure` 

2052 Dark exposure of the same size as ``exposure``. 

2053 invert : `Bool`, optional 

2054 If True, re-add the dark to an already corrected image. 

2055 

2056 Raises 

2057 ------ 

2058 RuntimeError 

2059 Raised if either ``exposure`` or ``darkExposure`` do not 

2060 have their dark time defined. 

2061 

2062 See Also 

2063 -------- 

2064 lsst.ip.isr.isrFunctions.darkCorrection 

2065 """ 

2066 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2067 if math.isnan(expScale): 

2068 raise RuntimeError("Exposure darktime is NAN.") 

2069 if darkExposure.getInfo().getVisitInfo() is not None \ 

2070 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2071 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2072 else: 

2073 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2074 # so getDarkTime() does not exist. 

2075 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2076 darkScale = 1.0 

2077 

2078 isrFunctions.darkCorrection( 

2079 maskedImage=exposure.getMaskedImage(), 

2080 darkMaskedImage=darkExposure.getMaskedImage(), 

2081 expScale=expScale, 

2082 darkScale=darkScale, 

2083 invert=invert, 

2084 trimToFit=self.config.doTrimToMatchCalib 

2085 ) 

2086 

2087 def doLinearize(self, detector): 

2088 """!Check if linearization is needed for the detector cameraGeom. 

2089 

2090 Checks config.doLinearize and the linearity type of the first 

2091 amplifier. 

2092 

2093 Parameters 

2094 ---------- 

2095 detector : `lsst.afw.cameraGeom.Detector` 

2096 Detector to get linearity type from. 

2097 

2098 Returns 

2099 ------- 

2100 doLinearize : `Bool` 

2101 If True, linearization should be performed. 

2102 """ 

2103 return self.config.doLinearize and \ 

2104 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2105 

2106 def flatCorrection(self, exposure, flatExposure, invert=False): 

2107 """!Apply flat correction in place. 

2108 

2109 Parameters 

2110 ---------- 

2111 exposure : `lsst.afw.image.Exposure` 

2112 Exposure to process. 

2113 flatExposure : `lsst.afw.image.Exposure` 

2114 Flat exposure of the same size as ``exposure``. 

2115 invert : `Bool`, optional 

2116 If True, unflatten an already flattened image. 

2117 

2118 See Also 

2119 -------- 

2120 lsst.ip.isr.isrFunctions.flatCorrection 

2121 """ 

2122 isrFunctions.flatCorrection( 

2123 maskedImage=exposure.getMaskedImage(), 

2124 flatMaskedImage=flatExposure.getMaskedImage(), 

2125 scalingType=self.config.flatScalingType, 

2126 userScale=self.config.flatUserScale, 

2127 invert=invert, 

2128 trimToFit=self.config.doTrimToMatchCalib 

2129 ) 

2130 

2131 def saturationDetection(self, exposure, amp): 

2132 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2133 

2134 Parameters 

2135 ---------- 

2136 exposure : `lsst.afw.image.Exposure` 

2137 Exposure to process. Only the amplifier DataSec is processed. 

2138 amp : `lsst.afw.table.AmpInfoCatalog` 

2139 Amplifier detector data. 

2140 

2141 See Also 

2142 -------- 

2143 lsst.ip.isr.isrFunctions.makeThresholdMask 

2144 """ 

2145 if not math.isnan(amp.getSaturation()): 

2146 maskedImage = exposure.getMaskedImage() 

2147 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2148 isrFunctions.makeThresholdMask( 

2149 maskedImage=dataView, 

2150 threshold=amp.getSaturation(), 

2151 growFootprints=0, 

2152 maskName=self.config.saturatedMaskName, 

2153 ) 

2154 

2155 def saturationInterpolation(self, exposure): 

2156 """!Interpolate over saturated pixels, in place. 

2157 

2158 This method should be called after `saturationDetection`, to 

2159 ensure that the saturated pixels have been identified in the 

2160 SAT mask. It should also be called after `assembleCcd`, since 

2161 saturated regions may cross amplifier boundaries. 

2162 

2163 Parameters 

2164 ---------- 

2165 exposure : `lsst.afw.image.Exposure` 

2166 Exposure to process. 

2167 

2168 See Also 

2169 -------- 

2170 lsst.ip.isr.isrTask.saturationDetection 

2171 lsst.ip.isr.isrFunctions.interpolateFromMask 

2172 """ 

2173 isrFunctions.interpolateFromMask( 

2174 maskedImage=exposure.getMaskedImage(), 

2175 fwhm=self.config.fwhm, 

2176 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2177 maskNameList=list(self.config.saturatedMaskName), 

2178 ) 

2179 

2180 def suspectDetection(self, exposure, amp): 

2181 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2182 

2183 Parameters 

2184 ---------- 

2185 exposure : `lsst.afw.image.Exposure` 

2186 Exposure to process. Only the amplifier DataSec is processed. 

2187 amp : `lsst.afw.table.AmpInfoCatalog` 

2188 Amplifier detector data. 

2189 

2190 See Also 

2191 -------- 

2192 lsst.ip.isr.isrFunctions.makeThresholdMask 

2193 

2194 Notes 

2195 ----- 

2196 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2197 This is intended to indicate pixels that may be affected by unknown systematics; 

2198 for example if non-linearity corrections above a certain level are unstable 

2199 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2200 that no such level exists and no pixels are to be masked as suspicious. 

2201 """ 

2202 suspectLevel = amp.getSuspectLevel() 

2203 if math.isnan(suspectLevel): 

2204 return 

2205 

2206 maskedImage = exposure.getMaskedImage() 

2207 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2208 isrFunctions.makeThresholdMask( 

2209 maskedImage=dataView, 

2210 threshold=suspectLevel, 

2211 growFootprints=0, 

2212 maskName=self.config.suspectMaskName, 

2213 ) 

2214 

2215 def maskDefect(self, exposure, defectBaseList): 

2216 """!Mask defects using mask plane "BAD", in place. 

2217 

2218 Parameters 

2219 ---------- 

2220 exposure : `lsst.afw.image.Exposure` 

2221 Exposure to process. 

2222 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2223 `lsst.afw.image.DefectBase`. 

2224 List of defects to mask. 

2225 

2226 Notes 

2227 ----- 

2228 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2229 """ 

2230 maskedImage = exposure.getMaskedImage() 

2231 if not isinstance(defectBaseList, Defects): 

2232 # Promotes DefectBase to Defect 

2233 defectList = Defects(defectBaseList) 

2234 else: 

2235 defectList = defectBaseList 

2236 defectList.maskPixels(maskedImage, maskName="BAD") 

2237 

2238 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"): 

2239 """!Mask edge pixels with applicable mask plane. 

2240 

2241 Parameters 

2242 ---------- 

2243 exposure : `lsst.afw.image.Exposure` 

2244 Exposure to process. 

2245 numEdgePixels : `int`, optional 

2246 Number of edge pixels to mask. 

2247 maskPlane : `str`, optional 

2248 Mask plane name to use. 

2249 """ 

2250 maskedImage = exposure.getMaskedImage() 

2251 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2252 

2253 if numEdgePixels > 0: 

2254 goodBBox = maskedImage.getBBox() 

2255 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2256 goodBBox.grow(-numEdgePixels) 

2257 # Mask pixels outside goodBBox 

2258 SourceDetectionTask.setEdgeBits( 

2259 maskedImage, 

2260 goodBBox, 

2261 maskBitMask 

2262 ) 

2263 

2264 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2265 """Mask and interpolate defects using mask plane "BAD", in place. 

2266 

2267 Parameters 

2268 ---------- 

2269 exposure : `lsst.afw.image.Exposure` 

2270 Exposure to process. 

2271 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2272 `lsst.afw.image.DefectBase`. 

2273 List of defects to mask and interpolate. 

2274 

2275 See Also 

2276 -------- 

2277 lsst.ip.isr.isrTask.maskDefect() 

2278 """ 

2279 self.maskDefect(exposure, defectBaseList) 

2280 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2281 maskPlane="SUSPECT") 

2282 isrFunctions.interpolateFromMask( 

2283 maskedImage=exposure.getMaskedImage(), 

2284 fwhm=self.config.fwhm, 

2285 growSaturatedFootprints=0, 

2286 maskNameList=["BAD"], 

2287 ) 

2288 

2289 def maskNan(self, exposure): 

2290 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2291 

2292 Parameters 

2293 ---------- 

2294 exposure : `lsst.afw.image.Exposure` 

2295 Exposure to process. 

2296 

2297 Notes 

2298 ----- 

2299 We mask over all NaNs, including those that are masked with 

2300 other bits (because those may or may not be interpolated over 

2301 later, and we want to remove all NaNs). Despite this 

2302 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2303 the historical name. 

2304 """ 

2305 maskedImage = exposure.getMaskedImage() 

2306 

2307 # Find and mask NaNs 

2308 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2309 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2310 numNans = maskNans(maskedImage, maskVal) 

2311 self.metadata.set("NUMNANS", numNans) 

2312 if numNans > 0: 

2313 self.log.warn("There were %d unmasked NaNs.", numNans) 

2314 

2315 def maskAndInterpolateNan(self, exposure): 

2316 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2317 

2318 Parameters 

2319 ---------- 

2320 exposure : `lsst.afw.image.Exposure` 

2321 Exposure to process. 

2322 

2323 See Also 

2324 -------- 

2325 lsst.ip.isr.isrTask.maskNan() 

2326 """ 

2327 self.maskNan(exposure) 

2328 isrFunctions.interpolateFromMask( 

2329 maskedImage=exposure.getMaskedImage(), 

2330 fwhm=self.config.fwhm, 

2331 growSaturatedFootprints=0, 

2332 maskNameList=["UNMASKEDNAN"], 

2333 ) 

2334 

2335 def measureBackground(self, exposure, IsrQaConfig=None): 

2336 """Measure the image background in subgrids, for quality control purposes. 

2337 

2338 Parameters 

2339 ---------- 

2340 exposure : `lsst.afw.image.Exposure` 

2341 Exposure to process. 

2342 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2343 Configuration object containing parameters on which background 

2344 statistics and subgrids to use. 

2345 """ 

2346 if IsrQaConfig is not None: 

2347 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2348 IsrQaConfig.flatness.nIter) 

2349 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2350 statsControl.setAndMask(maskVal) 

2351 maskedImage = exposure.getMaskedImage() 

2352 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2353 skyLevel = stats.getValue(afwMath.MEDIAN) 

2354 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2355 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2356 metadata = exposure.getMetadata() 

2357 metadata.set('SKYLEVEL', skyLevel) 

2358 metadata.set('SKYSIGMA', skySigma) 

2359 

2360 # calcluating flatlevel over the subgrids 

2361 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2362 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2363 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2364 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2365 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2366 skyLevels = numpy.zeros((nX, nY)) 

2367 

2368 for j in range(nY): 

2369 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2370 for i in range(nX): 

2371 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2372 

2373 xLLC = xc - meshXHalf 

2374 yLLC = yc - meshYHalf 

2375 xURC = xc + meshXHalf - 1 

2376 yURC = yc + meshYHalf - 1 

2377 

2378 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2379 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2380 

2381 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2382 

2383 good = numpy.where(numpy.isfinite(skyLevels)) 

2384 skyMedian = numpy.median(skyLevels[good]) 

2385 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2386 flatness_rms = numpy.std(flatness) 

2387 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2388 

2389 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2390 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2391 nX, nY, flatness_pp, flatness_rms) 

2392 

2393 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2394 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2395 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2396 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2397 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2398 

2399 def roughZeroPoint(self, exposure): 

2400 """Set an approximate magnitude zero point for the exposure. 

2401 

2402 Parameters 

2403 ---------- 

2404 exposure : `lsst.afw.image.Exposure` 

2405 Exposure to process. 

2406 """ 

2407 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2408 if filterName in self.config.fluxMag0T1: 

2409 fluxMag0 = self.config.fluxMag0T1[filterName] 

2410 else: 

2411 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2412 fluxMag0 = self.config.defaultFluxMag0T1 

2413 

2414 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2415 if not expTime > 0: # handle NaN as well as <= 0 

2416 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2417 return 

2418 

2419 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2420 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2421 

2422 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2423 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2424 

2425 Parameters 

2426 ---------- 

2427 ccdExposure : `lsst.afw.image.Exposure` 

2428 Exposure to process. 

2429 fpPolygon : `lsst.afw.geom.Polygon` 

2430 Polygon in focal plane coordinates. 

2431 """ 

2432 # Get ccd corners in focal plane coordinates 

2433 ccd = ccdExposure.getDetector() 

2434 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2435 ccdPolygon = Polygon(fpCorners) 

2436 

2437 # Get intersection of ccd corners with fpPolygon 

2438 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2439 

2440 # Transform back to pixel positions and build new polygon 

2441 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2442 validPolygon = Polygon(ccdPoints) 

2443 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2444 

2445 @contextmanager 

2446 def flatContext(self, exp, flat, dark=None): 

2447 """Context manager that applies and removes flats and darks, 

2448 if the task is configured to apply them. 

2449 

2450 Parameters 

2451 ---------- 

2452 exp : `lsst.afw.image.Exposure` 

2453 Exposure to process. 

2454 flat : `lsst.afw.image.Exposure` 

2455 Flat exposure the same size as ``exp``. 

2456 dark : `lsst.afw.image.Exposure`, optional 

2457 Dark exposure the same size as ``exp``. 

2458 

2459 Yields 

2460 ------ 

2461 exp : `lsst.afw.image.Exposure` 

2462 The flat and dark corrected exposure. 

2463 """ 

2464 if self.config.doDark and dark is not None: 

2465 self.darkCorrection(exp, dark) 

2466 if self.config.doFlat: 

2467 self.flatCorrection(exp, flat) 

2468 try: 

2469 yield exp 

2470 finally: 

2471 if self.config.doFlat: 

2472 self.flatCorrection(exp, flat, invert=True) 

2473 if self.config.doDark and dark is not None: 

2474 self.darkCorrection(exp, dark, invert=True) 

2475 

2476 def debugView(self, exposure, stepname): 

2477 """Utility function to examine ISR exposure at different stages. 

2478 

2479 Parameters 

2480 ---------- 

2481 exposure : `lsst.afw.image.Exposure` 

2482 Exposure to view. 

2483 stepname : `str` 

2484 State of processing to view. 

2485 """ 

2486 frame = getDebugFrame(self._display, stepname) 

2487 if frame: 

2488 display = getDisplay(frame) 

2489 display.scale('asinh', 'zscale') 

2490 display.mtv(exposure) 

2491 prompt = "Press Enter to continue [c]... " 

2492 while True: 

2493 ans = input(prompt).lower() 

2494 if ans in ("", "c",): 

2495 break 

2496 

2497 

2498class FakeAmp(object): 

2499 """A Detector-like object that supports returning gain and saturation level 

2500 

2501 This is used when the input exposure does not have a detector. 

2502 

2503 Parameters 

2504 ---------- 

2505 exposure : `lsst.afw.image.Exposure` 

2506 Exposure to generate a fake amplifier for. 

2507 config : `lsst.ip.isr.isrTaskConfig` 

2508 Configuration to apply to the fake amplifier. 

2509 """ 

2510 

2511 def __init__(self, exposure, config): 

2512 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2513 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2514 self._gain = config.gain 

2515 self._readNoise = config.readNoise 

2516 self._saturation = config.saturation 

2517 

2518 def getBBox(self): 

2519 return self._bbox 

2520 

2521 def getRawBBox(self): 

2522 return self._bbox 

2523 

2524 def getRawHorizontalOverscanBBox(self): 

2525 return self._RawHorizontalOverscanBBox 

2526 

2527 def getGain(self): 

2528 return self._gain 

2529 

2530 def getReadNoise(self): 

2531 return self._readNoise 

2532 

2533 def getSaturation(self): 

2534 return self._saturation 

2535 

2536 def getSuspectLevel(self): 

2537 return float("NaN") 

2538 

2539 

2540class RunIsrConfig(pexConfig.Config): 

2541 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2542 

2543 

2544class RunIsrTask(pipeBase.CmdLineTask): 

2545 """Task to wrap the default IsrTask to allow it to be retargeted. 

2546 

2547 The standard IsrTask can be called directly from a command line 

2548 program, but doing so removes the ability of the task to be 

2549 retargeted. As most cameras override some set of the IsrTask 

2550 methods, this would remove those data-specific methods in the 

2551 output post-ISR images. This wrapping class fixes the issue, 

2552 allowing identical post-ISR images to be generated by both the 

2553 processCcd and isrTask code. 

2554 """ 

2555 ConfigClass = RunIsrConfig 

2556 _DefaultName = "runIsr" 

2557 

2558 def __init__(self, *args, **kwargs): 

2559 super().__init__(*args, **kwargs) 

2560 self.makeSubtask("isr") 

2561 

2562 def runDataRef(self, dataRef): 

2563 """ 

2564 Parameters 

2565 ---------- 

2566 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2567 data reference of the detector data to be processed 

2568 

2569 Returns 

2570 ------- 

2571 result : `pipeBase.Struct` 

2572 Result struct with component: 

2573 

2574 - exposure : `lsst.afw.image.Exposure` 

2575 Post-ISR processed exposure. 

2576 """ 

2577 return self.isr.runDataRef(dataRef)