Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42from lsst.meas.algorithms import Defects 

43 

44from . import isrFunctions 

45from . import isrQa 

46from . import linearize 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 deduplicate=True, 

95 expand=True)) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument", "calibration_label"], 

113 ) 

114 

115 crosstalk = cT.PrerequisiteInput( 

116 name="crosstalk", 

117 doc="Input crosstalk object", 

118 storageClass="CrosstalkCalib", 

119 dimensions=["instrument", "calibration_label", "detector"], 

120 ) 

121 # TODO: DM-25348. This does not work yet to correctly load 

122 # possible crosstalk sources. 

123 crosstalkSources = cT.PrerequisiteInput( 

124 name="isrOverscanCorrected", 

125 doc="Overscan corrected input images.", 

126 storageClass="Exposure", 

127 dimensions=["instrument", "exposure", "detector"], 

128 deferLoad=True, 

129 multiple=True, 

130 lookupFunction=crosstalkSourceLookup, 

131 ) 

132 bias = cT.PrerequisiteInput( 

133 name="bias", 

134 doc="Input bias calibration.", 

135 storageClass="ExposureF", 

136 dimensions=["instrument", "calibration_label", "detector"], 

137 ) 

138 dark = cT.PrerequisiteInput( 

139 name='dark', 

140 doc="Input dark calibration.", 

141 storageClass="ExposureF", 

142 dimensions=["instrument", "calibration_label", "detector"], 

143 ) 

144 flat = cT.PrerequisiteInput( 

145 name="flat", 

146 doc="Input flat calibration.", 

147 storageClass="ExposureF", 

148 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

149 ) 

150 fringes = cT.PrerequisiteInput( 

151 name="fringe", 

152 doc="Input fringe calibration.", 

153 storageClass="ExposureF", 

154 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

155 ) 

156 strayLightData = cT.PrerequisiteInput( 

157 name='yBackground', 

158 doc="Input stray light calibration.", 

159 storageClass="StrayLightData", 

160 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

161 ) 

162 bfKernel = cT.PrerequisiteInput( 

163 name='bfKernel', 

164 doc="Input brighter-fatter kernel.", 

165 storageClass="NumpyArray", 

166 dimensions=["instrument", "calibration_label"], 

167 ) 

168 newBFKernel = cT.PrerequisiteInput( 

169 name='brighterFatterKernel', 

170 doc="Newer complete kernel + gain solutions.", 

171 storageClass="BrighterFatterKernel", 

172 dimensions=["instrument", "calibration_label", "detector"], 

173 ) 

174 defects = cT.PrerequisiteInput( 

175 name='defects', 

176 doc="Input defect tables.", 

177 storageClass="Defects", 

178 dimensions=["instrument", "calibration_label", "detector"], 

179 ) 

180 opticsTransmission = cT.PrerequisiteInput( 

181 name="transmission_optics", 

182 storageClass="TransmissionCurve", 

183 doc="Transmission curve due to the optics.", 

184 dimensions=["instrument", "calibration_label"], 

185 ) 

186 filterTransmission = cT.PrerequisiteInput( 

187 name="transmission_filter", 

188 storageClass="TransmissionCurve", 

189 doc="Transmission curve due to the filter.", 

190 dimensions=["instrument", "physical_filter", "calibration_label"], 

191 ) 

192 sensorTransmission = cT.PrerequisiteInput( 

193 name="transmission_sensor", 

194 storageClass="TransmissionCurve", 

195 doc="Transmission curve due to the sensor.", 

196 dimensions=["instrument", "calibration_label", "detector"], 

197 ) 

198 atmosphereTransmission = cT.PrerequisiteInput( 

199 name="transmission_atmosphere", 

200 storageClass="TransmissionCurve", 

201 doc="Transmission curve due to the atmosphere.", 

202 dimensions=["instrument"], 

203 ) 

204 illumMaskedImage = cT.PrerequisiteInput( 

205 name="illum", 

206 doc="Input illumination correction.", 

207 storageClass="MaskedImageF", 

208 dimensions=["instrument", "physical_filter", "calibration_label", "detector"], 

209 ) 

210 

211 outputExposure = cT.Output( 

212 name='postISRCCD', 

213 doc="Output ISR processed exposure.", 

214 storageClass="Exposure", 

215 dimensions=["instrument", "exposure", "detector"], 

216 ) 

217 preInterpExposure = cT.Output( 

218 name='preInterpISRCCD', 

219 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

220 storageClass="ExposureF", 

221 dimensions=["instrument", "exposure", "detector"], 

222 ) 

223 outputOssThumbnail = cT.Output( 

224 name="OssThumb", 

225 doc="Output Overscan-subtracted thumbnail image.", 

226 storageClass="Thumbnail", 

227 dimensions=["instrument", "exposure", "detector"], 

228 ) 

229 outputFlattenedThumbnail = cT.Output( 

230 name="FlattenedThumb", 

231 doc="Output flat-corrected thumbnail image.", 

232 storageClass="Thumbnail", 

233 dimensions=["instrument", "exposure", "detector"], 

234 ) 

235 

236 def __init__(self, *, config=None): 

237 super().__init__(config=config) 

238 

239 if config.doBias is not True: 

240 self.prerequisiteInputs.discard("bias") 

241 if config.doLinearize is not True: 

242 self.prerequisiteInputs.discard("linearizer") 

243 if config.doCrosstalk is not True: 

244 self.inputs.discard("crosstalkSources") 

245 self.prerequisiteInputs.discard("crosstalk") 

246 if config.doBrighterFatter is not True: 

247 self.prerequisiteInputs.discard("bfKernel") 

248 self.prerequisiteInputs.discard("newBFKernel") 

249 if config.doDefect is not True: 

250 self.prerequisiteInputs.discard("defects") 

251 if config.doDark is not True: 

252 self.prerequisiteInputs.discard("dark") 

253 if config.doFlat is not True: 

254 self.prerequisiteInputs.discard("flat") 

255 if config.doAttachTransmissionCurve is not True: 

256 self.prerequisiteInputs.discard("opticsTransmission") 

257 self.prerequisiteInputs.discard("filterTransmission") 

258 self.prerequisiteInputs.discard("sensorTransmission") 

259 self.prerequisiteInputs.discard("atmosphereTransmission") 

260 if config.doUseOpticsTransmission is not True: 

261 self.prerequisiteInputs.discard("opticsTransmission") 

262 if config.doUseFilterTransmission is not True: 

263 self.prerequisiteInputs.discard("filterTransmission") 

264 if config.doUseSensorTransmission is not True: 

265 self.prerequisiteInputs.discard("sensorTransmission") 

266 if config.doUseAtmosphereTransmission is not True: 

267 self.prerequisiteInputs.discard("atmosphereTransmission") 

268 if config.doIlluminationCorrection is not True: 

269 self.prerequisiteInputs.discard("illumMaskedImage") 

270 

271 if config.doWrite is not True: 

272 self.outputs.discard("outputExposure") 

273 self.outputs.discard("preInterpExposure") 

274 self.outputs.discard("outputFlattenedThumbnail") 

275 self.outputs.discard("outputOssThumbnail") 

276 if config.doSaveInterpPixels is not True: 

277 self.outputs.discard("preInterpExposure") 

278 if config.qa.doThumbnailOss is not True: 

279 self.outputs.discard("outputOssThumbnail") 

280 if config.qa.doThumbnailFlattened is not True: 

281 self.outputs.discard("outputFlattenedThumbnail") 

282 

283 

284class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

285 pipelineConnections=IsrTaskConnections): 

286 """Configuration parameters for IsrTask. 

287 

288 Items are grouped in the order in which they are executed by the task. 

289 """ 

290 datasetType = pexConfig.Field( 

291 dtype=str, 

292 doc="Dataset type for input data; users will typically leave this alone, " 

293 "but camera-specific ISR tasks will override it", 

294 default="raw", 

295 ) 

296 

297 fallbackFilterName = pexConfig.Field( 

298 dtype=str, 

299 doc="Fallback default filter name for calibrations.", 

300 optional=True 

301 ) 

302 useFallbackDate = pexConfig.Field( 

303 dtype=bool, 

304 doc="Pass observation date when using fallback filter.", 

305 default=False, 

306 ) 

307 expectWcs = pexConfig.Field( 

308 dtype=bool, 

309 default=True, 

310 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

311 ) 

312 fwhm = pexConfig.Field( 

313 dtype=float, 

314 doc="FWHM of PSF in arcseconds.", 

315 default=1.0, 

316 ) 

317 qa = pexConfig.ConfigField( 

318 dtype=isrQa.IsrQaConfig, 

319 doc="QA related configuration options.", 

320 ) 

321 

322 # Image conversion configuration 

323 doConvertIntToFloat = pexConfig.Field( 

324 dtype=bool, 

325 doc="Convert integer raw images to floating point values?", 

326 default=True, 

327 ) 

328 

329 # Saturated pixel handling. 

330 doSaturation = pexConfig.Field( 

331 dtype=bool, 

332 doc="Mask saturated pixels? NB: this is totally independent of the" 

333 " interpolation option - this is ONLY setting the bits in the mask." 

334 " To have them interpolated make sure doSaturationInterpolation=True", 

335 default=True, 

336 ) 

337 saturatedMaskName = pexConfig.Field( 

338 dtype=str, 

339 doc="Name of mask plane to use in saturation detection and interpolation", 

340 default="SAT", 

341 ) 

342 saturation = pexConfig.Field( 

343 dtype=float, 

344 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

345 default=float("NaN"), 

346 ) 

347 growSaturationFootprintSize = pexConfig.Field( 

348 dtype=int, 

349 doc="Number of pixels by which to grow the saturation footprints", 

350 default=1, 

351 ) 

352 

353 # Suspect pixel handling. 

354 doSuspect = pexConfig.Field( 

355 dtype=bool, 

356 doc="Mask suspect pixels?", 

357 default=False, 

358 ) 

359 suspectMaskName = pexConfig.Field( 

360 dtype=str, 

361 doc="Name of mask plane to use for suspect pixels", 

362 default="SUSPECT", 

363 ) 

364 numEdgeSuspect = pexConfig.Field( 

365 dtype=int, 

366 doc="Number of edge pixels to be flagged as untrustworthy.", 

367 default=0, 

368 ) 

369 

370 # Initial masking options. 

371 doSetBadRegions = pexConfig.Field( 

372 dtype=bool, 

373 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

374 default=True, 

375 ) 

376 badStatistic = pexConfig.ChoiceField( 

377 dtype=str, 

378 doc="How to estimate the average value for BAD regions.", 

379 default='MEANCLIP', 

380 allowed={ 

381 "MEANCLIP": "Correct using the (clipped) mean of good data", 

382 "MEDIAN": "Correct using the median of the good data", 

383 }, 

384 ) 

385 

386 # Overscan subtraction configuration. 

387 doOverscan = pexConfig.Field( 

388 dtype=bool, 

389 doc="Do overscan subtraction?", 

390 default=True, 

391 ) 

392 overscan = pexConfig.ConfigurableField( 

393 target=OverscanCorrectionTask, 

394 doc="Overscan subtraction task for image segments.", 

395 ) 

396 

397 overscanFitType = pexConfig.ChoiceField( 

398 dtype=str, 

399 doc="The method for fitting the overscan bias level.", 

400 default='MEDIAN', 

401 allowed={ 

402 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

403 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

404 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

405 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

406 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

407 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

408 "MEAN": "Correct using the mean of the overscan region", 

409 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

410 "MEDIAN": "Correct using the median of the overscan region", 

411 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

412 }, 

413 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

414 " This option will no longer be used, and will be removed after v20.") 

415 ) 

416 overscanOrder = pexConfig.Field( 

417 dtype=int, 

418 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

419 "or number of spline knots if overscan fit type is a spline."), 

420 default=1, 

421 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

422 " This option will no longer be used, and will be removed after v20.") 

423 ) 

424 overscanNumSigmaClip = pexConfig.Field( 

425 dtype=float, 

426 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

427 default=3.0, 

428 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

429 " This option will no longer be used, and will be removed after v20.") 

430 ) 

431 overscanIsInt = pexConfig.Field( 

432 dtype=bool, 

433 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

434 " and overscan.FitType=MEDIAN_PER_ROW.", 

435 default=True, 

436 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

437 " This option will no longer be used, and will be removed after v20.") 

438 ) 

439 # These options do not get deprecated, as they define how we slice up the image data. 

440 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

441 dtype=int, 

442 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

443 default=0, 

444 ) 

445 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

446 dtype=int, 

447 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

448 default=0, 

449 ) 

450 overscanMaxDev = pexConfig.Field( 450 ↛ exitline 450 didn't jump to the function exit

451 dtype=float, 

452 doc="Maximum deviation from the median for overscan", 

453 default=1000.0, check=lambda x: x > 0 

454 ) 

455 overscanBiasJump = pexConfig.Field( 

456 dtype=bool, 

457 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

458 default=False, 

459 ) 

460 overscanBiasJumpKeyword = pexConfig.Field( 

461 dtype=str, 

462 doc="Header keyword containing information about devices.", 

463 default="NO_SUCH_KEY", 

464 ) 

465 overscanBiasJumpDevices = pexConfig.ListField( 

466 dtype=str, 

467 doc="List of devices that need piecewise overscan correction.", 

468 default=(), 

469 ) 

470 overscanBiasJumpLocation = pexConfig.Field( 

471 dtype=int, 

472 doc="Location of bias jump along y-axis.", 

473 default=0, 

474 ) 

475 

476 # Amplifier to CCD assembly configuration 

477 doAssembleCcd = pexConfig.Field( 

478 dtype=bool, 

479 default=True, 

480 doc="Assemble amp-level exposures into a ccd-level exposure?" 

481 ) 

482 assembleCcd = pexConfig.ConfigurableField( 

483 target=AssembleCcdTask, 

484 doc="CCD assembly task", 

485 ) 

486 

487 # General calibration configuration. 

488 doAssembleIsrExposures = pexConfig.Field( 

489 dtype=bool, 

490 default=False, 

491 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

492 ) 

493 doTrimToMatchCalib = pexConfig.Field( 

494 dtype=bool, 

495 default=False, 

496 doc="Trim raw data to match calibration bounding boxes?" 

497 ) 

498 

499 # Bias subtraction. 

500 doBias = pexConfig.Field( 

501 dtype=bool, 

502 doc="Apply bias frame correction?", 

503 default=True, 

504 ) 

505 biasDataProductName = pexConfig.Field( 

506 dtype=str, 

507 doc="Name of the bias data product", 

508 default="bias", 

509 ) 

510 

511 # Variance construction 

512 doVariance = pexConfig.Field( 

513 dtype=bool, 

514 doc="Calculate variance?", 

515 default=True 

516 ) 

517 gain = pexConfig.Field( 

518 dtype=float, 

519 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

520 default=float("NaN"), 

521 ) 

522 readNoise = pexConfig.Field( 

523 dtype=float, 

524 doc="The read noise to use if no Detector is present in the Exposure", 

525 default=0.0, 

526 ) 

527 doEmpiricalReadNoise = pexConfig.Field( 

528 dtype=bool, 

529 default=False, 

530 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

531 ) 

532 

533 # Linearization. 

534 doLinearize = pexConfig.Field( 

535 dtype=bool, 

536 doc="Correct for nonlinearity of the detector's response?", 

537 default=True, 

538 ) 

539 

540 # Crosstalk. 

541 doCrosstalk = pexConfig.Field( 

542 dtype=bool, 

543 doc="Apply intra-CCD crosstalk correction?", 

544 default=False, 

545 ) 

546 doCrosstalkBeforeAssemble = pexConfig.Field( 

547 dtype=bool, 

548 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

549 default=False, 

550 ) 

551 crosstalk = pexConfig.ConfigurableField( 

552 target=CrosstalkTask, 

553 doc="Intra-CCD crosstalk correction", 

554 ) 

555 

556 # Masking options. 

557 doDefect = pexConfig.Field( 

558 dtype=bool, 

559 doc="Apply correction for CCD defects, e.g. hot pixels?", 

560 default=True, 

561 ) 

562 doNanMasking = pexConfig.Field( 

563 dtype=bool, 

564 doc="Mask NAN pixels?", 

565 default=True, 

566 ) 

567 doWidenSaturationTrails = pexConfig.Field( 

568 dtype=bool, 

569 doc="Widen bleed trails based on their width?", 

570 default=True 

571 ) 

572 

573 # Brighter-Fatter correction. 

574 doBrighterFatter = pexConfig.Field( 

575 dtype=bool, 

576 default=False, 

577 doc="Apply the brighter fatter correction" 

578 ) 

579 brighterFatterLevel = pexConfig.ChoiceField( 

580 dtype=str, 

581 default="DETECTOR", 

582 doc="The level at which to correct for brighter-fatter.", 

583 allowed={ 

584 "AMP": "Every amplifier treated separately.", 

585 "DETECTOR": "One kernel per detector", 

586 } 

587 ) 

588 brighterFatterMaxIter = pexConfig.Field( 

589 dtype=int, 

590 default=10, 

591 doc="Maximum number of iterations for the brighter fatter correction" 

592 ) 

593 brighterFatterThreshold = pexConfig.Field( 

594 dtype=float, 

595 default=1000, 

596 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

597 " absolute value of the difference between the current corrected image and the one" 

598 " from the previous iteration summed over all the pixels." 

599 ) 

600 brighterFatterApplyGain = pexConfig.Field( 

601 dtype=bool, 

602 default=True, 

603 doc="Should the gain be applied when applying the brighter fatter correction?" 

604 ) 

605 brighterFatterMaskGrowSize = pexConfig.Field( 

606 dtype=int, 

607 default=0, 

608 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

609 " when brighter-fatter correction is applied." 

610 ) 

611 

612 # Dark subtraction. 

613 doDark = pexConfig.Field( 

614 dtype=bool, 

615 doc="Apply dark frame correction?", 

616 default=True, 

617 ) 

618 darkDataProductName = pexConfig.Field( 

619 dtype=str, 

620 doc="Name of the dark data product", 

621 default="dark", 

622 ) 

623 

624 # Camera-specific stray light removal. 

625 doStrayLight = pexConfig.Field( 

626 dtype=bool, 

627 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

628 default=False, 

629 ) 

630 strayLight = pexConfig.ConfigurableField( 

631 target=StrayLightTask, 

632 doc="y-band stray light correction" 

633 ) 

634 

635 # Flat correction. 

636 doFlat = pexConfig.Field( 

637 dtype=bool, 

638 doc="Apply flat field correction?", 

639 default=True, 

640 ) 

641 flatDataProductName = pexConfig.Field( 

642 dtype=str, 

643 doc="Name of the flat data product", 

644 default="flat", 

645 ) 

646 flatScalingType = pexConfig.ChoiceField( 

647 dtype=str, 

648 doc="The method for scaling the flat on the fly.", 

649 default='USER', 

650 allowed={ 

651 "USER": "Scale by flatUserScale", 

652 "MEAN": "Scale by the inverse of the mean", 

653 "MEDIAN": "Scale by the inverse of the median", 

654 }, 

655 ) 

656 flatUserScale = pexConfig.Field( 

657 dtype=float, 

658 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

659 default=1.0, 

660 ) 

661 doTweakFlat = pexConfig.Field( 

662 dtype=bool, 

663 doc="Tweak flats to match observed amplifier ratios?", 

664 default=False 

665 ) 

666 

667 # Amplifier normalization based on gains instead of using flats configuration. 

668 doApplyGains = pexConfig.Field( 

669 dtype=bool, 

670 doc="Correct the amplifiers for their gains instead of applying flat correction", 

671 default=False, 

672 ) 

673 normalizeGains = pexConfig.Field( 

674 dtype=bool, 

675 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

676 default=False, 

677 ) 

678 

679 # Fringe correction. 

680 doFringe = pexConfig.Field( 

681 dtype=bool, 

682 doc="Apply fringe correction?", 

683 default=True, 

684 ) 

685 fringe = pexConfig.ConfigurableField( 

686 target=FringeTask, 

687 doc="Fringe subtraction task", 

688 ) 

689 fringeAfterFlat = pexConfig.Field( 

690 dtype=bool, 

691 doc="Do fringe subtraction after flat-fielding?", 

692 default=True, 

693 ) 

694 

695 # Initial CCD-level background statistics options. 

696 doMeasureBackground = pexConfig.Field( 

697 dtype=bool, 

698 doc="Measure the background level on the reduced image?", 

699 default=False, 

700 ) 

701 

702 # Camera-specific masking configuration. 

703 doCameraSpecificMasking = pexConfig.Field( 

704 dtype=bool, 

705 doc="Mask camera-specific bad regions?", 

706 default=False, 

707 ) 

708 masking = pexConfig.ConfigurableField( 

709 target=MaskingTask, 

710 doc="Masking task." 

711 ) 

712 

713 # Interpolation options. 

714 

715 doInterpolate = pexConfig.Field( 

716 dtype=bool, 

717 doc="Interpolate masked pixels?", 

718 default=True, 

719 ) 

720 doSaturationInterpolation = pexConfig.Field( 

721 dtype=bool, 

722 doc="Perform interpolation over pixels masked as saturated?" 

723 " NB: This is independent of doSaturation; if that is False this plane" 

724 " will likely be blank, resulting in a no-op here.", 

725 default=True, 

726 ) 

727 doNanInterpolation = pexConfig.Field( 

728 dtype=bool, 

729 doc="Perform interpolation over pixels masked as NaN?" 

730 " NB: This is independent of doNanMasking; if that is False this plane" 

731 " will likely be blank, resulting in a no-op here.", 

732 default=True, 

733 ) 

734 doNanInterpAfterFlat = pexConfig.Field( 

735 dtype=bool, 

736 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

737 "also have to interpolate them before flat-fielding."), 

738 default=False, 

739 ) 

740 maskListToInterpolate = pexConfig.ListField( 

741 dtype=str, 

742 doc="List of mask planes that should be interpolated.", 

743 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

744 ) 

745 doSaveInterpPixels = pexConfig.Field( 

746 dtype=bool, 

747 doc="Save a copy of the pre-interpolated pixel values?", 

748 default=False, 

749 ) 

750 

751 # Default photometric calibration options. 

752 fluxMag0T1 = pexConfig.DictField( 

753 keytype=str, 

754 itemtype=float, 

755 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

756 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

757 )) 

758 ) 

759 defaultFluxMag0T1 = pexConfig.Field( 

760 dtype=float, 

761 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

762 default=pow(10.0, 0.4*28.0) 

763 ) 

764 

765 # Vignette correction configuration. 

766 doVignette = pexConfig.Field( 

767 dtype=bool, 

768 doc="Apply vignetting parameters?", 

769 default=False, 

770 ) 

771 vignette = pexConfig.ConfigurableField( 

772 target=VignetteTask, 

773 doc="Vignetting task.", 

774 ) 

775 

776 # Transmission curve configuration. 

777 doAttachTransmissionCurve = pexConfig.Field( 

778 dtype=bool, 

779 default=False, 

780 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

781 ) 

782 doUseOpticsTransmission = pexConfig.Field( 

783 dtype=bool, 

784 default=True, 

785 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

786 ) 

787 doUseFilterTransmission = pexConfig.Field( 

788 dtype=bool, 

789 default=True, 

790 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

791 ) 

792 doUseSensorTransmission = pexConfig.Field( 

793 dtype=bool, 

794 default=True, 

795 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

796 ) 

797 doUseAtmosphereTransmission = pexConfig.Field( 

798 dtype=bool, 

799 default=True, 

800 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

801 ) 

802 

803 # Illumination correction. 

804 doIlluminationCorrection = pexConfig.Field( 

805 dtype=bool, 

806 default=False, 

807 doc="Perform illumination correction?" 

808 ) 

809 illuminationCorrectionDataProductName = pexConfig.Field( 

810 dtype=str, 

811 doc="Name of the illumination correction data product.", 

812 default="illumcor", 

813 ) 

814 illumScale = pexConfig.Field( 

815 dtype=float, 

816 doc="Scale factor for the illumination correction.", 

817 default=1.0, 

818 ) 

819 illumFilters = pexConfig.ListField( 

820 dtype=str, 

821 default=[], 

822 doc="Only perform illumination correction for these filters." 

823 ) 

824 

825 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

826 doWrite = pexConfig.Field( 

827 dtype=bool, 

828 doc="Persist postISRCCD?", 

829 default=True, 

830 ) 

831 

832 def validate(self): 

833 super().validate() 

834 if self.doFlat and self.doApplyGains: 

835 raise ValueError("You may not specify both doFlat and doApplyGains") 

836 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate: 

837 self.config.maskListToInterpolate.append("SAT") 

838 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

839 self.config.maskListToInterpolate.append("UNMASKEDNAN") 

840 

841 

842class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

843 """Apply common instrument signature correction algorithms to a raw frame. 

844 

845 The process for correcting imaging data is very similar from 

846 camera to camera. This task provides a vanilla implementation of 

847 doing these corrections, including the ability to turn certain 

848 corrections off if they are not needed. The inputs to the primary 

849 method, `run()`, are a raw exposure to be corrected and the 

850 calibration data products. The raw input is a single chip sized 

851 mosaic of all amps including overscans and other non-science 

852 pixels. The method `runDataRef()` identifies and defines the 

853 calibration data products, and is intended for use by a 

854 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

855 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

856 subclassed for different camera, although the most camera specific 

857 methods have been split into subtasks that can be redirected 

858 appropriately. 

859 

860 The __init__ method sets up the subtasks for ISR processing, using 

861 the defaults from `lsst.ip.isr`. 

862 

863 Parameters 

864 ---------- 

865 args : `list` 

866 Positional arguments passed to the Task constructor. None used at this time. 

867 kwargs : `dict`, optional 

868 Keyword arguments passed on to the Task constructor. None used at this time. 

869 """ 

870 ConfigClass = IsrTaskConfig 

871 _DefaultName = "isr" 

872 

873 def __init__(self, **kwargs): 

874 super().__init__(**kwargs) 

875 self.makeSubtask("assembleCcd") 

876 self.makeSubtask("crosstalk") 

877 self.makeSubtask("strayLight") 

878 self.makeSubtask("fringe") 

879 self.makeSubtask("masking") 

880 self.makeSubtask("overscan") 

881 self.makeSubtask("vignette") 

882 

883 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

884 inputs = butlerQC.get(inputRefs) 

885 

886 try: 

887 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

888 except Exception as e: 

889 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

890 (inputRefs, e)) 

891 

892 inputs['isGen3'] = True 

893 

894 detector = inputs['ccdExposure'].getDetector() 

895 

896 if self.config.doCrosstalk is True: 

897 # Crosstalk sources need to be defined by the pipeline 

898 # yaml if they exist. 

899 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

900 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

901 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

902 else: 

903 coeffVector = (self.config.crosstalk.crosstalkValues 

904 if self.config.crosstalk.useConfigCoefficients else None) 

905 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

906 inputs['crosstalk'] = crosstalkCalib 

907 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

908 if 'crosstalkSources' not in inputs: 

909 self.log.warn("No crosstalkSources found for chip with interChip terms!") 

910 

911 if self.doLinearize(detector) is True: 

912 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict): 

913 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

914 linearizer.fromYaml(inputs['linearizer']) 

915 else: 

916 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector, 

917 log=self.log) 

918 inputs['linearizer'] = linearizer 

919 

920 if self.config.doDefect is True: 

921 if "defects" in inputs and inputs['defects'] is not None: 

922 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

923 # masking expects a list of defects defined by their bounding box 

924 if not isinstance(inputs["defects"], Defects): 

925 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

926 

927 # Load the correct style of brighter fatter kernel, and repack 

928 # the information as a numpy array. 

929 if self.config.doBrighterFatter: 

930 brighterFatterKernel = inputs.pop('newBFKernel', None) 

931 if brighterFatterKernel is None: 

932 brighterFatterKernel = inputs.get('bfKernel', None) 

933 

934 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

935 detId = detector.getId() 

936 inputs['bfGains'] = brighterFatterKernel.gain 

937 # If the kernel is not an ndarray, it's the cp_pipe version 

938 # so extract the kernel for this detector, or raise an error 

939 if self.config.brighterFatterLevel == 'DETECTOR': 

940 if brighterFatterKernel.detectorKernel: 

941 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

942 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

943 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

944 else: 

945 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

946 else: 

947 # TODO DM-15631 for implementing this 

948 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

949 

950 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

951 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

952 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

953 expId=expId, 

954 assembler=self.assembleCcd 

955 if self.config.doAssembleIsrExposures else None) 

956 else: 

957 inputs['fringes'] = pipeBase.Struct(fringes=None) 

958 

959 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

960 if 'strayLightData' not in inputs: 

961 inputs['strayLightData'] = None 

962 

963 outputs = self.run(**inputs) 

964 butlerQC.put(outputs, outputRefs) 

965 

966 def readIsrData(self, dataRef, rawExposure): 

967 """!Retrieve necessary frames for instrument signature removal. 

968 

969 Pre-fetching all required ISR data products limits the IO 

970 required by the ISR. Any conflict between the calibration data 

971 available and that needed for ISR is also detected prior to 

972 doing processing, allowing it to fail quickly. 

973 

974 Parameters 

975 ---------- 

976 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

977 Butler reference of the detector data to be processed 

978 rawExposure : `afw.image.Exposure` 

979 The raw exposure that will later be corrected with the 

980 retrieved calibration data; should not be modified in this 

981 method. 

982 

983 Returns 

984 ------- 

985 result : `lsst.pipe.base.Struct` 

986 Result struct with components (which may be `None`): 

987 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

988 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

989 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

990 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

991 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

992 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

993 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 

994 - ``fringes``: `lsst.pipe.base.Struct` with components: 

995 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

996 - ``seed``: random seed derived from the ccdExposureId for random 

997 number generator (`uint32`). 

998 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

999 A ``TransmissionCurve`` that represents the throughput of the optics, 

1000 to be evaluated in focal-plane coordinates. 

1001 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1002 A ``TransmissionCurve`` that represents the throughput of the filter 

1003 itself, to be evaluated in focal-plane coordinates. 

1004 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1005 A ``TransmissionCurve`` that represents the throughput of the sensor 

1006 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1007 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1008 A ``TransmissionCurve`` that represents the throughput of the 

1009 atmosphere, assumed to be spatially constant. 

1010 - ``strayLightData`` : `object` 

1011 An opaque object containing calibration information for 

1012 stray-light correction. If `None`, no correction will be 

1013 performed. 

1014 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1015 

1016 Raises 

1017 ------ 

1018 NotImplementedError : 

1019 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1020 """ 

1021 try: 

1022 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1023 dateObs = dateObs.toPython().isoformat() 

1024 except RuntimeError: 

1025 self.log.warn("Unable to identify dateObs for rawExposure.") 

1026 dateObs = None 

1027 

1028 ccd = rawExposure.getDetector() 

1029 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

1030 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1031 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1032 if self.config.doBias else None) 

1033 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1034 linearizer = (dataRef.get("linearizer", immediate=True) 

1035 if self.doLinearize(ccd) else None) 

1036 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1037 linearizer.log = self.log 

1038 if isinstance(linearizer, numpy.ndarray): 

1039 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1040 

1041 crosstalkCalib = None 

1042 if self.config.doCrosstalk: 

1043 try: 

1044 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1045 except NoResults: 

1046 coeffVector = (self.config.crosstalk.crosstalkValues 

1047 if self.config.crosstalk.useConfigCoefficients else None) 

1048 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1049 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1050 if self.config.doCrosstalk else None) 

1051 

1052 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1053 if self.config.doDark else None) 

1054 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1055 dateObs=dateObs) 

1056 if self.config.doFlat else None) 

1057 

1058 brighterFatterKernel = None 

1059 brighterFatterGains = None 

1060 if self.config.doBrighterFatter is True: 

1061 try: 

1062 # Use the new-style cp_pipe version of the kernel if it exists 

1063 # If using a new-style kernel, always use the self-consistent 

1064 # gains, i.e. the ones inside the kernel object itself 

1065 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1066 brighterFatterGains = brighterFatterKernel.gain 

1067 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

1068 except NoResults: 

1069 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1070 brighterFatterKernel = dataRef.get("bfKernel") 

1071 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

1072 except NoResults: 

1073 brighterFatterKernel = None 

1074 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1075 # If the kernel is not an ndarray, it's the cp_pipe version 

1076 # so extract the kernel for this detector, or raise an error 

1077 if self.config.brighterFatterLevel == 'DETECTOR': 

1078 if brighterFatterKernel.detectorKernel: 

1079 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

1080 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

1081 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

1082 else: 

1083 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1084 else: 

1085 # TODO DM-15631 for implementing this 

1086 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1087 

1088 defectList = (dataRef.get("defects") 

1089 if self.config.doDefect else None) 

1090 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

1091 if self.config.doAssembleIsrExposures else None) 

1092 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1093 else pipeBase.Struct(fringes=None)) 

1094 

1095 if self.config.doAttachTransmissionCurve: 

1096 opticsTransmission = (dataRef.get("transmission_optics") 

1097 if self.config.doUseOpticsTransmission else None) 

1098 filterTransmission = (dataRef.get("transmission_filter") 

1099 if self.config.doUseFilterTransmission else None) 

1100 sensorTransmission = (dataRef.get("transmission_sensor") 

1101 if self.config.doUseSensorTransmission else None) 

1102 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1103 if self.config.doUseAtmosphereTransmission else None) 

1104 else: 

1105 opticsTransmission = None 

1106 filterTransmission = None 

1107 sensorTransmission = None 

1108 atmosphereTransmission = None 

1109 

1110 if self.config.doStrayLight: 

1111 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1112 else: 

1113 strayLightData = None 

1114 

1115 illumMaskedImage = (self.getIsrExposure(dataRef, 

1116 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1117 if (self.config.doIlluminationCorrection 

1118 and filterName in self.config.illumFilters) 

1119 else None) 

1120 

1121 # Struct should include only kwargs to run() 

1122 return pipeBase.Struct(bias=biasExposure, 

1123 linearizer=linearizer, 

1124 crosstalk=crosstalkCalib, 

1125 crosstalkSources=crosstalkSources, 

1126 dark=darkExposure, 

1127 flat=flatExposure, 

1128 bfKernel=brighterFatterKernel, 

1129 bfGains=brighterFatterGains, 

1130 defects=defectList, 

1131 fringes=fringeStruct, 

1132 opticsTransmission=opticsTransmission, 

1133 filterTransmission=filterTransmission, 

1134 sensorTransmission=sensorTransmission, 

1135 atmosphereTransmission=atmosphereTransmission, 

1136 strayLightData=strayLightData, 

1137 illumMaskedImage=illumMaskedImage 

1138 ) 

1139 

1140 @pipeBase.timeMethod 

1141 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, 

1142 crosstalk=None, crosstalkSources=None, 

1143 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1144 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1145 sensorTransmission=None, atmosphereTransmission=None, 

1146 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1147 isGen3=False, 

1148 ): 

1149 """!Perform instrument signature removal on an exposure. 

1150 

1151 Steps included in the ISR processing, in order performed, are: 

1152 - saturation and suspect pixel masking 

1153 - overscan subtraction 

1154 - CCD assembly of individual amplifiers 

1155 - bias subtraction 

1156 - variance image construction 

1157 - linearization of non-linear response 

1158 - crosstalk masking 

1159 - brighter-fatter correction 

1160 - dark subtraction 

1161 - fringe correction 

1162 - stray light subtraction 

1163 - flat correction 

1164 - masking of known defects and camera specific features 

1165 - vignette calculation 

1166 - appending transmission curve and distortion model 

1167 

1168 Parameters 

1169 ---------- 

1170 ccdExposure : `lsst.afw.image.Exposure` 

1171 The raw exposure that is to be run through ISR. The 

1172 exposure is modified by this method. 

1173 camera : `lsst.afw.cameraGeom.Camera`, optional 

1174 The camera geometry for this exposure. Required if ``isGen3`` is 

1175 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1176 ``flat`` does not have an associated detector. 

1177 bias : `lsst.afw.image.Exposure`, optional 

1178 Bias calibration frame. 

1179 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1180 Functor for linearization. 

1181 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1182 Calibration for crosstalk. 

1183 crosstalkSources : `list`, optional 

1184 List of possible crosstalk sources. 

1185 dark : `lsst.afw.image.Exposure`, optional 

1186 Dark calibration frame. 

1187 flat : `lsst.afw.image.Exposure`, optional 

1188 Flat calibration frame. 

1189 bfKernel : `numpy.ndarray`, optional 

1190 Brighter-fatter kernel. 

1191 bfGains : `dict` of `float`, optional 

1192 Gains used to override the detector's nominal gains for the 

1193 brighter-fatter correction. A dict keyed by amplifier name for 

1194 the detector in question. 

1195 defects : `lsst.meas.algorithms.Defects`, optional 

1196 List of defects. 

1197 fringes : `lsst.pipe.base.Struct`, optional 

1198 Struct containing the fringe correction data, with 

1199 elements: 

1200 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1201 - ``seed``: random seed derived from the ccdExposureId for random 

1202 number generator (`uint32`) 

1203 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1204 A ``TransmissionCurve`` that represents the throughput of the optics, 

1205 to be evaluated in focal-plane coordinates. 

1206 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1207 A ``TransmissionCurve`` that represents the throughput of the filter 

1208 itself, to be evaluated in focal-plane coordinates. 

1209 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1210 A ``TransmissionCurve`` that represents the throughput of the sensor 

1211 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1212 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1213 A ``TransmissionCurve`` that represents the throughput of the 

1214 atmosphere, assumed to be spatially constant. 

1215 detectorNum : `int`, optional 

1216 The integer number for the detector to process. 

1217 isGen3 : bool, optional 

1218 Flag this call to run() as using the Gen3 butler environment. 

1219 strayLightData : `object`, optional 

1220 Opaque object containing calibration information for stray-light 

1221 correction. If `None`, no correction will be performed. 

1222 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1223 Illumination correction image. 

1224 

1225 Returns 

1226 ------- 

1227 result : `lsst.pipe.base.Struct` 

1228 Result struct with component: 

1229 - ``exposure`` : `afw.image.Exposure` 

1230 The fully ISR corrected exposure. 

1231 - ``outputExposure`` : `afw.image.Exposure` 

1232 An alias for `exposure` 

1233 - ``ossThumb`` : `numpy.ndarray` 

1234 Thumbnail image of the exposure after overscan subtraction. 

1235 - ``flattenedThumb`` : `numpy.ndarray` 

1236 Thumbnail image of the exposure after flat-field correction. 

1237 

1238 Raises 

1239 ------ 

1240 RuntimeError 

1241 Raised if a configuration option is set to True, but the 

1242 required calibration data has not been specified. 

1243 

1244 Notes 

1245 ----- 

1246 The current processed exposure can be viewed by setting the 

1247 appropriate lsstDebug entries in the `debug.display` 

1248 dictionary. The names of these entries correspond to some of 

1249 the IsrTaskConfig Boolean options, with the value denoting the 

1250 frame to use. The exposure is shown inside the matching 

1251 option check and after the processing of that step has 

1252 finished. The steps with debug points are: 

1253 

1254 doAssembleCcd 

1255 doBias 

1256 doCrosstalk 

1257 doBrighterFatter 

1258 doDark 

1259 doFringe 

1260 doStrayLight 

1261 doFlat 

1262 

1263 In addition, setting the "postISRCCD" entry displays the 

1264 exposure after all ISR processing has finished. 

1265 

1266 """ 

1267 

1268 if isGen3 is True: 

1269 # Gen3 currently cannot automatically do configuration overrides. 

1270 # DM-15257 looks to discuss this issue. 

1271 # Configure input exposures; 

1272 if detectorNum is None: 

1273 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1274 

1275 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1276 bias = self.ensureExposure(bias, camera, detectorNum) 

1277 dark = self.ensureExposure(dark, camera, detectorNum) 

1278 flat = self.ensureExposure(flat, camera, detectorNum) 

1279 else: 

1280 if isinstance(ccdExposure, ButlerDataRef): 

1281 return self.runDataRef(ccdExposure) 

1282 

1283 ccd = ccdExposure.getDetector() 

1284 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1285 

1286 if not ccd: 

1287 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1288 ccd = [FakeAmp(ccdExposure, self.config)] 

1289 

1290 # Validate Input 

1291 if self.config.doBias and bias is None: 

1292 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1293 if self.doLinearize(ccd) and linearizer is None: 

1294 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1295 if self.config.doBrighterFatter and bfKernel is None: 

1296 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1297 if self.config.doDark and dark is None: 

1298 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1299 if self.config.doFlat and flat is None: 

1300 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1301 if self.config.doDefect and defects is None: 

1302 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1303 if (self.config.doFringe and filterName in self.fringe.config.filters 

1304 and fringes.fringes is None): 

1305 # The `fringes` object needs to be a pipeBase.Struct, as 

1306 # we use it as a `dict` for the parameters of 

1307 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1308 # not be `None` if `doFringe=True`. Otherwise, raise. 

1309 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1310 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters 

1311 and illumMaskedImage is None): 

1312 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1313 

1314 # Begin ISR processing. 

1315 if self.config.doConvertIntToFloat: 

1316 self.log.info("Converting exposure to floating point values.") 

1317 ccdExposure = self.convertIntToFloat(ccdExposure) 

1318 

1319 # Amplifier level processing. 

1320 overscans = [] 

1321 for amp in ccd: 

1322 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1323 if ccdExposure.getBBox().contains(amp.getBBox()): 

1324 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1325 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1326 

1327 if self.config.doOverscan and not badAmp: 

1328 # Overscan correction on amp-by-amp basis. 

1329 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1330 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1331 if overscanResults is not None and \ 

1332 self.config.qa is not None and self.config.qa.saveStats is True: 

1333 if isinstance(overscanResults.overscanFit, float): 

1334 qaMedian = overscanResults.overscanFit 

1335 qaStdev = float("NaN") 

1336 else: 

1337 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1338 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1339 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1340 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1341 

1342 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian) 

1343 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev) 

1344 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1345 amp.getName(), qaMedian, qaStdev) 

1346 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1347 else: 

1348 if badAmp: 

1349 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1350 overscanResults = None 

1351 

1352 overscans.append(overscanResults if overscanResults is not None else None) 

1353 else: 

1354 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1355 

1356 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1357 self.log.info("Applying crosstalk correction.") 

1358 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1359 crosstalkSources=crosstalkSources) 

1360 self.debugView(ccdExposure, "doCrosstalk") 

1361 

1362 if self.config.doAssembleCcd: 

1363 self.log.info("Assembling CCD from amplifiers.") 

1364 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1365 

1366 if self.config.expectWcs and not ccdExposure.getWcs(): 

1367 self.log.warn("No WCS found in input exposure.") 

1368 self.debugView(ccdExposure, "doAssembleCcd") 

1369 

1370 ossThumb = None 

1371 if self.config.qa.doThumbnailOss: 

1372 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1373 

1374 if self.config.doBias: 

1375 self.log.info("Applying bias correction.") 

1376 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1377 trimToFit=self.config.doTrimToMatchCalib) 

1378 self.debugView(ccdExposure, "doBias") 

1379 

1380 if self.config.doVariance: 

1381 for amp, overscanResults in zip(ccd, overscans): 

1382 if ccdExposure.getBBox().contains(amp.getBBox()): 

1383 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1384 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1385 if overscanResults is not None: 

1386 self.updateVariance(ampExposure, amp, 

1387 overscanImage=overscanResults.overscanImage) 

1388 else: 

1389 self.updateVariance(ampExposure, amp, 

1390 overscanImage=None) 

1391 if self.config.qa is not None and self.config.qa.saveStats is True: 

1392 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1393 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1394 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1395 qaStats.getValue(afwMath.MEDIAN)) 

1396 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1397 qaStats.getValue(afwMath.STDEVCLIP)) 

1398 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1399 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1400 qaStats.getValue(afwMath.STDEVCLIP)) 

1401 

1402 if self.doLinearize(ccd): 

1403 self.log.info("Applying linearizer.") 

1404 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1405 detector=ccd, log=self.log) 

1406 

1407 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1408 self.log.info("Applying crosstalk correction.") 

1409 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1410 crosstalkSources=crosstalkSources, isTrimmed=True) 

1411 self.debugView(ccdExposure, "doCrosstalk") 

1412 

1413 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1414 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1415 if self.config.doDefect: 

1416 self.log.info("Masking defects.") 

1417 self.maskDefect(ccdExposure, defects) 

1418 

1419 if self.config.numEdgeSuspect > 0: 

1420 self.log.info("Masking edges as SUSPECT.") 

1421 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1422 maskPlane="SUSPECT") 

1423 

1424 if self.config.doNanMasking: 

1425 self.log.info("Masking NAN value pixels.") 

1426 self.maskNan(ccdExposure) 

1427 

1428 if self.config.doWidenSaturationTrails: 

1429 self.log.info("Widening saturation trails.") 

1430 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1431 

1432 if self.config.doCameraSpecificMasking: 

1433 self.log.info("Masking regions for camera specific reasons.") 

1434 self.masking.run(ccdExposure) 

1435 

1436 if self.config.doBrighterFatter: 

1437 # We need to apply flats and darks before we can interpolate, and we 

1438 # need to interpolate before we do B-F, but we do B-F without the 

1439 # flats and darks applied so we can work in units of electrons or holes. 

1440 # This context manager applies and then removes the darks and flats. 

1441 # 

1442 # We also do not want to interpolate values here, so operate on temporary 

1443 # images so we can apply only the BF-correction and roll back the 

1444 # interpolation. 

1445 interpExp = ccdExposure.clone() 

1446 with self.flatContext(interpExp, flat, dark): 

1447 isrFunctions.interpolateFromMask( 

1448 maskedImage=interpExp.getMaskedImage(), 

1449 fwhm=self.config.fwhm, 

1450 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1451 maskNameList=self.config.maskListToInterpolate 

1452 ) 

1453 bfExp = interpExp.clone() 

1454 

1455 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1456 type(bfKernel), type(bfGains)) 

1457 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1458 self.config.brighterFatterMaxIter, 

1459 self.config.brighterFatterThreshold, 

1460 self.config.brighterFatterApplyGain, 

1461 bfGains) 

1462 if bfResults[1] == self.config.brighterFatterMaxIter: 

1463 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1464 bfResults[0]) 

1465 else: 

1466 self.log.info("Finished brighter fatter correction in %d iterations.", 

1467 bfResults[1]) 

1468 image = ccdExposure.getMaskedImage().getImage() 

1469 bfCorr = bfExp.getMaskedImage().getImage() 

1470 bfCorr -= interpExp.getMaskedImage().getImage() 

1471 image += bfCorr 

1472 

1473 # Applying the brighter-fatter correction applies a 

1474 # convolution to the science image. At the edges this 

1475 # convolution may not have sufficient valid pixels to 

1476 # produce a valid correction. Mark pixels within the size 

1477 # of the brighter-fatter kernel as EDGE to warn of this 

1478 # fact. 

1479 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1480 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1481 maskPlane="EDGE") 

1482 

1483 if self.config.brighterFatterMaskGrowSize > 0: 

1484 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1485 for maskPlane in self.config.maskListToInterpolate: 

1486 isrFunctions.growMasks(ccdExposure.getMask(), 

1487 radius=self.config.brighterFatterMaskGrowSize, 

1488 maskNameList=maskPlane, 

1489 maskValue=maskPlane) 

1490 

1491 self.debugView(ccdExposure, "doBrighterFatter") 

1492 

1493 if self.config.doDark: 

1494 self.log.info("Applying dark correction.") 

1495 self.darkCorrection(ccdExposure, dark) 

1496 self.debugView(ccdExposure, "doDark") 

1497 

1498 if self.config.doFringe and not self.config.fringeAfterFlat: 

1499 self.log.info("Applying fringe correction before flat.") 

1500 self.fringe.run(ccdExposure, **fringes.getDict()) 

1501 self.debugView(ccdExposure, "doFringe") 

1502 

1503 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1504 self.log.info("Checking strayLight correction.") 

1505 self.strayLight.run(ccdExposure, strayLightData) 

1506 self.debugView(ccdExposure, "doStrayLight") 

1507 

1508 if self.config.doFlat: 

1509 self.log.info("Applying flat correction.") 

1510 self.flatCorrection(ccdExposure, flat) 

1511 self.debugView(ccdExposure, "doFlat") 

1512 

1513 if self.config.doApplyGains: 

1514 self.log.info("Applying gain correction instead of flat.") 

1515 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1516 

1517 if self.config.doFringe and self.config.fringeAfterFlat: 

1518 self.log.info("Applying fringe correction after flat.") 

1519 self.fringe.run(ccdExposure, **fringes.getDict()) 

1520 

1521 if self.config.doVignette: 

1522 self.log.info("Constructing Vignette polygon.") 

1523 self.vignettePolygon = self.vignette.run(ccdExposure) 

1524 

1525 if self.config.vignette.doWriteVignettePolygon: 

1526 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1527 

1528 if self.config.doAttachTransmissionCurve: 

1529 self.log.info("Adding transmission curves.") 

1530 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1531 filterTransmission=filterTransmission, 

1532 sensorTransmission=sensorTransmission, 

1533 atmosphereTransmission=atmosphereTransmission) 

1534 

1535 flattenedThumb = None 

1536 if self.config.qa.doThumbnailFlattened: 

1537 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1538 

1539 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1540 self.log.info("Performing illumination correction.") 

1541 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1542 illumMaskedImage, illumScale=self.config.illumScale, 

1543 trimToFit=self.config.doTrimToMatchCalib) 

1544 

1545 preInterpExp = None 

1546 if self.config.doSaveInterpPixels: 

1547 preInterpExp = ccdExposure.clone() 

1548 

1549 # Reset and interpolate bad pixels. 

1550 # 

1551 # Large contiguous bad regions (which should have the BAD mask 

1552 # bit set) should have their values set to the image median. 

1553 # This group should include defects and bad amplifiers. As the 

1554 # area covered by these defects are large, there's little 

1555 # reason to expect that interpolation would provide a more 

1556 # useful value. 

1557 # 

1558 # Smaller defects can be safely interpolated after the larger 

1559 # regions have had their pixel values reset. This ensures 

1560 # that the remaining defects adjacent to bad amplifiers (as an 

1561 # example) do not attempt to interpolate extreme values. 

1562 if self.config.doSetBadRegions: 

1563 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1564 if badPixelCount > 0: 

1565 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1566 

1567 if self.config.doInterpolate: 

1568 self.log.info("Interpolating masked pixels.") 

1569 isrFunctions.interpolateFromMask( 

1570 maskedImage=ccdExposure.getMaskedImage(), 

1571 fwhm=self.config.fwhm, 

1572 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1573 maskNameList=list(self.config.maskListToInterpolate) 

1574 ) 

1575 

1576 self.roughZeroPoint(ccdExposure) 

1577 

1578 if self.config.doMeasureBackground: 

1579 self.log.info("Measuring background level.") 

1580 self.measureBackground(ccdExposure, self.config.qa) 

1581 

1582 if self.config.qa is not None and self.config.qa.saveStats is True: 

1583 for amp in ccd: 

1584 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1585 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1586 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1587 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1588 qaStats.getValue(afwMath.MEDIAN)) 

1589 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1590 qaStats.getValue(afwMath.STDEVCLIP)) 

1591 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1592 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1593 qaStats.getValue(afwMath.STDEVCLIP)) 

1594 

1595 self.debugView(ccdExposure, "postISRCCD") 

1596 

1597 return pipeBase.Struct( 

1598 exposure=ccdExposure, 

1599 ossThumb=ossThumb, 

1600 flattenedThumb=flattenedThumb, 

1601 

1602 preInterpolatedExposure=preInterpExp, 

1603 outputExposure=ccdExposure, 

1604 outputOssThumbnail=ossThumb, 

1605 outputFlattenedThumbnail=flattenedThumb, 

1606 ) 

1607 

1608 @pipeBase.timeMethod 

1609 def runDataRef(self, sensorRef): 

1610 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1611 

1612 This method contains the `CmdLineTask` interface to the ISR 

1613 processing. All IO is handled here, freeing the `run()` method 

1614 to manage only pixel-level calculations. The steps performed 

1615 are: 

1616 - Read in necessary detrending/isr/calibration data. 

1617 - Process raw exposure in `run()`. 

1618 - Persist the ISR-corrected exposure as "postISRCCD" if 

1619 config.doWrite=True. 

1620 

1621 Parameters 

1622 ---------- 

1623 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1624 DataRef of the detector data to be processed 

1625 

1626 Returns 

1627 ------- 

1628 result : `lsst.pipe.base.Struct` 

1629 Result struct with component: 

1630 - ``exposure`` : `afw.image.Exposure` 

1631 The fully ISR corrected exposure. 

1632 

1633 Raises 

1634 ------ 

1635 RuntimeError 

1636 Raised if a configuration option is set to True, but the 

1637 required calibration data does not exist. 

1638 

1639 """ 

1640 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1641 

1642 ccdExposure = sensorRef.get(self.config.datasetType) 

1643 

1644 camera = sensorRef.get("camera") 

1645 isrData = self.readIsrData(sensorRef, ccdExposure) 

1646 

1647 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1648 

1649 if self.config.doWrite: 

1650 sensorRef.put(result.exposure, "postISRCCD") 

1651 if result.preInterpolatedExposure is not None: 

1652 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1653 if result.ossThumb is not None: 

1654 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1655 if result.flattenedThumb is not None: 

1656 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1657 

1658 return result 

1659 

1660 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1661 """!Retrieve a calibration dataset for removing instrument signature. 

1662 

1663 Parameters 

1664 ---------- 

1665 

1666 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1667 DataRef of the detector data to find calibration datasets 

1668 for. 

1669 datasetType : `str` 

1670 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1671 dateObs : `str`, optional 

1672 Date of the observation. Used to correct butler failures 

1673 when using fallback filters. 

1674 immediate : `Bool` 

1675 If True, disable butler proxies to enable error handling 

1676 within this routine. 

1677 

1678 Returns 

1679 ------- 

1680 exposure : `lsst.afw.image.Exposure` 

1681 Requested calibration frame. 

1682 

1683 Raises 

1684 ------ 

1685 RuntimeError 

1686 Raised if no matching calibration frame can be found. 

1687 """ 

1688 try: 

1689 exp = dataRef.get(datasetType, immediate=immediate) 

1690 except Exception as exc1: 

1691 if not self.config.fallbackFilterName: 

1692 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1693 try: 

1694 if self.config.useFallbackDate and dateObs: 

1695 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1696 dateObs=dateObs, immediate=immediate) 

1697 else: 

1698 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1699 except Exception as exc2: 

1700 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1701 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1702 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1703 

1704 if self.config.doAssembleIsrExposures: 

1705 exp = self.assembleCcd.assembleCcd(exp) 

1706 return exp 

1707 

1708 def ensureExposure(self, inputExp, camera, detectorNum): 

1709 """Ensure that the data returned by Butler is a fully constructed exposure. 

1710 

1711 ISR requires exposure-level image data for historical reasons, so if we did 

1712 not recieve that from Butler, construct it from what we have, modifying the 

1713 input in place. 

1714 

1715 Parameters 

1716 ---------- 

1717 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1718 `lsst.afw.image.ImageF` 

1719 The input data structure obtained from Butler. 

1720 camera : `lsst.afw.cameraGeom.camera` 

1721 The camera associated with the image. Used to find the appropriate 

1722 detector. 

1723 detectorNum : `int` 

1724 The detector this exposure should match. 

1725 

1726 Returns 

1727 ------- 

1728 inputExp : `lsst.afw.image.Exposure` 

1729 The re-constructed exposure, with appropriate detector parameters. 

1730 

1731 Raises 

1732 ------ 

1733 TypeError 

1734 Raised if the input data cannot be used to construct an exposure. 

1735 """ 

1736 if isinstance(inputExp, afwImage.DecoratedImageU): 

1737 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1738 elif isinstance(inputExp, afwImage.ImageF): 

1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1740 elif isinstance(inputExp, afwImage.MaskedImageF): 

1741 inputExp = afwImage.makeExposure(inputExp) 

1742 elif isinstance(inputExp, afwImage.Exposure): 

1743 pass 

1744 elif inputExp is None: 

1745 # Assume this will be caught by the setup if it is a problem. 

1746 return inputExp 

1747 else: 

1748 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1749 (type(inputExp), )) 

1750 

1751 if inputExp.getDetector() is None: 

1752 inputExp.setDetector(camera[detectorNum]) 

1753 

1754 return inputExp 

1755 

1756 def convertIntToFloat(self, exposure): 

1757 """Convert exposure image from uint16 to float. 

1758 

1759 If the exposure does not need to be converted, the input is 

1760 immediately returned. For exposures that are converted to use 

1761 floating point pixels, the variance is set to unity and the 

1762 mask to zero. 

1763 

1764 Parameters 

1765 ---------- 

1766 exposure : `lsst.afw.image.Exposure` 

1767 The raw exposure to be converted. 

1768 

1769 Returns 

1770 ------- 

1771 newexposure : `lsst.afw.image.Exposure` 

1772 The input ``exposure``, converted to floating point pixels. 

1773 

1774 Raises 

1775 ------ 

1776 RuntimeError 

1777 Raised if the exposure type cannot be converted to float. 

1778 

1779 """ 

1780 if isinstance(exposure, afwImage.ExposureF): 

1781 # Nothing to be done 

1782 self.log.debug("Exposure already of type float.") 

1783 return exposure 

1784 if not hasattr(exposure, "convertF"): 

1785 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1786 

1787 newexposure = exposure.convertF() 

1788 newexposure.variance[:] = 1 

1789 newexposure.mask[:] = 0x0 

1790 

1791 return newexposure 

1792 

1793 def maskAmplifier(self, ccdExposure, amp, defects): 

1794 """Identify bad amplifiers, saturated and suspect pixels. 

1795 

1796 Parameters 

1797 ---------- 

1798 ccdExposure : `lsst.afw.image.Exposure` 

1799 Input exposure to be masked. 

1800 amp : `lsst.afw.table.AmpInfoCatalog` 

1801 Catalog of parameters defining the amplifier on this 

1802 exposure to mask. 

1803 defects : `lsst.meas.algorithms.Defects` 

1804 List of defects. Used to determine if the entire 

1805 amplifier is bad. 

1806 

1807 Returns 

1808 ------- 

1809 badAmp : `Bool` 

1810 If this is true, the entire amplifier area is covered by 

1811 defects and unusable. 

1812 

1813 """ 

1814 maskedImage = ccdExposure.getMaskedImage() 

1815 

1816 badAmp = False 

1817 

1818 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1819 # comparison with current defects definition. 

1820 if defects is not None: 

1821 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1822 

1823 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1824 # association with pixels in current ccdExposure). 

1825 if badAmp: 

1826 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1827 afwImage.PARENT) 

1828 maskView = dataView.getMask() 

1829 maskView |= maskView.getPlaneBitMask("BAD") 

1830 del maskView 

1831 return badAmp 

1832 

1833 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1834 # Saturation and suspect pixels can be masked now, though. 

1835 limits = dict() 

1836 if self.config.doSaturation and not badAmp: 

1837 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1838 if self.config.doSuspect and not badAmp: 

1839 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1840 if math.isfinite(self.config.saturation): 

1841 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1842 

1843 for maskName, maskThreshold in limits.items(): 

1844 if not math.isnan(maskThreshold): 

1845 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1846 isrFunctions.makeThresholdMask( 

1847 maskedImage=dataView, 

1848 threshold=maskThreshold, 

1849 growFootprints=0, 

1850 maskName=maskName 

1851 ) 

1852 

1853 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1854 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1855 afwImage.PARENT) 

1856 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1857 self.config.suspectMaskName]) 

1858 if numpy.all(maskView.getArray() & maskVal > 0): 

1859 badAmp = True 

1860 maskView |= maskView.getPlaneBitMask("BAD") 

1861 

1862 return badAmp 

1863 

1864 def overscanCorrection(self, ccdExposure, amp): 

1865 """Apply overscan correction in place. 

1866 

1867 This method does initial pixel rejection of the overscan 

1868 region. The overscan can also be optionally segmented to 

1869 allow for discontinuous overscan responses to be fit 

1870 separately. The actual overscan subtraction is performed by 

1871 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1872 which is called here after the amplifier is preprocessed. 

1873 

1874 Parameters 

1875 ---------- 

1876 ccdExposure : `lsst.afw.image.Exposure` 

1877 Exposure to have overscan correction performed. 

1878 amp : `lsst.afw.table.AmpInfoCatalog` 

1879 The amplifier to consider while correcting the overscan. 

1880 

1881 Returns 

1882 ------- 

1883 overscanResults : `lsst.pipe.base.Struct` 

1884 Result struct with components: 

1885 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1886 Value or fit subtracted from the amplifier image data. 

1887 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1888 Value or fit subtracted from the overscan image data. 

1889 - ``overscanImage`` : `lsst.afw.image.Image` 

1890 Image of the overscan region with the overscan 

1891 correction applied. This quantity is used to estimate 

1892 the amplifier read noise empirically. 

1893 

1894 Raises 

1895 ------ 

1896 RuntimeError 

1897 Raised if the ``amp`` does not contain raw pixel information. 

1898 

1899 See Also 

1900 -------- 

1901 lsst.ip.isr.isrFunctions.overscanCorrection 

1902 """ 

1903 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1904 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1905 return None 

1906 

1907 statControl = afwMath.StatisticsControl() 

1908 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1909 

1910 # Determine the bounding boxes 

1911 dataBBox = amp.getRawDataBBox() 

1912 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1913 dx0 = 0 

1914 dx1 = 0 

1915 

1916 prescanBBox = amp.getRawPrescanBBox() 

1917 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1918 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1919 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1920 else: 

1921 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1922 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1923 

1924 # Determine if we need to work on subregions of the amplifier and overscan. 

1925 imageBBoxes = [] 

1926 overscanBBoxes = [] 

1927 

1928 if ((self.config.overscanBiasJump 

1929 and self.config.overscanBiasJumpLocation) 

1930 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

1931 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1932 self.config.overscanBiasJumpDevices)): 

1933 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1934 yLower = self.config.overscanBiasJumpLocation 

1935 yUpper = dataBBox.getHeight() - yLower 

1936 else: 

1937 yUpper = self.config.overscanBiasJumpLocation 

1938 yLower = dataBBox.getHeight() - yUpper 

1939 

1940 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1941 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1942 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1943 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1944 yLower))) 

1945 

1946 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1947 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1948 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1949 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1950 yUpper))) 

1951 else: 

1952 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1953 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1954 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1955 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1956 oscanBBox.getHeight()))) 

1957 

1958 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

1959 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1960 ampImage = ccdExposure.maskedImage[imageBBox] 

1961 overscanImage = ccdExposure.maskedImage[overscanBBox] 

1962 

1963 overscanArray = overscanImage.image.array 

1964 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

1965 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

1966 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

1967 

1968 statControl = afwMath.StatisticsControl() 

1969 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1970 

1971 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage) 

1972 

1973 # Measure average overscan levels and record them in the metadata. 

1974 levelStat = afwMath.MEDIAN 

1975 sigmaStat = afwMath.STDEVCLIP 

1976 

1977 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

1978 self.config.qa.flatness.nIter) 

1979 metadata = ccdExposure.getMetadata() 

1980 ampNum = amp.getName() 

1981 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

1982 if isinstance(overscanResults.overscanFit, float): 

1983 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

1984 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

1985 else: 

1986 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

1987 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

1988 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

1989 

1990 return overscanResults 

1991 

1992 def updateVariance(self, ampExposure, amp, overscanImage=None): 

1993 """Set the variance plane using the amplifier gain and read noise 

1994 

1995 The read noise is calculated from the ``overscanImage`` if the 

1996 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

1997 the value from the amplifier data is used. 

1998 

1999 Parameters 

2000 ---------- 

2001 ampExposure : `lsst.afw.image.Exposure` 

2002 Exposure to process. 

2003 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2004 Amplifier detector data. 

2005 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2006 Image of overscan, required only for empirical read noise. 

2007 

2008 See also 

2009 -------- 

2010 lsst.ip.isr.isrFunctions.updateVariance 

2011 """ 

2012 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2013 gain = amp.getGain() 

2014 

2015 if math.isnan(gain): 

2016 gain = 1.0 

2017 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2018 elif gain <= 0: 

2019 patchedGain = 1.0 

2020 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

2021 amp.getName(), gain, patchedGain) 

2022 gain = patchedGain 

2023 

2024 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2025 self.log.info("Overscan is none for EmpiricalReadNoise.") 

2026 

2027 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2028 stats = afwMath.StatisticsControl() 

2029 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2030 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2031 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2032 amp.getName(), readNoise) 

2033 else: 

2034 readNoise = amp.getReadNoise() 

2035 

2036 isrFunctions.updateVariance( 

2037 maskedImage=ampExposure.getMaskedImage(), 

2038 gain=gain, 

2039 readNoise=readNoise, 

2040 ) 

2041 

2042 def darkCorrection(self, exposure, darkExposure, invert=False): 

2043 """!Apply dark correction in place. 

2044 

2045 Parameters 

2046 ---------- 

2047 exposure : `lsst.afw.image.Exposure` 

2048 Exposure to process. 

2049 darkExposure : `lsst.afw.image.Exposure` 

2050 Dark exposure of the same size as ``exposure``. 

2051 invert : `Bool`, optional 

2052 If True, re-add the dark to an already corrected image. 

2053 

2054 Raises 

2055 ------ 

2056 RuntimeError 

2057 Raised if either ``exposure`` or ``darkExposure`` do not 

2058 have their dark time defined. 

2059 

2060 See Also 

2061 -------- 

2062 lsst.ip.isr.isrFunctions.darkCorrection 

2063 """ 

2064 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2065 if math.isnan(expScale): 

2066 raise RuntimeError("Exposure darktime is NAN.") 

2067 if darkExposure.getInfo().getVisitInfo() is not None \ 

2068 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2069 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2070 else: 

2071 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2072 # so getDarkTime() does not exist. 

2073 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2074 darkScale = 1.0 

2075 

2076 isrFunctions.darkCorrection( 

2077 maskedImage=exposure.getMaskedImage(), 

2078 darkMaskedImage=darkExposure.getMaskedImage(), 

2079 expScale=expScale, 

2080 darkScale=darkScale, 

2081 invert=invert, 

2082 trimToFit=self.config.doTrimToMatchCalib 

2083 ) 

2084 

2085 def doLinearize(self, detector): 

2086 """!Check if linearization is needed for the detector cameraGeom. 

2087 

2088 Checks config.doLinearize and the linearity type of the first 

2089 amplifier. 

2090 

2091 Parameters 

2092 ---------- 

2093 detector : `lsst.afw.cameraGeom.Detector` 

2094 Detector to get linearity type from. 

2095 

2096 Returns 

2097 ------- 

2098 doLinearize : `Bool` 

2099 If True, linearization should be performed. 

2100 """ 

2101 return self.config.doLinearize and \ 

2102 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2103 

2104 def flatCorrection(self, exposure, flatExposure, invert=False): 

2105 """!Apply flat correction in place. 

2106 

2107 Parameters 

2108 ---------- 

2109 exposure : `lsst.afw.image.Exposure` 

2110 Exposure to process. 

2111 flatExposure : `lsst.afw.image.Exposure` 

2112 Flat exposure of the same size as ``exposure``. 

2113 invert : `Bool`, optional 

2114 If True, unflatten an already flattened image. 

2115 

2116 See Also 

2117 -------- 

2118 lsst.ip.isr.isrFunctions.flatCorrection 

2119 """ 

2120 isrFunctions.flatCorrection( 

2121 maskedImage=exposure.getMaskedImage(), 

2122 flatMaskedImage=flatExposure.getMaskedImage(), 

2123 scalingType=self.config.flatScalingType, 

2124 userScale=self.config.flatUserScale, 

2125 invert=invert, 

2126 trimToFit=self.config.doTrimToMatchCalib 

2127 ) 

2128 

2129 def saturationDetection(self, exposure, amp): 

2130 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2131 

2132 Parameters 

2133 ---------- 

2134 exposure : `lsst.afw.image.Exposure` 

2135 Exposure to process. Only the amplifier DataSec is processed. 

2136 amp : `lsst.afw.table.AmpInfoCatalog` 

2137 Amplifier detector data. 

2138 

2139 See Also 

2140 -------- 

2141 lsst.ip.isr.isrFunctions.makeThresholdMask 

2142 """ 

2143 if not math.isnan(amp.getSaturation()): 

2144 maskedImage = exposure.getMaskedImage() 

2145 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2146 isrFunctions.makeThresholdMask( 

2147 maskedImage=dataView, 

2148 threshold=amp.getSaturation(), 

2149 growFootprints=0, 

2150 maskName=self.config.saturatedMaskName, 

2151 ) 

2152 

2153 def saturationInterpolation(self, exposure): 

2154 """!Interpolate over saturated pixels, in place. 

2155 

2156 This method should be called after `saturationDetection`, to 

2157 ensure that the saturated pixels have been identified in the 

2158 SAT mask. It should also be called after `assembleCcd`, since 

2159 saturated regions may cross amplifier boundaries. 

2160 

2161 Parameters 

2162 ---------- 

2163 exposure : `lsst.afw.image.Exposure` 

2164 Exposure to process. 

2165 

2166 See Also 

2167 -------- 

2168 lsst.ip.isr.isrTask.saturationDetection 

2169 lsst.ip.isr.isrFunctions.interpolateFromMask 

2170 """ 

2171 isrFunctions.interpolateFromMask( 

2172 maskedImage=exposure.getMaskedImage(), 

2173 fwhm=self.config.fwhm, 

2174 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2175 maskNameList=list(self.config.saturatedMaskName), 

2176 ) 

2177 

2178 def suspectDetection(self, exposure, amp): 

2179 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2180 

2181 Parameters 

2182 ---------- 

2183 exposure : `lsst.afw.image.Exposure` 

2184 Exposure to process. Only the amplifier DataSec is processed. 

2185 amp : `lsst.afw.table.AmpInfoCatalog` 

2186 Amplifier detector data. 

2187 

2188 See Also 

2189 -------- 

2190 lsst.ip.isr.isrFunctions.makeThresholdMask 

2191 

2192 Notes 

2193 ----- 

2194 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2195 This is intended to indicate pixels that may be affected by unknown systematics; 

2196 for example if non-linearity corrections above a certain level are unstable 

2197 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2198 that no such level exists and no pixels are to be masked as suspicious. 

2199 """ 

2200 suspectLevel = amp.getSuspectLevel() 

2201 if math.isnan(suspectLevel): 

2202 return 

2203 

2204 maskedImage = exposure.getMaskedImage() 

2205 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2206 isrFunctions.makeThresholdMask( 

2207 maskedImage=dataView, 

2208 threshold=suspectLevel, 

2209 growFootprints=0, 

2210 maskName=self.config.suspectMaskName, 

2211 ) 

2212 

2213 def maskDefect(self, exposure, defectBaseList): 

2214 """!Mask defects using mask plane "BAD", in place. 

2215 

2216 Parameters 

2217 ---------- 

2218 exposure : `lsst.afw.image.Exposure` 

2219 Exposure to process. 

2220 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2221 `lsst.afw.image.DefectBase`. 

2222 List of defects to mask. 

2223 

2224 Notes 

2225 ----- 

2226 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2227 """ 

2228 maskedImage = exposure.getMaskedImage() 

2229 if not isinstance(defectBaseList, Defects): 

2230 # Promotes DefectBase to Defect 

2231 defectList = Defects(defectBaseList) 

2232 else: 

2233 defectList = defectBaseList 

2234 defectList.maskPixels(maskedImage, maskName="BAD") 

2235 

2236 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"): 

2237 """!Mask edge pixels with applicable mask plane. 

2238 

2239 Parameters 

2240 ---------- 

2241 exposure : `lsst.afw.image.Exposure` 

2242 Exposure to process. 

2243 numEdgePixels : `int`, optional 

2244 Number of edge pixels to mask. 

2245 maskPlane : `str`, optional 

2246 Mask plane name to use. 

2247 """ 

2248 maskedImage = exposure.getMaskedImage() 

2249 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2250 

2251 if numEdgePixels > 0: 

2252 goodBBox = maskedImage.getBBox() 

2253 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2254 goodBBox.grow(-numEdgePixels) 

2255 # Mask pixels outside goodBBox 

2256 SourceDetectionTask.setEdgeBits( 

2257 maskedImage, 

2258 goodBBox, 

2259 maskBitMask 

2260 ) 

2261 

2262 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2263 """Mask and interpolate defects using mask plane "BAD", in place. 

2264 

2265 Parameters 

2266 ---------- 

2267 exposure : `lsst.afw.image.Exposure` 

2268 Exposure to process. 

2269 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2270 `lsst.afw.image.DefectBase`. 

2271 List of defects to mask and interpolate. 

2272 

2273 See Also 

2274 -------- 

2275 lsst.ip.isr.isrTask.maskDefect() 

2276 """ 

2277 self.maskDefect(exposure, defectBaseList) 

2278 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2279 maskPlane="SUSPECT") 

2280 isrFunctions.interpolateFromMask( 

2281 maskedImage=exposure.getMaskedImage(), 

2282 fwhm=self.config.fwhm, 

2283 growSaturatedFootprints=0, 

2284 maskNameList=["BAD"], 

2285 ) 

2286 

2287 def maskNan(self, exposure): 

2288 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2289 

2290 Parameters 

2291 ---------- 

2292 exposure : `lsst.afw.image.Exposure` 

2293 Exposure to process. 

2294 

2295 Notes 

2296 ----- 

2297 We mask over all NaNs, including those that are masked with 

2298 other bits (because those may or may not be interpolated over 

2299 later, and we want to remove all NaNs). Despite this 

2300 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2301 the historical name. 

2302 """ 

2303 maskedImage = exposure.getMaskedImage() 

2304 

2305 # Find and mask NaNs 

2306 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2307 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2308 numNans = maskNans(maskedImage, maskVal) 

2309 self.metadata.set("NUMNANS", numNans) 

2310 if numNans > 0: 

2311 self.log.warn("There were %d unmasked NaNs.", numNans) 

2312 

2313 def maskAndInterpolateNan(self, exposure): 

2314 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2315 

2316 Parameters 

2317 ---------- 

2318 exposure : `lsst.afw.image.Exposure` 

2319 Exposure to process. 

2320 

2321 See Also 

2322 -------- 

2323 lsst.ip.isr.isrTask.maskNan() 

2324 """ 

2325 self.maskNan(exposure) 

2326 isrFunctions.interpolateFromMask( 

2327 maskedImage=exposure.getMaskedImage(), 

2328 fwhm=self.config.fwhm, 

2329 growSaturatedFootprints=0, 

2330 maskNameList=["UNMASKEDNAN"], 

2331 ) 

2332 

2333 def measureBackground(self, exposure, IsrQaConfig=None): 

2334 """Measure the image background in subgrids, for quality control purposes. 

2335 

2336 Parameters 

2337 ---------- 

2338 exposure : `lsst.afw.image.Exposure` 

2339 Exposure to process. 

2340 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2341 Configuration object containing parameters on which background 

2342 statistics and subgrids to use. 

2343 """ 

2344 if IsrQaConfig is not None: 

2345 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2346 IsrQaConfig.flatness.nIter) 

2347 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2348 statsControl.setAndMask(maskVal) 

2349 maskedImage = exposure.getMaskedImage() 

2350 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2351 skyLevel = stats.getValue(afwMath.MEDIAN) 

2352 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2353 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2354 metadata = exposure.getMetadata() 

2355 metadata.set('SKYLEVEL', skyLevel) 

2356 metadata.set('SKYSIGMA', skySigma) 

2357 

2358 # calcluating flatlevel over the subgrids 

2359 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2360 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2361 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2362 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2363 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2364 skyLevels = numpy.zeros((nX, nY)) 

2365 

2366 for j in range(nY): 

2367 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2368 for i in range(nX): 

2369 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2370 

2371 xLLC = xc - meshXHalf 

2372 yLLC = yc - meshYHalf 

2373 xURC = xc + meshXHalf - 1 

2374 yURC = yc + meshYHalf - 1 

2375 

2376 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2377 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2378 

2379 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2380 

2381 good = numpy.where(numpy.isfinite(skyLevels)) 

2382 skyMedian = numpy.median(skyLevels[good]) 

2383 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2384 flatness_rms = numpy.std(flatness) 

2385 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2386 

2387 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2388 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2389 nX, nY, flatness_pp, flatness_rms) 

2390 

2391 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2392 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2393 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2394 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2395 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2396 

2397 def roughZeroPoint(self, exposure): 

2398 """Set an approximate magnitude zero point for the exposure. 

2399 

2400 Parameters 

2401 ---------- 

2402 exposure : `lsst.afw.image.Exposure` 

2403 Exposure to process. 

2404 """ 

2405 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2406 if filterName in self.config.fluxMag0T1: 

2407 fluxMag0 = self.config.fluxMag0T1[filterName] 

2408 else: 

2409 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2410 fluxMag0 = self.config.defaultFluxMag0T1 

2411 

2412 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2413 if not expTime > 0: # handle NaN as well as <= 0 

2414 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2415 return 

2416 

2417 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2418 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2419 

2420 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2421 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2422 

2423 Parameters 

2424 ---------- 

2425 ccdExposure : `lsst.afw.image.Exposure` 

2426 Exposure to process. 

2427 fpPolygon : `lsst.afw.geom.Polygon` 

2428 Polygon in focal plane coordinates. 

2429 """ 

2430 # Get ccd corners in focal plane coordinates 

2431 ccd = ccdExposure.getDetector() 

2432 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2433 ccdPolygon = Polygon(fpCorners) 

2434 

2435 # Get intersection of ccd corners with fpPolygon 

2436 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2437 

2438 # Transform back to pixel positions and build new polygon 

2439 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2440 validPolygon = Polygon(ccdPoints) 

2441 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2442 

2443 @contextmanager 

2444 def flatContext(self, exp, flat, dark=None): 

2445 """Context manager that applies and removes flats and darks, 

2446 if the task is configured to apply them. 

2447 

2448 Parameters 

2449 ---------- 

2450 exp : `lsst.afw.image.Exposure` 

2451 Exposure to process. 

2452 flat : `lsst.afw.image.Exposure` 

2453 Flat exposure the same size as ``exp``. 

2454 dark : `lsst.afw.image.Exposure`, optional 

2455 Dark exposure the same size as ``exp``. 

2456 

2457 Yields 

2458 ------ 

2459 exp : `lsst.afw.image.Exposure` 

2460 The flat and dark corrected exposure. 

2461 """ 

2462 if self.config.doDark and dark is not None: 

2463 self.darkCorrection(exp, dark) 

2464 if self.config.doFlat: 

2465 self.flatCorrection(exp, flat) 

2466 try: 

2467 yield exp 

2468 finally: 

2469 if self.config.doFlat: 

2470 self.flatCorrection(exp, flat, invert=True) 

2471 if self.config.doDark and dark is not None: 

2472 self.darkCorrection(exp, dark, invert=True) 

2473 

2474 def debugView(self, exposure, stepname): 

2475 """Utility function to examine ISR exposure at different stages. 

2476 

2477 Parameters 

2478 ---------- 

2479 exposure : `lsst.afw.image.Exposure` 

2480 Exposure to view. 

2481 stepname : `str` 

2482 State of processing to view. 

2483 """ 

2484 frame = getDebugFrame(self._display, stepname) 

2485 if frame: 

2486 display = getDisplay(frame) 

2487 display.scale('asinh', 'zscale') 

2488 display.mtv(exposure) 

2489 prompt = "Press Enter to continue [c]... " 

2490 while True: 

2491 ans = input(prompt).lower() 

2492 if ans in ("", "c",): 

2493 break 

2494 

2495 

2496class FakeAmp(object): 

2497 """A Detector-like object that supports returning gain and saturation level 

2498 

2499 This is used when the input exposure does not have a detector. 

2500 

2501 Parameters 

2502 ---------- 

2503 exposure : `lsst.afw.image.Exposure` 

2504 Exposure to generate a fake amplifier for. 

2505 config : `lsst.ip.isr.isrTaskConfig` 

2506 Configuration to apply to the fake amplifier. 

2507 """ 

2508 

2509 def __init__(self, exposure, config): 

2510 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2511 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2512 self._gain = config.gain 

2513 self._readNoise = config.readNoise 

2514 self._saturation = config.saturation 

2515 

2516 def getBBox(self): 

2517 return self._bbox 

2518 

2519 def getRawBBox(self): 

2520 return self._bbox 

2521 

2522 def getRawHorizontalOverscanBBox(self): 

2523 return self._RawHorizontalOverscanBBox 

2524 

2525 def getGain(self): 

2526 return self._gain 

2527 

2528 def getReadNoise(self): 

2529 return self._readNoise 

2530 

2531 def getSaturation(self): 

2532 return self._saturation 

2533 

2534 def getSuspectLevel(self): 

2535 return float("NaN") 

2536 

2537 

2538class RunIsrConfig(pexConfig.Config): 

2539 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2540 

2541 

2542class RunIsrTask(pipeBase.CmdLineTask): 

2543 """Task to wrap the default IsrTask to allow it to be retargeted. 

2544 

2545 The standard IsrTask can be called directly from a command line 

2546 program, but doing so removes the ability of the task to be 

2547 retargeted. As most cameras override some set of the IsrTask 

2548 methods, this would remove those data-specific methods in the 

2549 output post-ISR images. This wrapping class fixes the issue, 

2550 allowing identical post-ISR images to be generated by both the 

2551 processCcd and isrTask code. 

2552 """ 

2553 ConfigClass = RunIsrConfig 

2554 _DefaultName = "runIsr" 

2555 

2556 def __init__(self, *args, **kwargs): 

2557 super().__init__(*args, **kwargs) 

2558 self.makeSubtask("isr") 

2559 

2560 def runDataRef(self, dataRef): 

2561 """ 

2562 Parameters 

2563 ---------- 

2564 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2565 data reference of the detector data to be processed 

2566 

2567 Returns 

2568 ------- 

2569 result : `pipeBase.Struct` 

2570 Result struct with component: 

2571 

2572 - exposure : `lsst.afw.image.Exposure` 

2573 Post-ISR processed exposure. 

2574 """ 

2575 return self.isr.runDataRef(dataRef)