Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42 

43from . import isrFunctions 

44from . import isrQa 

45from . import linearize 

46from .defects import Defects 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 ) 

123 # TODO: DM-25348. This does not work yet to correctly load 

124 # possible crosstalk sources. 

125 crosstalkSources = cT.PrerequisiteInput( 

126 name="isrOverscanCorrected", 

127 doc="Overscan corrected input images.", 

128 storageClass="Exposure", 

129 dimensions=["instrument", "exposure", "detector"], 

130 deferLoad=True, 

131 multiple=True, 

132 lookupFunction=crosstalkSourceLookup, 

133 ) 

134 bias = cT.PrerequisiteInput( 

135 name="bias", 

136 doc="Input bias calibration.", 

137 storageClass="ExposureF", 

138 dimensions=["instrument", "detector"], 

139 isCalibration=True, 

140 ) 

141 dark = cT.PrerequisiteInput( 

142 name='dark', 

143 doc="Input dark calibration.", 

144 storageClass="ExposureF", 

145 dimensions=["instrument", "detector"], 

146 isCalibration=True, 

147 ) 

148 flat = cT.PrerequisiteInput( 

149 name="flat", 

150 doc="Input flat calibration.", 

151 storageClass="ExposureF", 

152 dimensions=["instrument", "physical_filter", "detector"], 

153 isCalibration=True, 

154 ) 

155 fringes = cT.PrerequisiteInput( 

156 name="fringe", 

157 doc="Input fringe calibration.", 

158 storageClass="ExposureF", 

159 dimensions=["instrument", "physical_filter", "detector"], 

160 isCalibration=True, 

161 ) 

162 strayLightData = cT.PrerequisiteInput( 

163 name='yBackground', 

164 doc="Input stray light calibration.", 

165 storageClass="StrayLightData", 

166 dimensions=["instrument", "physical_filter", "detector"], 

167 isCalibration=True, 

168 ) 

169 bfKernel = cT.PrerequisiteInput( 

170 name='bfKernel', 

171 doc="Input brighter-fatter kernel.", 

172 storageClass="NumpyArray", 

173 dimensions=["instrument"], 

174 isCalibration=True, 

175 ) 

176 newBFKernel = cT.PrerequisiteInput( 

177 name='brighterFatterKernel', 

178 doc="Newer complete kernel + gain solutions.", 

179 storageClass="BrighterFatterKernel", 

180 dimensions=["instrument", "detector"], 

181 isCalibration=True, 

182 ) 

183 defects = cT.PrerequisiteInput( 

184 name='defects', 

185 doc="Input defect tables.", 

186 storageClass="Defects", 

187 dimensions=["instrument", "detector"], 

188 isCalibration=True, 

189 ) 

190 opticsTransmission = cT.PrerequisiteInput( 

191 name="transmission_optics", 

192 storageClass="TransmissionCurve", 

193 doc="Transmission curve due to the optics.", 

194 dimensions=["instrument"], 

195 isCalibration=True, 

196 ) 

197 filterTransmission = cT.PrerequisiteInput( 

198 name="transmission_filter", 

199 storageClass="TransmissionCurve", 

200 doc="Transmission curve due to the filter.", 

201 dimensions=["instrument", "physical_filter"], 

202 isCalibration=True, 

203 ) 

204 sensorTransmission = cT.PrerequisiteInput( 

205 name="transmission_sensor", 

206 storageClass="TransmissionCurve", 

207 doc="Transmission curve due to the sensor.", 

208 dimensions=["instrument", "detector"], 

209 isCalibration=True, 

210 ) 

211 atmosphereTransmission = cT.PrerequisiteInput( 

212 name="transmission_atmosphere", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the atmosphere.", 

215 dimensions=["instrument"], 

216 isCalibration=True, 

217 ) 

218 illumMaskedImage = cT.PrerequisiteInput( 

219 name="illum", 

220 doc="Input illumination correction.", 

221 storageClass="MaskedImageF", 

222 dimensions=["instrument", "physical_filter", "detector"], 

223 isCalibration=True, 

224 ) 

225 

226 outputExposure = cT.Output( 

227 name='postISRCCD', 

228 doc="Output ISR processed exposure.", 

229 storageClass="Exposure", 

230 dimensions=["instrument", "exposure", "detector"], 

231 ) 

232 preInterpExposure = cT.Output( 

233 name='preInterpISRCCD', 

234 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

235 storageClass="ExposureF", 

236 dimensions=["instrument", "exposure", "detector"], 

237 ) 

238 outputOssThumbnail = cT.Output( 

239 name="OssThumb", 

240 doc="Output Overscan-subtracted thumbnail image.", 

241 storageClass="Thumbnail", 

242 dimensions=["instrument", "exposure", "detector"], 

243 ) 

244 outputFlattenedThumbnail = cT.Output( 

245 name="FlattenedThumb", 

246 doc="Output flat-corrected thumbnail image.", 

247 storageClass="Thumbnail", 

248 dimensions=["instrument", "exposure", "detector"], 

249 ) 

250 

251 def __init__(self, *, config=None): 

252 super().__init__(config=config) 

253 

254 if config.doBias is not True: 

255 self.prerequisiteInputs.discard("bias") 

256 if config.doLinearize is not True: 

257 self.prerequisiteInputs.discard("linearizer") 

258 if config.doCrosstalk is not True: 

259 self.inputs.discard("crosstalkSources") 

260 self.prerequisiteInputs.discard("crosstalk") 

261 if config.doBrighterFatter is not True: 

262 self.prerequisiteInputs.discard("bfKernel") 

263 self.prerequisiteInputs.discard("newBFKernel") 

264 if config.doDefect is not True: 

265 self.prerequisiteInputs.discard("defects") 

266 if config.doDark is not True: 

267 self.prerequisiteInputs.discard("dark") 

268 if config.doFlat is not True: 

269 self.prerequisiteInputs.discard("flat") 

270 if config.doAttachTransmissionCurve is not True: 

271 self.prerequisiteInputs.discard("opticsTransmission") 

272 self.prerequisiteInputs.discard("filterTransmission") 

273 self.prerequisiteInputs.discard("sensorTransmission") 

274 self.prerequisiteInputs.discard("atmosphereTransmission") 

275 if config.doUseOpticsTransmission is not True: 

276 self.prerequisiteInputs.discard("opticsTransmission") 

277 if config.doUseFilterTransmission is not True: 

278 self.prerequisiteInputs.discard("filterTransmission") 

279 if config.doUseSensorTransmission is not True: 

280 self.prerequisiteInputs.discard("sensorTransmission") 

281 if config.doUseAtmosphereTransmission is not True: 

282 self.prerequisiteInputs.discard("atmosphereTransmission") 

283 if config.doIlluminationCorrection is not True: 

284 self.prerequisiteInputs.discard("illumMaskedImage") 

285 

286 if config.doWrite is not True: 

287 self.outputs.discard("outputExposure") 

288 self.outputs.discard("preInterpExposure") 

289 self.outputs.discard("outputFlattenedThumbnail") 

290 self.outputs.discard("outputOssThumbnail") 

291 if config.doSaveInterpPixels is not True: 

292 self.outputs.discard("preInterpExposure") 

293 if config.qa.doThumbnailOss is not True: 

294 self.outputs.discard("outputOssThumbnail") 

295 if config.qa.doThumbnailFlattened is not True: 

296 self.outputs.discard("outputFlattenedThumbnail") 

297 

298 

299class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

300 pipelineConnections=IsrTaskConnections): 

301 """Configuration parameters for IsrTask. 

302 

303 Items are grouped in the order in which they are executed by the task. 

304 """ 

305 datasetType = pexConfig.Field( 

306 dtype=str, 

307 doc="Dataset type for input data; users will typically leave this alone, " 

308 "but camera-specific ISR tasks will override it", 

309 default="raw", 

310 ) 

311 

312 fallbackFilterName = pexConfig.Field( 

313 dtype=str, 

314 doc="Fallback default filter name for calibrations.", 

315 optional=True 

316 ) 

317 useFallbackDate = pexConfig.Field( 

318 dtype=bool, 

319 doc="Pass observation date when using fallback filter.", 

320 default=False, 

321 ) 

322 expectWcs = pexConfig.Field( 

323 dtype=bool, 

324 default=True, 

325 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

326 ) 

327 fwhm = pexConfig.Field( 

328 dtype=float, 

329 doc="FWHM of PSF in arcseconds.", 

330 default=1.0, 

331 ) 

332 qa = pexConfig.ConfigField( 

333 dtype=isrQa.IsrQaConfig, 

334 doc="QA related configuration options.", 

335 ) 

336 

337 # Image conversion configuration 

338 doConvertIntToFloat = pexConfig.Field( 

339 dtype=bool, 

340 doc="Convert integer raw images to floating point values?", 

341 default=True, 

342 ) 

343 

344 # Saturated pixel handling. 

345 doSaturation = pexConfig.Field( 

346 dtype=bool, 

347 doc="Mask saturated pixels? NB: this is totally independent of the" 

348 " interpolation option - this is ONLY setting the bits in the mask." 

349 " To have them interpolated make sure doSaturationInterpolation=True", 

350 default=True, 

351 ) 

352 saturatedMaskName = pexConfig.Field( 

353 dtype=str, 

354 doc="Name of mask plane to use in saturation detection and interpolation", 

355 default="SAT", 

356 ) 

357 saturation = pexConfig.Field( 

358 dtype=float, 

359 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

360 default=float("NaN"), 

361 ) 

362 growSaturationFootprintSize = pexConfig.Field( 

363 dtype=int, 

364 doc="Number of pixels by which to grow the saturation footprints", 

365 default=1, 

366 ) 

367 

368 # Suspect pixel handling. 

369 doSuspect = pexConfig.Field( 

370 dtype=bool, 

371 doc="Mask suspect pixels?", 

372 default=False, 

373 ) 

374 suspectMaskName = pexConfig.Field( 

375 dtype=str, 

376 doc="Name of mask plane to use for suspect pixels", 

377 default="SUSPECT", 

378 ) 

379 numEdgeSuspect = pexConfig.Field( 

380 dtype=int, 

381 doc="Number of edge pixels to be flagged as untrustworthy.", 

382 default=0, 

383 ) 

384 edgeMaskLevel = pexConfig.ChoiceField( 

385 dtype=str, 

386 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

387 default="DETECTOR", 

388 allowed={ 

389 'DETECTOR': 'Mask only the edges of the full detector.', 

390 'AMP': 'Mask edges of each amplifier.', 

391 }, 

392 ) 

393 

394 # Initial masking options. 

395 doSetBadRegions = pexConfig.Field( 

396 dtype=bool, 

397 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

398 default=True, 

399 ) 

400 badStatistic = pexConfig.ChoiceField( 

401 dtype=str, 

402 doc="How to estimate the average value for BAD regions.", 

403 default='MEANCLIP', 

404 allowed={ 

405 "MEANCLIP": "Correct using the (clipped) mean of good data", 

406 "MEDIAN": "Correct using the median of the good data", 

407 }, 

408 ) 

409 

410 # Overscan subtraction configuration. 

411 doOverscan = pexConfig.Field( 

412 dtype=bool, 

413 doc="Do overscan subtraction?", 

414 default=True, 

415 ) 

416 overscan = pexConfig.ConfigurableField( 

417 target=OverscanCorrectionTask, 

418 doc="Overscan subtraction task for image segments.", 

419 ) 

420 

421 overscanFitType = pexConfig.ChoiceField( 

422 dtype=str, 

423 doc="The method for fitting the overscan bias level.", 

424 default='MEDIAN', 

425 allowed={ 

426 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

427 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

428 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

429 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

430 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

431 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

432 "MEAN": "Correct using the mean of the overscan region", 

433 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

434 "MEDIAN": "Correct using the median of the overscan region", 

435 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

436 }, 

437 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

438 " This option will no longer be used, and will be removed after v20.") 

439 ) 

440 overscanOrder = pexConfig.Field( 

441 dtype=int, 

442 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

443 "or number of spline knots if overscan fit type is a spline."), 

444 default=1, 

445 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

446 " This option will no longer be used, and will be removed after v20.") 

447 ) 

448 overscanNumSigmaClip = pexConfig.Field( 

449 dtype=float, 

450 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

451 default=3.0, 

452 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

453 " This option will no longer be used, and will be removed after v20.") 

454 ) 

455 overscanIsInt = pexConfig.Field( 

456 dtype=bool, 

457 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

458 " and overscan.FitType=MEDIAN_PER_ROW.", 

459 default=True, 

460 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

461 " This option will no longer be used, and will be removed after v20.") 

462 ) 

463 # These options do not get deprecated, as they define how we slice up the image data. 

464 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

465 dtype=int, 

466 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

467 default=0, 

468 ) 

469 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

470 dtype=int, 

471 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

472 default=0, 

473 ) 

474 overscanMaxDev = pexConfig.Field( 474 ↛ exitline 474 didn't jump to the function exit

475 dtype=float, 

476 doc="Maximum deviation from the median for overscan", 

477 default=1000.0, check=lambda x: x > 0 

478 ) 

479 overscanBiasJump = pexConfig.Field( 

480 dtype=bool, 

481 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

482 default=False, 

483 ) 

484 overscanBiasJumpKeyword = pexConfig.Field( 

485 dtype=str, 

486 doc="Header keyword containing information about devices.", 

487 default="NO_SUCH_KEY", 

488 ) 

489 overscanBiasJumpDevices = pexConfig.ListField( 

490 dtype=str, 

491 doc="List of devices that need piecewise overscan correction.", 

492 default=(), 

493 ) 

494 overscanBiasJumpLocation = pexConfig.Field( 

495 dtype=int, 

496 doc="Location of bias jump along y-axis.", 

497 default=0, 

498 ) 

499 

500 # Amplifier to CCD assembly configuration 

501 doAssembleCcd = pexConfig.Field( 

502 dtype=bool, 

503 default=True, 

504 doc="Assemble amp-level exposures into a ccd-level exposure?" 

505 ) 

506 assembleCcd = pexConfig.ConfigurableField( 

507 target=AssembleCcdTask, 

508 doc="CCD assembly task", 

509 ) 

510 

511 # General calibration configuration. 

512 doAssembleIsrExposures = pexConfig.Field( 

513 dtype=bool, 

514 default=False, 

515 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

516 ) 

517 doTrimToMatchCalib = pexConfig.Field( 

518 dtype=bool, 

519 default=False, 

520 doc="Trim raw data to match calibration bounding boxes?" 

521 ) 

522 

523 # Bias subtraction. 

524 doBias = pexConfig.Field( 

525 dtype=bool, 

526 doc="Apply bias frame correction?", 

527 default=True, 

528 ) 

529 biasDataProductName = pexConfig.Field( 

530 dtype=str, 

531 doc="Name of the bias data product", 

532 default="bias", 

533 ) 

534 doBiasBeforeOverscan = pexConfig.Field( 

535 dtype=bool, 

536 doc="Reverse order of overscan and bias correction.", 

537 default=False 

538 ) 

539 

540 # Variance construction 

541 doVariance = pexConfig.Field( 

542 dtype=bool, 

543 doc="Calculate variance?", 

544 default=True 

545 ) 

546 gain = pexConfig.Field( 

547 dtype=float, 

548 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

549 default=float("NaN"), 

550 ) 

551 readNoise = pexConfig.Field( 

552 dtype=float, 

553 doc="The read noise to use if no Detector is present in the Exposure", 

554 default=0.0, 

555 ) 

556 doEmpiricalReadNoise = pexConfig.Field( 

557 dtype=bool, 

558 default=False, 

559 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

560 ) 

561 

562 # Linearization. 

563 doLinearize = pexConfig.Field( 

564 dtype=bool, 

565 doc="Correct for nonlinearity of the detector's response?", 

566 default=True, 

567 ) 

568 

569 # Crosstalk. 

570 doCrosstalk = pexConfig.Field( 

571 dtype=bool, 

572 doc="Apply intra-CCD crosstalk correction?", 

573 default=False, 

574 ) 

575 doCrosstalkBeforeAssemble = pexConfig.Field( 

576 dtype=bool, 

577 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

578 default=False, 

579 ) 

580 crosstalk = pexConfig.ConfigurableField( 

581 target=CrosstalkTask, 

582 doc="Intra-CCD crosstalk correction", 

583 ) 

584 

585 # Masking options. 

586 doDefect = pexConfig.Field( 

587 dtype=bool, 

588 doc="Apply correction for CCD defects, e.g. hot pixels?", 

589 default=True, 

590 ) 

591 doNanMasking = pexConfig.Field( 

592 dtype=bool, 

593 doc="Mask NAN pixels?", 

594 default=True, 

595 ) 

596 doWidenSaturationTrails = pexConfig.Field( 

597 dtype=bool, 

598 doc="Widen bleed trails based on their width?", 

599 default=True 

600 ) 

601 

602 # Brighter-Fatter correction. 

603 doBrighterFatter = pexConfig.Field( 

604 dtype=bool, 

605 default=False, 

606 doc="Apply the brighter fatter correction" 

607 ) 

608 brighterFatterLevel = pexConfig.ChoiceField( 

609 dtype=str, 

610 default="DETECTOR", 

611 doc="The level at which to correct for brighter-fatter.", 

612 allowed={ 

613 "AMP": "Every amplifier treated separately.", 

614 "DETECTOR": "One kernel per detector", 

615 } 

616 ) 

617 brighterFatterMaxIter = pexConfig.Field( 

618 dtype=int, 

619 default=10, 

620 doc="Maximum number of iterations for the brighter fatter correction" 

621 ) 

622 brighterFatterThreshold = pexConfig.Field( 

623 dtype=float, 

624 default=1000, 

625 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

626 " absolute value of the difference between the current corrected image and the one" 

627 " from the previous iteration summed over all the pixels." 

628 ) 

629 brighterFatterApplyGain = pexConfig.Field( 

630 dtype=bool, 

631 default=True, 

632 doc="Should the gain be applied when applying the brighter fatter correction?" 

633 ) 

634 brighterFatterMaskGrowSize = pexConfig.Field( 

635 dtype=int, 

636 default=0, 

637 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

638 " when brighter-fatter correction is applied." 

639 ) 

640 

641 # Dark subtraction. 

642 doDark = pexConfig.Field( 

643 dtype=bool, 

644 doc="Apply dark frame correction?", 

645 default=True, 

646 ) 

647 darkDataProductName = pexConfig.Field( 

648 dtype=str, 

649 doc="Name of the dark data product", 

650 default="dark", 

651 ) 

652 

653 # Camera-specific stray light removal. 

654 doStrayLight = pexConfig.Field( 

655 dtype=bool, 

656 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

657 default=False, 

658 ) 

659 strayLight = pexConfig.ConfigurableField( 

660 target=StrayLightTask, 

661 doc="y-band stray light correction" 

662 ) 

663 

664 # Flat correction. 

665 doFlat = pexConfig.Field( 

666 dtype=bool, 

667 doc="Apply flat field correction?", 

668 default=True, 

669 ) 

670 flatDataProductName = pexConfig.Field( 

671 dtype=str, 

672 doc="Name of the flat data product", 

673 default="flat", 

674 ) 

675 flatScalingType = pexConfig.ChoiceField( 

676 dtype=str, 

677 doc="The method for scaling the flat on the fly.", 

678 default='USER', 

679 allowed={ 

680 "USER": "Scale by flatUserScale", 

681 "MEAN": "Scale by the inverse of the mean", 

682 "MEDIAN": "Scale by the inverse of the median", 

683 }, 

684 ) 

685 flatUserScale = pexConfig.Field( 

686 dtype=float, 

687 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

688 default=1.0, 

689 ) 

690 doTweakFlat = pexConfig.Field( 

691 dtype=bool, 

692 doc="Tweak flats to match observed amplifier ratios?", 

693 default=False 

694 ) 

695 

696 # Amplifier normalization based on gains instead of using flats configuration. 

697 doApplyGains = pexConfig.Field( 

698 dtype=bool, 

699 doc="Correct the amplifiers for their gains instead of applying flat correction", 

700 default=False, 

701 ) 

702 normalizeGains = pexConfig.Field( 

703 dtype=bool, 

704 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

705 default=False, 

706 ) 

707 

708 # Fringe correction. 

709 doFringe = pexConfig.Field( 

710 dtype=bool, 

711 doc="Apply fringe correction?", 

712 default=True, 

713 ) 

714 fringe = pexConfig.ConfigurableField( 

715 target=FringeTask, 

716 doc="Fringe subtraction task", 

717 ) 

718 fringeAfterFlat = pexConfig.Field( 

719 dtype=bool, 

720 doc="Do fringe subtraction after flat-fielding?", 

721 default=True, 

722 ) 

723 

724 # Initial CCD-level background statistics options. 

725 doMeasureBackground = pexConfig.Field( 

726 dtype=bool, 

727 doc="Measure the background level on the reduced image?", 

728 default=False, 

729 ) 

730 

731 # Camera-specific masking configuration. 

732 doCameraSpecificMasking = pexConfig.Field( 

733 dtype=bool, 

734 doc="Mask camera-specific bad regions?", 

735 default=False, 

736 ) 

737 masking = pexConfig.ConfigurableField( 

738 target=MaskingTask, 

739 doc="Masking task." 

740 ) 

741 

742 # Interpolation options. 

743 

744 doInterpolate = pexConfig.Field( 

745 dtype=bool, 

746 doc="Interpolate masked pixels?", 

747 default=True, 

748 ) 

749 doSaturationInterpolation = pexConfig.Field( 

750 dtype=bool, 

751 doc="Perform interpolation over pixels masked as saturated?" 

752 " NB: This is independent of doSaturation; if that is False this plane" 

753 " will likely be blank, resulting in a no-op here.", 

754 default=True, 

755 ) 

756 doNanInterpolation = pexConfig.Field( 

757 dtype=bool, 

758 doc="Perform interpolation over pixels masked as NaN?" 

759 " NB: This is independent of doNanMasking; if that is False this plane" 

760 " will likely be blank, resulting in a no-op here.", 

761 default=True, 

762 ) 

763 doNanInterpAfterFlat = pexConfig.Field( 

764 dtype=bool, 

765 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

766 "also have to interpolate them before flat-fielding."), 

767 default=False, 

768 ) 

769 maskListToInterpolate = pexConfig.ListField( 

770 dtype=str, 

771 doc="List of mask planes that should be interpolated.", 

772 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

773 ) 

774 doSaveInterpPixels = pexConfig.Field( 

775 dtype=bool, 

776 doc="Save a copy of the pre-interpolated pixel values?", 

777 default=False, 

778 ) 

779 

780 # Default photometric calibration options. 

781 fluxMag0T1 = pexConfig.DictField( 

782 keytype=str, 

783 itemtype=float, 

784 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

785 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

786 )) 

787 ) 

788 defaultFluxMag0T1 = pexConfig.Field( 

789 dtype=float, 

790 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

791 default=pow(10.0, 0.4*28.0) 

792 ) 

793 

794 # Vignette correction configuration. 

795 doVignette = pexConfig.Field( 

796 dtype=bool, 

797 doc="Apply vignetting parameters?", 

798 default=False, 

799 ) 

800 vignette = pexConfig.ConfigurableField( 

801 target=VignetteTask, 

802 doc="Vignetting task.", 

803 ) 

804 

805 # Transmission curve configuration. 

806 doAttachTransmissionCurve = pexConfig.Field( 

807 dtype=bool, 

808 default=False, 

809 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

810 ) 

811 doUseOpticsTransmission = pexConfig.Field( 

812 dtype=bool, 

813 default=True, 

814 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

815 ) 

816 doUseFilterTransmission = pexConfig.Field( 

817 dtype=bool, 

818 default=True, 

819 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

820 ) 

821 doUseSensorTransmission = pexConfig.Field( 

822 dtype=bool, 

823 default=True, 

824 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

825 ) 

826 doUseAtmosphereTransmission = pexConfig.Field( 

827 dtype=bool, 

828 default=True, 

829 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

830 ) 

831 

832 # Illumination correction. 

833 doIlluminationCorrection = pexConfig.Field( 

834 dtype=bool, 

835 default=False, 

836 doc="Perform illumination correction?" 

837 ) 

838 illuminationCorrectionDataProductName = pexConfig.Field( 

839 dtype=str, 

840 doc="Name of the illumination correction data product.", 

841 default="illumcor", 

842 ) 

843 illumScale = pexConfig.Field( 

844 dtype=float, 

845 doc="Scale factor for the illumination correction.", 

846 default=1.0, 

847 ) 

848 illumFilters = pexConfig.ListField( 

849 dtype=str, 

850 default=[], 

851 doc="Only perform illumination correction for these filters." 

852 ) 

853 

854 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

855 doWrite = pexConfig.Field( 

856 dtype=bool, 

857 doc="Persist postISRCCD?", 

858 default=True, 

859 ) 

860 

861 def validate(self): 

862 super().validate() 

863 if self.doFlat and self.doApplyGains: 

864 raise ValueError("You may not specify both doFlat and doApplyGains") 

865 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

866 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

867 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

868 self.maskListToInterpolate.append(self.saturatedMaskName) 

869 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

870 self.maskListToInterpolate.remove(self.saturatedMaskName) 

871 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

872 self.maskListToInterpolate.append("UNMASKEDNAN") 

873 

874 

875class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

876 """Apply common instrument signature correction algorithms to a raw frame. 

877 

878 The process for correcting imaging data is very similar from 

879 camera to camera. This task provides a vanilla implementation of 

880 doing these corrections, including the ability to turn certain 

881 corrections off if they are not needed. The inputs to the primary 

882 method, `run()`, are a raw exposure to be corrected and the 

883 calibration data products. The raw input is a single chip sized 

884 mosaic of all amps including overscans and other non-science 

885 pixels. The method `runDataRef()` identifies and defines the 

886 calibration data products, and is intended for use by a 

887 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

888 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

889 subclassed for different camera, although the most camera specific 

890 methods have been split into subtasks that can be redirected 

891 appropriately. 

892 

893 The __init__ method sets up the subtasks for ISR processing, using 

894 the defaults from `lsst.ip.isr`. 

895 

896 Parameters 

897 ---------- 

898 args : `list` 

899 Positional arguments passed to the Task constructor. None used at this time. 

900 kwargs : `dict`, optional 

901 Keyword arguments passed on to the Task constructor. None used at this time. 

902 """ 

903 ConfigClass = IsrTaskConfig 

904 _DefaultName = "isr" 

905 

906 def __init__(self, **kwargs): 

907 super().__init__(**kwargs) 

908 self.makeSubtask("assembleCcd") 

909 self.makeSubtask("crosstalk") 

910 self.makeSubtask("strayLight") 

911 self.makeSubtask("fringe") 

912 self.makeSubtask("masking") 

913 self.makeSubtask("overscan") 

914 self.makeSubtask("vignette") 

915 

916 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

917 inputs = butlerQC.get(inputRefs) 

918 

919 try: 

920 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

921 except Exception as e: 

922 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

923 (inputRefs, e)) 

924 

925 inputs['isGen3'] = True 

926 

927 detector = inputs['ccdExposure'].getDetector() 

928 

929 if self.config.doCrosstalk is True: 

930 # Crosstalk sources need to be defined by the pipeline 

931 # yaml if they exist. 

932 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

933 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

934 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

935 else: 

936 coeffVector = (self.config.crosstalk.crosstalkValues 

937 if self.config.crosstalk.useConfigCoefficients else None) 

938 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

939 inputs['crosstalk'] = crosstalkCalib 

940 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

941 if 'crosstalkSources' not in inputs: 

942 self.log.warn("No crosstalkSources found for chip with interChip terms!") 

943 

944 if self.doLinearize(detector) is True: 

945 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict): 

946 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

947 linearizer.fromYaml(inputs['linearizer']) 

948 else: 

949 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector, 

950 log=self.log) 

951 inputs['linearizer'] = linearizer 

952 

953 if self.config.doDefect is True: 

954 if "defects" in inputs and inputs['defects'] is not None: 

955 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

956 # masking expects a list of defects defined by their bounding box 

957 if not isinstance(inputs["defects"], Defects): 

958 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

959 

960 # Load the correct style of brighter fatter kernel, and repack 

961 # the information as a numpy array. 

962 if self.config.doBrighterFatter: 

963 brighterFatterKernel = inputs.pop('newBFKernel', None) 

964 if brighterFatterKernel is None: 

965 brighterFatterKernel = inputs.get('bfKernel', None) 

966 

967 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

968 detId = detector.getId() 

969 inputs['bfGains'] = brighterFatterKernel.gain 

970 # If the kernel is not an ndarray, it's the cp_pipe version 

971 # so extract the kernel for this detector, or raise an error 

972 if self.config.brighterFatterLevel == 'DETECTOR': 

973 if brighterFatterKernel.detectorKernel: 

974 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

975 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

976 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

977 else: 

978 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

979 else: 

980 # TODO DM-15631 for implementing this 

981 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

982 

983 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

984 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

985 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

986 expId=expId, 

987 assembler=self.assembleCcd 

988 if self.config.doAssembleIsrExposures else None) 

989 else: 

990 inputs['fringes'] = pipeBase.Struct(fringes=None) 

991 

992 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

993 if 'strayLightData' not in inputs: 

994 inputs['strayLightData'] = None 

995 

996 outputs = self.run(**inputs) 

997 butlerQC.put(outputs, outputRefs) 

998 

999 def readIsrData(self, dataRef, rawExposure): 

1000 """Retrieve necessary frames for instrument signature removal. 

1001 

1002 Pre-fetching all required ISR data products limits the IO 

1003 required by the ISR. Any conflict between the calibration data 

1004 available and that needed for ISR is also detected prior to 

1005 doing processing, allowing it to fail quickly. 

1006 

1007 Parameters 

1008 ---------- 

1009 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1010 Butler reference of the detector data to be processed 

1011 rawExposure : `afw.image.Exposure` 

1012 The raw exposure that will later be corrected with the 

1013 retrieved calibration data; should not be modified in this 

1014 method. 

1015 

1016 Returns 

1017 ------- 

1018 result : `lsst.pipe.base.Struct` 

1019 Result struct with components (which may be `None`): 

1020 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1021 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1022 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1023 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1024 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1025 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1026 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1027 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1028 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1029 - ``seed``: random seed derived from the ccdExposureId for random 

1030 number generator (`uint32`). 

1031 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1032 A ``TransmissionCurve`` that represents the throughput of the optics, 

1033 to be evaluated in focal-plane coordinates. 

1034 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1035 A ``TransmissionCurve`` that represents the throughput of the filter 

1036 itself, to be evaluated in focal-plane coordinates. 

1037 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1038 A ``TransmissionCurve`` that represents the throughput of the sensor 

1039 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1040 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1041 A ``TransmissionCurve`` that represents the throughput of the 

1042 atmosphere, assumed to be spatially constant. 

1043 - ``strayLightData`` : `object` 

1044 An opaque object containing calibration information for 

1045 stray-light correction. If `None`, no correction will be 

1046 performed. 

1047 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1048 

1049 Raises 

1050 ------ 

1051 NotImplementedError : 

1052 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1053 """ 

1054 try: 

1055 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1056 dateObs = dateObs.toPython().isoformat() 

1057 except RuntimeError: 

1058 self.log.warn("Unable to identify dateObs for rawExposure.") 

1059 dateObs = None 

1060 

1061 ccd = rawExposure.getDetector() 

1062 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

1063 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1064 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1065 if self.config.doBias else None) 

1066 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1067 linearizer = (dataRef.get("linearizer", immediate=True) 

1068 if self.doLinearize(ccd) else None) 

1069 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1070 linearizer.log = self.log 

1071 if isinstance(linearizer, numpy.ndarray): 

1072 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1073 

1074 crosstalkCalib = None 

1075 if self.config.doCrosstalk: 

1076 try: 

1077 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1078 except NoResults: 

1079 coeffVector = (self.config.crosstalk.crosstalkValues 

1080 if self.config.crosstalk.useConfigCoefficients else None) 

1081 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1082 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1083 if self.config.doCrosstalk else None) 

1084 

1085 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1086 if self.config.doDark else None) 

1087 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1088 dateObs=dateObs) 

1089 if self.config.doFlat else None) 

1090 

1091 brighterFatterKernel = None 

1092 brighterFatterGains = None 

1093 if self.config.doBrighterFatter is True: 

1094 try: 

1095 # Use the new-style cp_pipe version of the kernel if it exists 

1096 # If using a new-style kernel, always use the self-consistent 

1097 # gains, i.e. the ones inside the kernel object itself 

1098 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1099 brighterFatterGains = brighterFatterKernel.gain 

1100 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

1101 except NoResults: 

1102 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1103 brighterFatterKernel = dataRef.get("bfKernel") 

1104 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

1105 except NoResults: 

1106 brighterFatterKernel = None 

1107 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1108 # If the kernel is not an ndarray, it's the cp_pipe version 

1109 # so extract the kernel for this detector, or raise an error 

1110 if self.config.brighterFatterLevel == 'DETECTOR': 

1111 if brighterFatterKernel.detectorKernel: 

1112 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

1113 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

1114 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

1115 else: 

1116 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1117 else: 

1118 # TODO DM-15631 for implementing this 

1119 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1120 

1121 defectList = (dataRef.get("defects") 

1122 if self.config.doDefect else None) 

1123 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

1124 if self.config.doAssembleIsrExposures else None) 

1125 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1126 else pipeBase.Struct(fringes=None)) 

1127 

1128 if self.config.doAttachTransmissionCurve: 

1129 opticsTransmission = (dataRef.get("transmission_optics") 

1130 if self.config.doUseOpticsTransmission else None) 

1131 filterTransmission = (dataRef.get("transmission_filter") 

1132 if self.config.doUseFilterTransmission else None) 

1133 sensorTransmission = (dataRef.get("transmission_sensor") 

1134 if self.config.doUseSensorTransmission else None) 

1135 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1136 if self.config.doUseAtmosphereTransmission else None) 

1137 else: 

1138 opticsTransmission = None 

1139 filterTransmission = None 

1140 sensorTransmission = None 

1141 atmosphereTransmission = None 

1142 

1143 if self.config.doStrayLight: 

1144 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1145 else: 

1146 strayLightData = None 

1147 

1148 illumMaskedImage = (self.getIsrExposure(dataRef, 

1149 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1150 if (self.config.doIlluminationCorrection 

1151 and filterName in self.config.illumFilters) 

1152 else None) 

1153 

1154 # Struct should include only kwargs to run() 

1155 return pipeBase.Struct(bias=biasExposure, 

1156 linearizer=linearizer, 

1157 crosstalk=crosstalkCalib, 

1158 crosstalkSources=crosstalkSources, 

1159 dark=darkExposure, 

1160 flat=flatExposure, 

1161 bfKernel=brighterFatterKernel, 

1162 bfGains=brighterFatterGains, 

1163 defects=defectList, 

1164 fringes=fringeStruct, 

1165 opticsTransmission=opticsTransmission, 

1166 filterTransmission=filterTransmission, 

1167 sensorTransmission=sensorTransmission, 

1168 atmosphereTransmission=atmosphereTransmission, 

1169 strayLightData=strayLightData, 

1170 illumMaskedImage=illumMaskedImage 

1171 ) 

1172 

1173 @pipeBase.timeMethod 

1174 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, 

1175 crosstalk=None, crosstalkSources=None, 

1176 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1177 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1178 sensorTransmission=None, atmosphereTransmission=None, 

1179 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1180 isGen3=False, 

1181 ): 

1182 """Perform instrument signature removal on an exposure. 

1183 

1184 Steps included in the ISR processing, in order performed, are: 

1185 - saturation and suspect pixel masking 

1186 - overscan subtraction 

1187 - CCD assembly of individual amplifiers 

1188 - bias subtraction 

1189 - variance image construction 

1190 - linearization of non-linear response 

1191 - crosstalk masking 

1192 - brighter-fatter correction 

1193 - dark subtraction 

1194 - fringe correction 

1195 - stray light subtraction 

1196 - flat correction 

1197 - masking of known defects and camera specific features 

1198 - vignette calculation 

1199 - appending transmission curve and distortion model 

1200 

1201 Parameters 

1202 ---------- 

1203 ccdExposure : `lsst.afw.image.Exposure` 

1204 The raw exposure that is to be run through ISR. The 

1205 exposure is modified by this method. 

1206 camera : `lsst.afw.cameraGeom.Camera`, optional 

1207 The camera geometry for this exposure. Required if ``isGen3`` is 

1208 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1209 ``flat`` does not have an associated detector. 

1210 bias : `lsst.afw.image.Exposure`, optional 

1211 Bias calibration frame. 

1212 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1213 Functor for linearization. 

1214 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1215 Calibration for crosstalk. 

1216 crosstalkSources : `list`, optional 

1217 List of possible crosstalk sources. 

1218 dark : `lsst.afw.image.Exposure`, optional 

1219 Dark calibration frame. 

1220 flat : `lsst.afw.image.Exposure`, optional 

1221 Flat calibration frame. 

1222 bfKernel : `numpy.ndarray`, optional 

1223 Brighter-fatter kernel. 

1224 bfGains : `dict` of `float`, optional 

1225 Gains used to override the detector's nominal gains for the 

1226 brighter-fatter correction. A dict keyed by amplifier name for 

1227 the detector in question. 

1228 defects : `lsst.ip.isr.Defects`, optional 

1229 List of defects. 

1230 fringes : `lsst.pipe.base.Struct`, optional 

1231 Struct containing the fringe correction data, with 

1232 elements: 

1233 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1234 - ``seed``: random seed derived from the ccdExposureId for random 

1235 number generator (`uint32`) 

1236 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1237 A ``TransmissionCurve`` that represents the throughput of the optics, 

1238 to be evaluated in focal-plane coordinates. 

1239 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1240 A ``TransmissionCurve`` that represents the throughput of the filter 

1241 itself, to be evaluated in focal-plane coordinates. 

1242 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1243 A ``TransmissionCurve`` that represents the throughput of the sensor 

1244 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1245 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1246 A ``TransmissionCurve`` that represents the throughput of the 

1247 atmosphere, assumed to be spatially constant. 

1248 detectorNum : `int`, optional 

1249 The integer number for the detector to process. 

1250 isGen3 : bool, optional 

1251 Flag this call to run() as using the Gen3 butler environment. 

1252 strayLightData : `object`, optional 

1253 Opaque object containing calibration information for stray-light 

1254 correction. If `None`, no correction will be performed. 

1255 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1256 Illumination correction image. 

1257 

1258 Returns 

1259 ------- 

1260 result : `lsst.pipe.base.Struct` 

1261 Result struct with component: 

1262 - ``exposure`` : `afw.image.Exposure` 

1263 The fully ISR corrected exposure. 

1264 - ``outputExposure`` : `afw.image.Exposure` 

1265 An alias for `exposure` 

1266 - ``ossThumb`` : `numpy.ndarray` 

1267 Thumbnail image of the exposure after overscan subtraction. 

1268 - ``flattenedThumb`` : `numpy.ndarray` 

1269 Thumbnail image of the exposure after flat-field correction. 

1270 

1271 Raises 

1272 ------ 

1273 RuntimeError 

1274 Raised if a configuration option is set to True, but the 

1275 required calibration data has not been specified. 

1276 

1277 Notes 

1278 ----- 

1279 The current processed exposure can be viewed by setting the 

1280 appropriate lsstDebug entries in the `debug.display` 

1281 dictionary. The names of these entries correspond to some of 

1282 the IsrTaskConfig Boolean options, with the value denoting the 

1283 frame to use. The exposure is shown inside the matching 

1284 option check and after the processing of that step has 

1285 finished. The steps with debug points are: 

1286 

1287 doAssembleCcd 

1288 doBias 

1289 doCrosstalk 

1290 doBrighterFatter 

1291 doDark 

1292 doFringe 

1293 doStrayLight 

1294 doFlat 

1295 

1296 In addition, setting the "postISRCCD" entry displays the 

1297 exposure after all ISR processing has finished. 

1298 

1299 """ 

1300 

1301 if isGen3 is True: 

1302 # Gen3 currently cannot automatically do configuration overrides. 

1303 # DM-15257 looks to discuss this issue. 

1304 # Configure input exposures; 

1305 if detectorNum is None: 

1306 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1307 

1308 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1309 bias = self.ensureExposure(bias, camera, detectorNum) 

1310 dark = self.ensureExposure(dark, camera, detectorNum) 

1311 flat = self.ensureExposure(flat, camera, detectorNum) 

1312 else: 

1313 if isinstance(ccdExposure, ButlerDataRef): 

1314 return self.runDataRef(ccdExposure) 

1315 

1316 ccd = ccdExposure.getDetector() 

1317 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1318 

1319 if not ccd: 

1320 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1321 ccd = [FakeAmp(ccdExposure, self.config)] 

1322 

1323 # Validate Input 

1324 if self.config.doBias and bias is None: 

1325 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1326 if self.doLinearize(ccd) and linearizer is None: 

1327 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1328 if self.config.doBrighterFatter and bfKernel is None: 

1329 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1330 if self.config.doDark and dark is None: 

1331 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1332 if self.config.doFlat and flat is None: 

1333 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1334 if self.config.doDefect and defects is None: 

1335 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1336 if (self.config.doFringe and filterName in self.fringe.config.filters 

1337 and fringes.fringes is None): 

1338 # The `fringes` object needs to be a pipeBase.Struct, as 

1339 # we use it as a `dict` for the parameters of 

1340 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1341 # not be `None` if `doFringe=True`. Otherwise, raise. 

1342 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1343 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters 

1344 and illumMaskedImage is None): 

1345 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1346 

1347 # Begin ISR processing. 

1348 if self.config.doConvertIntToFloat: 

1349 self.log.info("Converting exposure to floating point values.") 

1350 ccdExposure = self.convertIntToFloat(ccdExposure) 

1351 

1352 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1353 self.log.info("Applying bias correction.") 

1354 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1355 trimToFit=self.config.doTrimToMatchCalib) 

1356 self.debugView(ccdExposure, "doBias") 

1357 

1358 # Amplifier level processing. 

1359 overscans = [] 

1360 for amp in ccd: 

1361 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1362 if ccdExposure.getBBox().contains(amp.getBBox()): 

1363 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1364 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1365 

1366 if self.config.doOverscan and not badAmp: 

1367 # Overscan correction on amp-by-amp basis. 

1368 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1369 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1370 if overscanResults is not None and \ 

1371 self.config.qa is not None and self.config.qa.saveStats is True: 

1372 if isinstance(overscanResults.overscanFit, float): 

1373 qaMedian = overscanResults.overscanFit 

1374 qaStdev = float("NaN") 

1375 else: 

1376 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1377 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1378 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1379 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1380 

1381 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian) 

1382 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev) 

1383 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1384 amp.getName(), qaMedian, qaStdev) 

1385 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1386 else: 

1387 if badAmp: 

1388 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1389 overscanResults = None 

1390 

1391 overscans.append(overscanResults if overscanResults is not None else None) 

1392 else: 

1393 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1394 

1395 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1396 self.log.info("Applying crosstalk correction.") 

1397 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1398 crosstalkSources=crosstalkSources) 

1399 self.debugView(ccdExposure, "doCrosstalk") 

1400 

1401 if self.config.doAssembleCcd: 

1402 self.log.info("Assembling CCD from amplifiers.") 

1403 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1404 

1405 if self.config.expectWcs and not ccdExposure.getWcs(): 

1406 self.log.warn("No WCS found in input exposure.") 

1407 self.debugView(ccdExposure, "doAssembleCcd") 

1408 

1409 ossThumb = None 

1410 if self.config.qa.doThumbnailOss: 

1411 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1412 

1413 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1414 self.log.info("Applying bias correction.") 

1415 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1416 trimToFit=self.config.doTrimToMatchCalib) 

1417 self.debugView(ccdExposure, "doBias") 

1418 

1419 if self.config.doVariance: 

1420 for amp, overscanResults in zip(ccd, overscans): 

1421 if ccdExposure.getBBox().contains(amp.getBBox()): 

1422 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1423 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1424 if overscanResults is not None: 

1425 self.updateVariance(ampExposure, amp, 

1426 overscanImage=overscanResults.overscanImage) 

1427 else: 

1428 self.updateVariance(ampExposure, amp, 

1429 overscanImage=None) 

1430 if self.config.qa is not None and self.config.qa.saveStats is True: 

1431 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1432 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1433 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1434 qaStats.getValue(afwMath.MEDIAN)) 

1435 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1436 qaStats.getValue(afwMath.STDEVCLIP)) 

1437 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1438 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1439 qaStats.getValue(afwMath.STDEVCLIP)) 

1440 

1441 if self.doLinearize(ccd): 

1442 self.log.info("Applying linearizer.") 

1443 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1444 detector=ccd, log=self.log) 

1445 

1446 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1447 self.log.info("Applying crosstalk correction.") 

1448 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1449 crosstalkSources=crosstalkSources, isTrimmed=True) 

1450 self.debugView(ccdExposure, "doCrosstalk") 

1451 

1452 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1453 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1454 if self.config.doDefect: 

1455 self.log.info("Masking defects.") 

1456 self.maskDefect(ccdExposure, defects) 

1457 

1458 if self.config.numEdgeSuspect > 0: 

1459 self.log.info("Masking edges as SUSPECT.") 

1460 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1461 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1462 

1463 if self.config.doNanMasking: 

1464 self.log.info("Masking NAN value pixels.") 

1465 self.maskNan(ccdExposure) 

1466 

1467 if self.config.doWidenSaturationTrails: 

1468 self.log.info("Widening saturation trails.") 

1469 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1470 

1471 if self.config.doCameraSpecificMasking: 

1472 self.log.info("Masking regions for camera specific reasons.") 

1473 self.masking.run(ccdExposure) 

1474 

1475 if self.config.doBrighterFatter: 

1476 # We need to apply flats and darks before we can interpolate, and we 

1477 # need to interpolate before we do B-F, but we do B-F without the 

1478 # flats and darks applied so we can work in units of electrons or holes. 

1479 # This context manager applies and then removes the darks and flats. 

1480 # 

1481 # We also do not want to interpolate values here, so operate on temporary 

1482 # images so we can apply only the BF-correction and roll back the 

1483 # interpolation. 

1484 interpExp = ccdExposure.clone() 

1485 with self.flatContext(interpExp, flat, dark): 

1486 isrFunctions.interpolateFromMask( 

1487 maskedImage=interpExp.getMaskedImage(), 

1488 fwhm=self.config.fwhm, 

1489 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1490 maskNameList=self.config.maskListToInterpolate 

1491 ) 

1492 bfExp = interpExp.clone() 

1493 

1494 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1495 type(bfKernel), type(bfGains)) 

1496 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1497 self.config.brighterFatterMaxIter, 

1498 self.config.brighterFatterThreshold, 

1499 self.config.brighterFatterApplyGain, 

1500 bfGains) 

1501 if bfResults[1] == self.config.brighterFatterMaxIter: 

1502 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1503 bfResults[0]) 

1504 else: 

1505 self.log.info("Finished brighter fatter correction in %d iterations.", 

1506 bfResults[1]) 

1507 image = ccdExposure.getMaskedImage().getImage() 

1508 bfCorr = bfExp.getMaskedImage().getImage() 

1509 bfCorr -= interpExp.getMaskedImage().getImage() 

1510 image += bfCorr 

1511 

1512 # Applying the brighter-fatter correction applies a 

1513 # convolution to the science image. At the edges this 

1514 # convolution may not have sufficient valid pixels to 

1515 # produce a valid correction. Mark pixels within the size 

1516 # of the brighter-fatter kernel as EDGE to warn of this 

1517 # fact. 

1518 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1519 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1520 maskPlane="EDGE") 

1521 

1522 if self.config.brighterFatterMaskGrowSize > 0: 

1523 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1524 for maskPlane in self.config.maskListToInterpolate: 

1525 isrFunctions.growMasks(ccdExposure.getMask(), 

1526 radius=self.config.brighterFatterMaskGrowSize, 

1527 maskNameList=maskPlane, 

1528 maskValue=maskPlane) 

1529 

1530 self.debugView(ccdExposure, "doBrighterFatter") 

1531 

1532 if self.config.doDark: 

1533 self.log.info("Applying dark correction.") 

1534 self.darkCorrection(ccdExposure, dark) 

1535 self.debugView(ccdExposure, "doDark") 

1536 

1537 if self.config.doFringe and not self.config.fringeAfterFlat: 

1538 self.log.info("Applying fringe correction before flat.") 

1539 self.fringe.run(ccdExposure, **fringes.getDict()) 

1540 self.debugView(ccdExposure, "doFringe") 

1541 

1542 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1543 self.log.info("Checking strayLight correction.") 

1544 self.strayLight.run(ccdExposure, strayLightData) 

1545 self.debugView(ccdExposure, "doStrayLight") 

1546 

1547 if self.config.doFlat: 

1548 self.log.info("Applying flat correction.") 

1549 self.flatCorrection(ccdExposure, flat) 

1550 self.debugView(ccdExposure, "doFlat") 

1551 

1552 if self.config.doApplyGains: 

1553 self.log.info("Applying gain correction instead of flat.") 

1554 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1555 

1556 if self.config.doFringe and self.config.fringeAfterFlat: 

1557 self.log.info("Applying fringe correction after flat.") 

1558 self.fringe.run(ccdExposure, **fringes.getDict()) 

1559 

1560 if self.config.doVignette: 

1561 self.log.info("Constructing Vignette polygon.") 

1562 self.vignettePolygon = self.vignette.run(ccdExposure) 

1563 

1564 if self.config.vignette.doWriteVignettePolygon: 

1565 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1566 

1567 if self.config.doAttachTransmissionCurve: 

1568 self.log.info("Adding transmission curves.") 

1569 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1570 filterTransmission=filterTransmission, 

1571 sensorTransmission=sensorTransmission, 

1572 atmosphereTransmission=atmosphereTransmission) 

1573 

1574 flattenedThumb = None 

1575 if self.config.qa.doThumbnailFlattened: 

1576 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1577 

1578 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1579 self.log.info("Performing illumination correction.") 

1580 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1581 illumMaskedImage, illumScale=self.config.illumScale, 

1582 trimToFit=self.config.doTrimToMatchCalib) 

1583 

1584 preInterpExp = None 

1585 if self.config.doSaveInterpPixels: 

1586 preInterpExp = ccdExposure.clone() 

1587 

1588 # Reset and interpolate bad pixels. 

1589 # 

1590 # Large contiguous bad regions (which should have the BAD mask 

1591 # bit set) should have their values set to the image median. 

1592 # This group should include defects and bad amplifiers. As the 

1593 # area covered by these defects are large, there's little 

1594 # reason to expect that interpolation would provide a more 

1595 # useful value. 

1596 # 

1597 # Smaller defects can be safely interpolated after the larger 

1598 # regions have had their pixel values reset. This ensures 

1599 # that the remaining defects adjacent to bad amplifiers (as an 

1600 # example) do not attempt to interpolate extreme values. 

1601 if self.config.doSetBadRegions: 

1602 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1603 if badPixelCount > 0: 

1604 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1605 

1606 if self.config.doInterpolate: 

1607 self.log.info("Interpolating masked pixels.") 

1608 isrFunctions.interpolateFromMask( 

1609 maskedImage=ccdExposure.getMaskedImage(), 

1610 fwhm=self.config.fwhm, 

1611 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1612 maskNameList=list(self.config.maskListToInterpolate) 

1613 ) 

1614 

1615 self.roughZeroPoint(ccdExposure) 

1616 

1617 if self.config.doMeasureBackground: 

1618 self.log.info("Measuring background level.") 

1619 self.measureBackground(ccdExposure, self.config.qa) 

1620 

1621 if self.config.qa is not None and self.config.qa.saveStats is True: 

1622 for amp in ccd: 

1623 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1624 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1625 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1626 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1627 qaStats.getValue(afwMath.MEDIAN)) 

1628 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1629 qaStats.getValue(afwMath.STDEVCLIP)) 

1630 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1631 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1632 qaStats.getValue(afwMath.STDEVCLIP)) 

1633 

1634 self.debugView(ccdExposure, "postISRCCD") 

1635 

1636 return pipeBase.Struct( 

1637 exposure=ccdExposure, 

1638 ossThumb=ossThumb, 

1639 flattenedThumb=flattenedThumb, 

1640 

1641 preInterpolatedExposure=preInterpExp, 

1642 outputExposure=ccdExposure, 

1643 outputOssThumbnail=ossThumb, 

1644 outputFlattenedThumbnail=flattenedThumb, 

1645 ) 

1646 

1647 @pipeBase.timeMethod 

1648 def runDataRef(self, sensorRef): 

1649 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1650 

1651 This method contains the `CmdLineTask` interface to the ISR 

1652 processing. All IO is handled here, freeing the `run()` method 

1653 to manage only pixel-level calculations. The steps performed 

1654 are: 

1655 - Read in necessary detrending/isr/calibration data. 

1656 - Process raw exposure in `run()`. 

1657 - Persist the ISR-corrected exposure as "postISRCCD" if 

1658 config.doWrite=True. 

1659 

1660 Parameters 

1661 ---------- 

1662 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1663 DataRef of the detector data to be processed 

1664 

1665 Returns 

1666 ------- 

1667 result : `lsst.pipe.base.Struct` 

1668 Result struct with component: 

1669 - ``exposure`` : `afw.image.Exposure` 

1670 The fully ISR corrected exposure. 

1671 

1672 Raises 

1673 ------ 

1674 RuntimeError 

1675 Raised if a configuration option is set to True, but the 

1676 required calibration data does not exist. 

1677 

1678 """ 

1679 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1680 

1681 ccdExposure = sensorRef.get(self.config.datasetType) 

1682 

1683 camera = sensorRef.get("camera") 

1684 isrData = self.readIsrData(sensorRef, ccdExposure) 

1685 

1686 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1687 

1688 if self.config.doWrite: 

1689 sensorRef.put(result.exposure, "postISRCCD") 

1690 if result.preInterpolatedExposure is not None: 

1691 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1692 if result.ossThumb is not None: 

1693 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1694 if result.flattenedThumb is not None: 

1695 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1696 

1697 return result 

1698 

1699 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1700 """Retrieve a calibration dataset for removing instrument signature. 

1701 

1702 Parameters 

1703 ---------- 

1704 

1705 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1706 DataRef of the detector data to find calibration datasets 

1707 for. 

1708 datasetType : `str` 

1709 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1710 dateObs : `str`, optional 

1711 Date of the observation. Used to correct butler failures 

1712 when using fallback filters. 

1713 immediate : `Bool` 

1714 If True, disable butler proxies to enable error handling 

1715 within this routine. 

1716 

1717 Returns 

1718 ------- 

1719 exposure : `lsst.afw.image.Exposure` 

1720 Requested calibration frame. 

1721 

1722 Raises 

1723 ------ 

1724 RuntimeError 

1725 Raised if no matching calibration frame can be found. 

1726 """ 

1727 try: 

1728 exp = dataRef.get(datasetType, immediate=immediate) 

1729 except Exception as exc1: 

1730 if not self.config.fallbackFilterName: 

1731 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1732 try: 

1733 if self.config.useFallbackDate and dateObs: 

1734 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1735 dateObs=dateObs, immediate=immediate) 

1736 else: 

1737 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1738 except Exception as exc2: 

1739 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1740 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1741 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1742 

1743 if self.config.doAssembleIsrExposures: 

1744 exp = self.assembleCcd.assembleCcd(exp) 

1745 return exp 

1746 

1747 def ensureExposure(self, inputExp, camera, detectorNum): 

1748 """Ensure that the data returned by Butler is a fully constructed exposure. 

1749 

1750 ISR requires exposure-level image data for historical reasons, so if we did 

1751 not recieve that from Butler, construct it from what we have, modifying the 

1752 input in place. 

1753 

1754 Parameters 

1755 ---------- 

1756 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1757 `lsst.afw.image.ImageF` 

1758 The input data structure obtained from Butler. 

1759 camera : `lsst.afw.cameraGeom.camera` 

1760 The camera associated with the image. Used to find the appropriate 

1761 detector. 

1762 detectorNum : `int` 

1763 The detector this exposure should match. 

1764 

1765 Returns 

1766 ------- 

1767 inputExp : `lsst.afw.image.Exposure` 

1768 The re-constructed exposure, with appropriate detector parameters. 

1769 

1770 Raises 

1771 ------ 

1772 TypeError 

1773 Raised if the input data cannot be used to construct an exposure. 

1774 """ 

1775 if isinstance(inputExp, afwImage.DecoratedImageU): 

1776 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1777 elif isinstance(inputExp, afwImage.ImageF): 

1778 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1779 elif isinstance(inputExp, afwImage.MaskedImageF): 

1780 inputExp = afwImage.makeExposure(inputExp) 

1781 elif isinstance(inputExp, afwImage.Exposure): 

1782 pass 

1783 elif inputExp is None: 

1784 # Assume this will be caught by the setup if it is a problem. 

1785 return inputExp 

1786 else: 

1787 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1788 (type(inputExp), )) 

1789 

1790 if inputExp.getDetector() is None: 

1791 inputExp.setDetector(camera[detectorNum]) 

1792 

1793 return inputExp 

1794 

1795 def convertIntToFloat(self, exposure): 

1796 """Convert exposure image from uint16 to float. 

1797 

1798 If the exposure does not need to be converted, the input is 

1799 immediately returned. For exposures that are converted to use 

1800 floating point pixels, the variance is set to unity and the 

1801 mask to zero. 

1802 

1803 Parameters 

1804 ---------- 

1805 exposure : `lsst.afw.image.Exposure` 

1806 The raw exposure to be converted. 

1807 

1808 Returns 

1809 ------- 

1810 newexposure : `lsst.afw.image.Exposure` 

1811 The input ``exposure``, converted to floating point pixels. 

1812 

1813 Raises 

1814 ------ 

1815 RuntimeError 

1816 Raised if the exposure type cannot be converted to float. 

1817 

1818 """ 

1819 if isinstance(exposure, afwImage.ExposureF): 

1820 # Nothing to be done 

1821 self.log.debug("Exposure already of type float.") 

1822 return exposure 

1823 if not hasattr(exposure, "convertF"): 

1824 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1825 

1826 newexposure = exposure.convertF() 

1827 newexposure.variance[:] = 1 

1828 newexposure.mask[:] = 0x0 

1829 

1830 return newexposure 

1831 

1832 def maskAmplifier(self, ccdExposure, amp, defects): 

1833 """Identify bad amplifiers, saturated and suspect pixels. 

1834 

1835 Parameters 

1836 ---------- 

1837 ccdExposure : `lsst.afw.image.Exposure` 

1838 Input exposure to be masked. 

1839 amp : `lsst.afw.table.AmpInfoCatalog` 

1840 Catalog of parameters defining the amplifier on this 

1841 exposure to mask. 

1842 defects : `lsst.ip.isr.Defects` 

1843 List of defects. Used to determine if the entire 

1844 amplifier is bad. 

1845 

1846 Returns 

1847 ------- 

1848 badAmp : `Bool` 

1849 If this is true, the entire amplifier area is covered by 

1850 defects and unusable. 

1851 

1852 """ 

1853 maskedImage = ccdExposure.getMaskedImage() 

1854 

1855 badAmp = False 

1856 

1857 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1858 # comparison with current defects definition. 

1859 if defects is not None: 

1860 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1861 

1862 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1863 # association with pixels in current ccdExposure). 

1864 if badAmp: 

1865 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1866 afwImage.PARENT) 

1867 maskView = dataView.getMask() 

1868 maskView |= maskView.getPlaneBitMask("BAD") 

1869 del maskView 

1870 return badAmp 

1871 

1872 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1873 # Saturation and suspect pixels can be masked now, though. 

1874 limits = dict() 

1875 if self.config.doSaturation and not badAmp: 

1876 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1877 if self.config.doSuspect and not badAmp: 

1878 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1879 if math.isfinite(self.config.saturation): 

1880 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1881 

1882 for maskName, maskThreshold in limits.items(): 

1883 if not math.isnan(maskThreshold): 

1884 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1885 isrFunctions.makeThresholdMask( 

1886 maskedImage=dataView, 

1887 threshold=maskThreshold, 

1888 growFootprints=0, 

1889 maskName=maskName 

1890 ) 

1891 

1892 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1893 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1894 afwImage.PARENT) 

1895 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1896 self.config.suspectMaskName]) 

1897 if numpy.all(maskView.getArray() & maskVal > 0): 

1898 badAmp = True 

1899 maskView |= maskView.getPlaneBitMask("BAD") 

1900 

1901 return badAmp 

1902 

1903 def overscanCorrection(self, ccdExposure, amp): 

1904 """Apply overscan correction in place. 

1905 

1906 This method does initial pixel rejection of the overscan 

1907 region. The overscan can also be optionally segmented to 

1908 allow for discontinuous overscan responses to be fit 

1909 separately. The actual overscan subtraction is performed by 

1910 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1911 which is called here after the amplifier is preprocessed. 

1912 

1913 Parameters 

1914 ---------- 

1915 ccdExposure : `lsst.afw.image.Exposure` 

1916 Exposure to have overscan correction performed. 

1917 amp : `lsst.afw.cameraGeom.Amplifer` 

1918 The amplifier to consider while correcting the overscan. 

1919 

1920 Returns 

1921 ------- 

1922 overscanResults : `lsst.pipe.base.Struct` 

1923 Result struct with components: 

1924 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1925 Value or fit subtracted from the amplifier image data. 

1926 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1927 Value or fit subtracted from the overscan image data. 

1928 - ``overscanImage`` : `lsst.afw.image.Image` 

1929 Image of the overscan region with the overscan 

1930 correction applied. This quantity is used to estimate 

1931 the amplifier read noise empirically. 

1932 

1933 Raises 

1934 ------ 

1935 RuntimeError 

1936 Raised if the ``amp`` does not contain raw pixel information. 

1937 

1938 See Also 

1939 -------- 

1940 lsst.ip.isr.isrFunctions.overscanCorrection 

1941 """ 

1942 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1943 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1944 return None 

1945 

1946 statControl = afwMath.StatisticsControl() 

1947 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1948 

1949 # Determine the bounding boxes 

1950 dataBBox = amp.getRawDataBBox() 

1951 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1952 dx0 = 0 

1953 dx1 = 0 

1954 

1955 prescanBBox = amp.getRawPrescanBBox() 

1956 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1957 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1958 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1959 else: 

1960 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1961 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1962 

1963 # Determine if we need to work on subregions of the amplifier and overscan. 

1964 imageBBoxes = [] 

1965 overscanBBoxes = [] 

1966 

1967 if ((self.config.overscanBiasJump 

1968 and self.config.overscanBiasJumpLocation) 

1969 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

1970 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1971 self.config.overscanBiasJumpDevices)): 

1972 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1973 yLower = self.config.overscanBiasJumpLocation 

1974 yUpper = dataBBox.getHeight() - yLower 

1975 else: 

1976 yUpper = self.config.overscanBiasJumpLocation 

1977 yLower = dataBBox.getHeight() - yUpper 

1978 

1979 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1980 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1981 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1982 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1983 yLower))) 

1984 

1985 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1986 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1987 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1988 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1989 yUpper))) 

1990 else: 

1991 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1992 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1993 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1994 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1995 oscanBBox.getHeight()))) 

1996 

1997 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

1998 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1999 ampImage = ccdExposure.maskedImage[imageBBox] 

2000 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2001 

2002 overscanArray = overscanImage.image.array 

2003 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2004 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2005 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2006 

2007 statControl = afwMath.StatisticsControl() 

2008 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2009 

2010 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2011 

2012 # Measure average overscan levels and record them in the metadata. 

2013 levelStat = afwMath.MEDIAN 

2014 sigmaStat = afwMath.STDEVCLIP 

2015 

2016 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2017 self.config.qa.flatness.nIter) 

2018 metadata = ccdExposure.getMetadata() 

2019 ampNum = amp.getName() 

2020 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2021 if isinstance(overscanResults.overscanFit, float): 

2022 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2023 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2024 else: 

2025 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2026 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2027 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2028 

2029 return overscanResults 

2030 

2031 def updateVariance(self, ampExposure, amp, overscanImage=None): 

2032 """Set the variance plane using the amplifier gain and read noise 

2033 

2034 The read noise is calculated from the ``overscanImage`` if the 

2035 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2036 the value from the amplifier data is used. 

2037 

2038 Parameters 

2039 ---------- 

2040 ampExposure : `lsst.afw.image.Exposure` 

2041 Exposure to process. 

2042 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2043 Amplifier detector data. 

2044 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2045 Image of overscan, required only for empirical read noise. 

2046 

2047 See also 

2048 -------- 

2049 lsst.ip.isr.isrFunctions.updateVariance 

2050 """ 

2051 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2052 gain = amp.getGain() 

2053 

2054 if math.isnan(gain): 

2055 gain = 1.0 

2056 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2057 elif gain <= 0: 

2058 patchedGain = 1.0 

2059 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

2060 amp.getName(), gain, patchedGain) 

2061 gain = patchedGain 

2062 

2063 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2064 self.log.info("Overscan is none for EmpiricalReadNoise.") 

2065 

2066 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2067 stats = afwMath.StatisticsControl() 

2068 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2069 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2070 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2071 amp.getName(), readNoise) 

2072 else: 

2073 readNoise = amp.getReadNoise() 

2074 

2075 isrFunctions.updateVariance( 

2076 maskedImage=ampExposure.getMaskedImage(), 

2077 gain=gain, 

2078 readNoise=readNoise, 

2079 ) 

2080 

2081 def darkCorrection(self, exposure, darkExposure, invert=False): 

2082 """Apply dark correction in place. 

2083 

2084 Parameters 

2085 ---------- 

2086 exposure : `lsst.afw.image.Exposure` 

2087 Exposure to process. 

2088 darkExposure : `lsst.afw.image.Exposure` 

2089 Dark exposure of the same size as ``exposure``. 

2090 invert : `Bool`, optional 

2091 If True, re-add the dark to an already corrected image. 

2092 

2093 Raises 

2094 ------ 

2095 RuntimeError 

2096 Raised if either ``exposure`` or ``darkExposure`` do not 

2097 have their dark time defined. 

2098 

2099 See Also 

2100 -------- 

2101 lsst.ip.isr.isrFunctions.darkCorrection 

2102 """ 

2103 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2104 if math.isnan(expScale): 

2105 raise RuntimeError("Exposure darktime is NAN.") 

2106 if darkExposure.getInfo().getVisitInfo() is not None \ 

2107 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2108 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2109 else: 

2110 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2111 # so getDarkTime() does not exist. 

2112 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2113 darkScale = 1.0 

2114 

2115 isrFunctions.darkCorrection( 

2116 maskedImage=exposure.getMaskedImage(), 

2117 darkMaskedImage=darkExposure.getMaskedImage(), 

2118 expScale=expScale, 

2119 darkScale=darkScale, 

2120 invert=invert, 

2121 trimToFit=self.config.doTrimToMatchCalib 

2122 ) 

2123 

2124 def doLinearize(self, detector): 

2125 """Check if linearization is needed for the detector cameraGeom. 

2126 

2127 Checks config.doLinearize and the linearity type of the first 

2128 amplifier. 

2129 

2130 Parameters 

2131 ---------- 

2132 detector : `lsst.afw.cameraGeom.Detector` 

2133 Detector to get linearity type from. 

2134 

2135 Returns 

2136 ------- 

2137 doLinearize : `Bool` 

2138 If True, linearization should be performed. 

2139 """ 

2140 return self.config.doLinearize and \ 

2141 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2142 

2143 def flatCorrection(self, exposure, flatExposure, invert=False): 

2144 """Apply flat correction in place. 

2145 

2146 Parameters 

2147 ---------- 

2148 exposure : `lsst.afw.image.Exposure` 

2149 Exposure to process. 

2150 flatExposure : `lsst.afw.image.Exposure` 

2151 Flat exposure of the same size as ``exposure``. 

2152 invert : `Bool`, optional 

2153 If True, unflatten an already flattened image. 

2154 

2155 See Also 

2156 -------- 

2157 lsst.ip.isr.isrFunctions.flatCorrection 

2158 """ 

2159 isrFunctions.flatCorrection( 

2160 maskedImage=exposure.getMaskedImage(), 

2161 flatMaskedImage=flatExposure.getMaskedImage(), 

2162 scalingType=self.config.flatScalingType, 

2163 userScale=self.config.flatUserScale, 

2164 invert=invert, 

2165 trimToFit=self.config.doTrimToMatchCalib 

2166 ) 

2167 

2168 def saturationDetection(self, exposure, amp): 

2169 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2170 

2171 Parameters 

2172 ---------- 

2173 exposure : `lsst.afw.image.Exposure` 

2174 Exposure to process. Only the amplifier DataSec is processed. 

2175 amp : `lsst.afw.table.AmpInfoCatalog` 

2176 Amplifier detector data. 

2177 

2178 See Also 

2179 -------- 

2180 lsst.ip.isr.isrFunctions.makeThresholdMask 

2181 """ 

2182 if not math.isnan(amp.getSaturation()): 

2183 maskedImage = exposure.getMaskedImage() 

2184 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2185 isrFunctions.makeThresholdMask( 

2186 maskedImage=dataView, 

2187 threshold=amp.getSaturation(), 

2188 growFootprints=0, 

2189 maskName=self.config.saturatedMaskName, 

2190 ) 

2191 

2192 def saturationInterpolation(self, exposure): 

2193 """Interpolate over saturated pixels, in place. 

2194 

2195 This method should be called after `saturationDetection`, to 

2196 ensure that the saturated pixels have been identified in the 

2197 SAT mask. It should also be called after `assembleCcd`, since 

2198 saturated regions may cross amplifier boundaries. 

2199 

2200 Parameters 

2201 ---------- 

2202 exposure : `lsst.afw.image.Exposure` 

2203 Exposure to process. 

2204 

2205 See Also 

2206 -------- 

2207 lsst.ip.isr.isrTask.saturationDetection 

2208 lsst.ip.isr.isrFunctions.interpolateFromMask 

2209 """ 

2210 isrFunctions.interpolateFromMask( 

2211 maskedImage=exposure.getMaskedImage(), 

2212 fwhm=self.config.fwhm, 

2213 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2214 maskNameList=list(self.config.saturatedMaskName), 

2215 ) 

2216 

2217 def suspectDetection(self, exposure, amp): 

2218 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2219 

2220 Parameters 

2221 ---------- 

2222 exposure : `lsst.afw.image.Exposure` 

2223 Exposure to process. Only the amplifier DataSec is processed. 

2224 amp : `lsst.afw.table.AmpInfoCatalog` 

2225 Amplifier detector data. 

2226 

2227 See Also 

2228 -------- 

2229 lsst.ip.isr.isrFunctions.makeThresholdMask 

2230 

2231 Notes 

2232 ----- 

2233 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2234 This is intended to indicate pixels that may be affected by unknown systematics; 

2235 for example if non-linearity corrections above a certain level are unstable 

2236 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2237 that no such level exists and no pixels are to be masked as suspicious. 

2238 """ 

2239 suspectLevel = amp.getSuspectLevel() 

2240 if math.isnan(suspectLevel): 

2241 return 

2242 

2243 maskedImage = exposure.getMaskedImage() 

2244 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2245 isrFunctions.makeThresholdMask( 

2246 maskedImage=dataView, 

2247 threshold=suspectLevel, 

2248 growFootprints=0, 

2249 maskName=self.config.suspectMaskName, 

2250 ) 

2251 

2252 def maskDefect(self, exposure, defectBaseList): 

2253 """Mask defects using mask plane "BAD", in place. 

2254 

2255 Parameters 

2256 ---------- 

2257 exposure : `lsst.afw.image.Exposure` 

2258 Exposure to process. 

2259 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2260 `lsst.afw.image.DefectBase`. 

2261 List of defects to mask. 

2262 

2263 Notes 

2264 ----- 

2265 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2266 """ 

2267 maskedImage = exposure.getMaskedImage() 

2268 if not isinstance(defectBaseList, Defects): 

2269 # Promotes DefectBase to Defect 

2270 defectList = Defects(defectBaseList) 

2271 else: 

2272 defectList = defectBaseList 

2273 defectList.maskPixels(maskedImage, maskName="BAD") 

2274 

2275 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2276 """Mask edge pixels with applicable mask plane. 

2277 

2278 Parameters 

2279 ---------- 

2280 exposure : `lsst.afw.image.Exposure` 

2281 Exposure to process. 

2282 numEdgePixels : `int`, optional 

2283 Number of edge pixels to mask. 

2284 maskPlane : `str`, optional 

2285 Mask plane name to use. 

2286 level : `str`, optional 

2287 Level at which to mask edges. 

2288 """ 

2289 maskedImage = exposure.getMaskedImage() 

2290 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2291 

2292 if numEdgePixels > 0: 

2293 if level == 'DETECTOR': 

2294 boxes = [maskedImage.getBBox()] 

2295 elif level == 'AMP': 

2296 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2297 

2298 for box in boxes: 

2299 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2300 subImage = maskedImage[box] 

2301 box.grow(-numEdgePixels) 

2302 # Mask pixels outside box 

2303 SourceDetectionTask.setEdgeBits( 

2304 subImage, 

2305 box, 

2306 maskBitMask) 

2307 

2308 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2309 """Mask and interpolate defects using mask plane "BAD", in place. 

2310 

2311 Parameters 

2312 ---------- 

2313 exposure : `lsst.afw.image.Exposure` 

2314 Exposure to process. 

2315 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2316 `lsst.afw.image.DefectBase`. 

2317 List of defects to mask and interpolate. 

2318 

2319 See Also 

2320 -------- 

2321 lsst.ip.isr.isrTask.maskDefect 

2322 """ 

2323 self.maskDefect(exposure, defectBaseList) 

2324 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2325 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2326 isrFunctions.interpolateFromMask( 

2327 maskedImage=exposure.getMaskedImage(), 

2328 fwhm=self.config.fwhm, 

2329 growSaturatedFootprints=0, 

2330 maskNameList=["BAD"], 

2331 ) 

2332 

2333 def maskNan(self, exposure): 

2334 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2335 

2336 Parameters 

2337 ---------- 

2338 exposure : `lsst.afw.image.Exposure` 

2339 Exposure to process. 

2340 

2341 Notes 

2342 ----- 

2343 We mask over all NaNs, including those that are masked with 

2344 other bits (because those may or may not be interpolated over 

2345 later, and we want to remove all NaNs). Despite this 

2346 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2347 the historical name. 

2348 """ 

2349 maskedImage = exposure.getMaskedImage() 

2350 

2351 # Find and mask NaNs 

2352 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2353 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2354 numNans = maskNans(maskedImage, maskVal) 

2355 self.metadata.set("NUMNANS", numNans) 

2356 if numNans > 0: 

2357 self.log.warn("There were %d unmasked NaNs.", numNans) 

2358 

2359 def maskAndInterpolateNan(self, exposure): 

2360 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2361 

2362 Parameters 

2363 ---------- 

2364 exposure : `lsst.afw.image.Exposure` 

2365 Exposure to process. 

2366 

2367 See Also 

2368 -------- 

2369 lsst.ip.isr.isrTask.maskNan 

2370 """ 

2371 self.maskNan(exposure) 

2372 isrFunctions.interpolateFromMask( 

2373 maskedImage=exposure.getMaskedImage(), 

2374 fwhm=self.config.fwhm, 

2375 growSaturatedFootprints=0, 

2376 maskNameList=["UNMASKEDNAN"], 

2377 ) 

2378 

2379 def measureBackground(self, exposure, IsrQaConfig=None): 

2380 """Measure the image background in subgrids, for quality control purposes. 

2381 

2382 Parameters 

2383 ---------- 

2384 exposure : `lsst.afw.image.Exposure` 

2385 Exposure to process. 

2386 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2387 Configuration object containing parameters on which background 

2388 statistics and subgrids to use. 

2389 """ 

2390 if IsrQaConfig is not None: 

2391 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2392 IsrQaConfig.flatness.nIter) 

2393 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2394 statsControl.setAndMask(maskVal) 

2395 maskedImage = exposure.getMaskedImage() 

2396 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2397 skyLevel = stats.getValue(afwMath.MEDIAN) 

2398 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2399 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2400 metadata = exposure.getMetadata() 

2401 metadata.set('SKYLEVEL', skyLevel) 

2402 metadata.set('SKYSIGMA', skySigma) 

2403 

2404 # calcluating flatlevel over the subgrids 

2405 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2406 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2407 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2408 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2409 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2410 skyLevels = numpy.zeros((nX, nY)) 

2411 

2412 for j in range(nY): 

2413 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2414 for i in range(nX): 

2415 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2416 

2417 xLLC = xc - meshXHalf 

2418 yLLC = yc - meshYHalf 

2419 xURC = xc + meshXHalf - 1 

2420 yURC = yc + meshYHalf - 1 

2421 

2422 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2423 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2424 

2425 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2426 

2427 good = numpy.where(numpy.isfinite(skyLevels)) 

2428 skyMedian = numpy.median(skyLevels[good]) 

2429 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2430 flatness_rms = numpy.std(flatness) 

2431 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2432 

2433 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2434 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2435 nX, nY, flatness_pp, flatness_rms) 

2436 

2437 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2438 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2439 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2440 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2441 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2442 

2443 def roughZeroPoint(self, exposure): 

2444 """Set an approximate magnitude zero point for the exposure. 

2445 

2446 Parameters 

2447 ---------- 

2448 exposure : `lsst.afw.image.Exposure` 

2449 Exposure to process. 

2450 """ 

2451 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2452 if filterName in self.config.fluxMag0T1: 

2453 fluxMag0 = self.config.fluxMag0T1[filterName] 

2454 else: 

2455 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2456 fluxMag0 = self.config.defaultFluxMag0T1 

2457 

2458 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2459 if not expTime > 0: # handle NaN as well as <= 0 

2460 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2461 return 

2462 

2463 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2464 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2465 

2466 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2467 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2468 

2469 Parameters 

2470 ---------- 

2471 ccdExposure : `lsst.afw.image.Exposure` 

2472 Exposure to process. 

2473 fpPolygon : `lsst.afw.geom.Polygon` 

2474 Polygon in focal plane coordinates. 

2475 """ 

2476 # Get ccd corners in focal plane coordinates 

2477 ccd = ccdExposure.getDetector() 

2478 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2479 ccdPolygon = Polygon(fpCorners) 

2480 

2481 # Get intersection of ccd corners with fpPolygon 

2482 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2483 

2484 # Transform back to pixel positions and build new polygon 

2485 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2486 validPolygon = Polygon(ccdPoints) 

2487 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2488 

2489 @contextmanager 

2490 def flatContext(self, exp, flat, dark=None): 

2491 """Context manager that applies and removes flats and darks, 

2492 if the task is configured to apply them. 

2493 

2494 Parameters 

2495 ---------- 

2496 exp : `lsst.afw.image.Exposure` 

2497 Exposure to process. 

2498 flat : `lsst.afw.image.Exposure` 

2499 Flat exposure the same size as ``exp``. 

2500 dark : `lsst.afw.image.Exposure`, optional 

2501 Dark exposure the same size as ``exp``. 

2502 

2503 Yields 

2504 ------ 

2505 exp : `lsst.afw.image.Exposure` 

2506 The flat and dark corrected exposure. 

2507 """ 

2508 if self.config.doDark and dark is not None: 

2509 self.darkCorrection(exp, dark) 

2510 if self.config.doFlat: 

2511 self.flatCorrection(exp, flat) 

2512 try: 

2513 yield exp 

2514 finally: 

2515 if self.config.doFlat: 

2516 self.flatCorrection(exp, flat, invert=True) 

2517 if self.config.doDark and dark is not None: 

2518 self.darkCorrection(exp, dark, invert=True) 

2519 

2520 def debugView(self, exposure, stepname): 

2521 """Utility function to examine ISR exposure at different stages. 

2522 

2523 Parameters 

2524 ---------- 

2525 exposure : `lsst.afw.image.Exposure` 

2526 Exposure to view. 

2527 stepname : `str` 

2528 State of processing to view. 

2529 """ 

2530 frame = getDebugFrame(self._display, stepname) 

2531 if frame: 

2532 display = getDisplay(frame) 

2533 display.scale('asinh', 'zscale') 

2534 display.mtv(exposure) 

2535 prompt = "Press Enter to continue [c]... " 

2536 while True: 

2537 ans = input(prompt).lower() 

2538 if ans in ("", "c",): 

2539 break 

2540 

2541 

2542class FakeAmp(object): 

2543 """A Detector-like object that supports returning gain and saturation level 

2544 

2545 This is used when the input exposure does not have a detector. 

2546 

2547 Parameters 

2548 ---------- 

2549 exposure : `lsst.afw.image.Exposure` 

2550 Exposure to generate a fake amplifier for. 

2551 config : `lsst.ip.isr.isrTaskConfig` 

2552 Configuration to apply to the fake amplifier. 

2553 """ 

2554 

2555 def __init__(self, exposure, config): 

2556 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2557 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2558 self._gain = config.gain 

2559 self._readNoise = config.readNoise 

2560 self._saturation = config.saturation 

2561 

2562 def getBBox(self): 

2563 return self._bbox 

2564 

2565 def getRawBBox(self): 

2566 return self._bbox 

2567 

2568 def getRawHorizontalOverscanBBox(self): 

2569 return self._RawHorizontalOverscanBBox 

2570 

2571 def getGain(self): 

2572 return self._gain 

2573 

2574 def getReadNoise(self): 

2575 return self._readNoise 

2576 

2577 def getSaturation(self): 

2578 return self._saturation 

2579 

2580 def getSuspectLevel(self): 

2581 return float("NaN") 

2582 

2583 

2584class RunIsrConfig(pexConfig.Config): 

2585 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2586 

2587 

2588class RunIsrTask(pipeBase.CmdLineTask): 

2589 """Task to wrap the default IsrTask to allow it to be retargeted. 

2590 

2591 The standard IsrTask can be called directly from a command line 

2592 program, but doing so removes the ability of the task to be 

2593 retargeted. As most cameras override some set of the IsrTask 

2594 methods, this would remove those data-specific methods in the 

2595 output post-ISR images. This wrapping class fixes the issue, 

2596 allowing identical post-ISR images to be generated by both the 

2597 processCcd and isrTask code. 

2598 """ 

2599 ConfigClass = RunIsrConfig 

2600 _DefaultName = "runIsr" 

2601 

2602 def __init__(self, *args, **kwargs): 

2603 super().__init__(*args, **kwargs) 

2604 self.makeSubtask("isr") 

2605 

2606 def runDataRef(self, dataRef): 

2607 """ 

2608 Parameters 

2609 ---------- 

2610 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2611 data reference of the detector data to be processed 

2612 

2613 Returns 

2614 ------- 

2615 result : `pipeBase.Struct` 

2616 Result struct with component: 

2617 

2618 - exposure : `lsst.afw.image.Exposure` 

2619 Post-ISR processed exposure. 

2620 """ 

2621 return self.isr.runDataRef(dataRef)