Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42from lsst.meas.algorithms import Defects 

43 

44from . import isrFunctions 

45from . import isrQa 

46from . import linearize 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 ) 

123 # TODO: DM-25348. This does not work yet to correctly load 

124 # possible crosstalk sources. 

125 crosstalkSources = cT.PrerequisiteInput( 

126 name="isrOverscanCorrected", 

127 doc="Overscan corrected input images.", 

128 storageClass="Exposure", 

129 dimensions=["instrument", "exposure", "detector"], 

130 deferLoad=True, 

131 multiple=True, 

132 lookupFunction=crosstalkSourceLookup, 

133 ) 

134 bias = cT.PrerequisiteInput( 

135 name="bias", 

136 doc="Input bias calibration.", 

137 storageClass="ExposureF", 

138 dimensions=["instrument", "detector"], 

139 isCalibration=True, 

140 ) 

141 dark = cT.PrerequisiteInput( 

142 name='dark', 

143 doc="Input dark calibration.", 

144 storageClass="ExposureF", 

145 dimensions=["instrument", "detector"], 

146 isCalibration=True, 

147 ) 

148 flat = cT.PrerequisiteInput( 

149 name="flat", 

150 doc="Input flat calibration.", 

151 storageClass="ExposureF", 

152 dimensions=["instrument", "physical_filter", "detector"], 

153 isCalibration=True, 

154 ) 

155 fringes = cT.PrerequisiteInput( 

156 name="fringe", 

157 doc="Input fringe calibration.", 

158 storageClass="ExposureF", 

159 dimensions=["instrument", "physical_filter", "detector"], 

160 isCalibration=True, 

161 ) 

162 strayLightData = cT.PrerequisiteInput( 

163 name='yBackground', 

164 doc="Input stray light calibration.", 

165 storageClass="StrayLightData", 

166 dimensions=["instrument", "physical_filter", "detector"], 

167 isCalibration=True, 

168 ) 

169 bfKernel = cT.PrerequisiteInput( 

170 name='bfKernel', 

171 doc="Input brighter-fatter kernel.", 

172 storageClass="NumpyArray", 

173 dimensions=["instrument"], 

174 isCalibration=True, 

175 ) 

176 newBFKernel = cT.PrerequisiteInput( 

177 name='brighterFatterKernel', 

178 doc="Newer complete kernel + gain solutions.", 

179 storageClass="BrighterFatterKernel", 

180 dimensions=["instrument", "detector"], 

181 isCalibration=True, 

182 ) 

183 defects = cT.PrerequisiteInput( 

184 name='defects', 

185 doc="Input defect tables.", 

186 storageClass="Defects", 

187 dimensions=["instrument", "detector"], 

188 isCalibration=True, 

189 ) 

190 opticsTransmission = cT.PrerequisiteInput( 

191 name="transmission_optics", 

192 storageClass="TransmissionCurve", 

193 doc="Transmission curve due to the optics.", 

194 dimensions=["instrument"], 

195 isCalibration=True, 

196 ) 

197 filterTransmission = cT.PrerequisiteInput( 

198 name="transmission_filter", 

199 storageClass="TransmissionCurve", 

200 doc="Transmission curve due to the filter.", 

201 dimensions=["instrument", "physical_filter"], 

202 isCalibration=True, 

203 ) 

204 sensorTransmission = cT.PrerequisiteInput( 

205 name="transmission_sensor", 

206 storageClass="TransmissionCurve", 

207 doc="Transmission curve due to the sensor.", 

208 dimensions=["instrument", "detector"], 

209 isCalibration=True, 

210 ) 

211 atmosphereTransmission = cT.PrerequisiteInput( 

212 name="transmission_atmosphere", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the atmosphere.", 

215 dimensions=["instrument"], 

216 isCalibration=True, 

217 ) 

218 illumMaskedImage = cT.PrerequisiteInput( 

219 name="illum", 

220 doc="Input illumination correction.", 

221 storageClass="MaskedImageF", 

222 dimensions=["instrument", "physical_filter", "detector"], 

223 isCalibration=True, 

224 ) 

225 

226 outputExposure = cT.Output( 

227 name='postISRCCD', 

228 doc="Output ISR processed exposure.", 

229 storageClass="Exposure", 

230 dimensions=["instrument", "exposure", "detector"], 

231 ) 

232 preInterpExposure = cT.Output( 

233 name='preInterpISRCCD', 

234 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

235 storageClass="ExposureF", 

236 dimensions=["instrument", "exposure", "detector"], 

237 ) 

238 outputOssThumbnail = cT.Output( 

239 name="OssThumb", 

240 doc="Output Overscan-subtracted thumbnail image.", 

241 storageClass="Thumbnail", 

242 dimensions=["instrument", "exposure", "detector"], 

243 ) 

244 outputFlattenedThumbnail = cT.Output( 

245 name="FlattenedThumb", 

246 doc="Output flat-corrected thumbnail image.", 

247 storageClass="Thumbnail", 

248 dimensions=["instrument", "exposure", "detector"], 

249 ) 

250 

251 def __init__(self, *, config=None): 

252 super().__init__(config=config) 

253 

254 if config.doBias is not True: 

255 self.prerequisiteInputs.discard("bias") 

256 if config.doLinearize is not True: 

257 self.prerequisiteInputs.discard("linearizer") 

258 if config.doCrosstalk is not True: 

259 self.inputs.discard("crosstalkSources") 

260 self.prerequisiteInputs.discard("crosstalk") 

261 if config.doBrighterFatter is not True: 

262 self.prerequisiteInputs.discard("bfKernel") 

263 self.prerequisiteInputs.discard("newBFKernel") 

264 if config.doDefect is not True: 

265 self.prerequisiteInputs.discard("defects") 

266 if config.doDark is not True: 

267 self.prerequisiteInputs.discard("dark") 

268 if config.doFlat is not True: 

269 self.prerequisiteInputs.discard("flat") 

270 if config.doAttachTransmissionCurve is not True: 

271 self.prerequisiteInputs.discard("opticsTransmission") 

272 self.prerequisiteInputs.discard("filterTransmission") 

273 self.prerequisiteInputs.discard("sensorTransmission") 

274 self.prerequisiteInputs.discard("atmosphereTransmission") 

275 if config.doUseOpticsTransmission is not True: 

276 self.prerequisiteInputs.discard("opticsTransmission") 

277 if config.doUseFilterTransmission is not True: 

278 self.prerequisiteInputs.discard("filterTransmission") 

279 if config.doUseSensorTransmission is not True: 

280 self.prerequisiteInputs.discard("sensorTransmission") 

281 if config.doUseAtmosphereTransmission is not True: 

282 self.prerequisiteInputs.discard("atmosphereTransmission") 

283 if config.doIlluminationCorrection is not True: 

284 self.prerequisiteInputs.discard("illumMaskedImage") 

285 

286 if config.doWrite is not True: 

287 self.outputs.discard("outputExposure") 

288 self.outputs.discard("preInterpExposure") 

289 self.outputs.discard("outputFlattenedThumbnail") 

290 self.outputs.discard("outputOssThumbnail") 

291 if config.doSaveInterpPixels is not True: 

292 self.outputs.discard("preInterpExposure") 

293 if config.qa.doThumbnailOss is not True: 

294 self.outputs.discard("outputOssThumbnail") 

295 if config.qa.doThumbnailFlattened is not True: 

296 self.outputs.discard("outputFlattenedThumbnail") 

297 

298 

299class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

300 pipelineConnections=IsrTaskConnections): 

301 """Configuration parameters for IsrTask. 

302 

303 Items are grouped in the order in which they are executed by the task. 

304 """ 

305 datasetType = pexConfig.Field( 

306 dtype=str, 

307 doc="Dataset type for input data; users will typically leave this alone, " 

308 "but camera-specific ISR tasks will override it", 

309 default="raw", 

310 ) 

311 

312 fallbackFilterName = pexConfig.Field( 

313 dtype=str, 

314 doc="Fallback default filter name for calibrations.", 

315 optional=True 

316 ) 

317 useFallbackDate = pexConfig.Field( 

318 dtype=bool, 

319 doc="Pass observation date when using fallback filter.", 

320 default=False, 

321 ) 

322 expectWcs = pexConfig.Field( 

323 dtype=bool, 

324 default=True, 

325 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

326 ) 

327 fwhm = pexConfig.Field( 

328 dtype=float, 

329 doc="FWHM of PSF in arcseconds.", 

330 default=1.0, 

331 ) 

332 qa = pexConfig.ConfigField( 

333 dtype=isrQa.IsrQaConfig, 

334 doc="QA related configuration options.", 

335 ) 

336 

337 # Image conversion configuration 

338 doConvertIntToFloat = pexConfig.Field( 

339 dtype=bool, 

340 doc="Convert integer raw images to floating point values?", 

341 default=True, 

342 ) 

343 

344 # Saturated pixel handling. 

345 doSaturation = pexConfig.Field( 

346 dtype=bool, 

347 doc="Mask saturated pixels? NB: this is totally independent of the" 

348 " interpolation option - this is ONLY setting the bits in the mask." 

349 " To have them interpolated make sure doSaturationInterpolation=True", 

350 default=True, 

351 ) 

352 saturatedMaskName = pexConfig.Field( 

353 dtype=str, 

354 doc="Name of mask plane to use in saturation detection and interpolation", 

355 default="SAT", 

356 ) 

357 saturation = pexConfig.Field( 

358 dtype=float, 

359 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

360 default=float("NaN"), 

361 ) 

362 growSaturationFootprintSize = pexConfig.Field( 

363 dtype=int, 

364 doc="Number of pixels by which to grow the saturation footprints", 

365 default=1, 

366 ) 

367 

368 # Suspect pixel handling. 

369 doSuspect = pexConfig.Field( 

370 dtype=bool, 

371 doc="Mask suspect pixels?", 

372 default=False, 

373 ) 

374 suspectMaskName = pexConfig.Field( 

375 dtype=str, 

376 doc="Name of mask plane to use for suspect pixels", 

377 default="SUSPECT", 

378 ) 

379 numEdgeSuspect = pexConfig.Field( 

380 dtype=int, 

381 doc="Number of edge pixels to be flagged as untrustworthy.", 

382 default=0, 

383 ) 

384 edgeMaskLevel = pexConfig.ChoiceField( 

385 dtype=str, 

386 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

387 default="DETECTOR", 

388 allowed={ 

389 'DETECTOR': 'Mask only the edges of the full detector.', 

390 'AMP': 'Mask edges of each amplifier.', 

391 }, 

392 ) 

393 

394 # Initial masking options. 

395 doSetBadRegions = pexConfig.Field( 

396 dtype=bool, 

397 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

398 default=True, 

399 ) 

400 badStatistic = pexConfig.ChoiceField( 

401 dtype=str, 

402 doc="How to estimate the average value for BAD regions.", 

403 default='MEANCLIP', 

404 allowed={ 

405 "MEANCLIP": "Correct using the (clipped) mean of good data", 

406 "MEDIAN": "Correct using the median of the good data", 

407 }, 

408 ) 

409 

410 # Overscan subtraction configuration. 

411 doOverscan = pexConfig.Field( 

412 dtype=bool, 

413 doc="Do overscan subtraction?", 

414 default=True, 

415 ) 

416 overscan = pexConfig.ConfigurableField( 

417 target=OverscanCorrectionTask, 

418 doc="Overscan subtraction task for image segments.", 

419 ) 

420 

421 overscanFitType = pexConfig.ChoiceField( 

422 dtype=str, 

423 doc="The method for fitting the overscan bias level.", 

424 default='MEDIAN', 

425 allowed={ 

426 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

427 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

428 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

429 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

430 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

431 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

432 "MEAN": "Correct using the mean of the overscan region", 

433 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

434 "MEDIAN": "Correct using the median of the overscan region", 

435 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

436 }, 

437 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

438 " This option will no longer be used, and will be removed after v20.") 

439 ) 

440 overscanOrder = pexConfig.Field( 

441 dtype=int, 

442 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

443 "or number of spline knots if overscan fit type is a spline."), 

444 default=1, 

445 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

446 " This option will no longer be used, and will be removed after v20.") 

447 ) 

448 overscanNumSigmaClip = pexConfig.Field( 

449 dtype=float, 

450 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

451 default=3.0, 

452 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

453 " This option will no longer be used, and will be removed after v20.") 

454 ) 

455 overscanIsInt = pexConfig.Field( 

456 dtype=bool, 

457 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

458 " and overscan.FitType=MEDIAN_PER_ROW.", 

459 default=True, 

460 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

461 " This option will no longer be used, and will be removed after v20.") 

462 ) 

463 # These options do not get deprecated, as they define how we slice up the image data. 

464 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

465 dtype=int, 

466 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

467 default=0, 

468 ) 

469 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

470 dtype=int, 

471 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

472 default=0, 

473 ) 

474 overscanMaxDev = pexConfig.Field( 474 ↛ exitline 474 didn't jump to the function exit

475 dtype=float, 

476 doc="Maximum deviation from the median for overscan", 

477 default=1000.0, check=lambda x: x > 0 

478 ) 

479 overscanBiasJump = pexConfig.Field( 

480 dtype=bool, 

481 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

482 default=False, 

483 ) 

484 overscanBiasJumpKeyword = pexConfig.Field( 

485 dtype=str, 

486 doc="Header keyword containing information about devices.", 

487 default="NO_SUCH_KEY", 

488 ) 

489 overscanBiasJumpDevices = pexConfig.ListField( 

490 dtype=str, 

491 doc="List of devices that need piecewise overscan correction.", 

492 default=(), 

493 ) 

494 overscanBiasJumpLocation = pexConfig.Field( 

495 dtype=int, 

496 doc="Location of bias jump along y-axis.", 

497 default=0, 

498 ) 

499 

500 # Amplifier to CCD assembly configuration 

501 doAssembleCcd = pexConfig.Field( 

502 dtype=bool, 

503 default=True, 

504 doc="Assemble amp-level exposures into a ccd-level exposure?" 

505 ) 

506 assembleCcd = pexConfig.ConfigurableField( 

507 target=AssembleCcdTask, 

508 doc="CCD assembly task", 

509 ) 

510 

511 # General calibration configuration. 

512 doAssembleIsrExposures = pexConfig.Field( 

513 dtype=bool, 

514 default=False, 

515 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

516 ) 

517 doTrimToMatchCalib = pexConfig.Field( 

518 dtype=bool, 

519 default=False, 

520 doc="Trim raw data to match calibration bounding boxes?" 

521 ) 

522 

523 # Bias subtraction. 

524 doBias = pexConfig.Field( 

525 dtype=bool, 

526 doc="Apply bias frame correction?", 

527 default=True, 

528 ) 

529 biasDataProductName = pexConfig.Field( 

530 dtype=str, 

531 doc="Name of the bias data product", 

532 default="bias", 

533 ) 

534 doBiasBeforeOverscan = pexConfig.Field( 

535 dtype=bool, 

536 doc="Reverse order of overscan and bias correction.", 

537 default=False 

538 ) 

539 

540 # Variance construction 

541 doVariance = pexConfig.Field( 

542 dtype=bool, 

543 doc="Calculate variance?", 

544 default=True 

545 ) 

546 gain = pexConfig.Field( 

547 dtype=float, 

548 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

549 default=float("NaN"), 

550 ) 

551 readNoise = pexConfig.Field( 

552 dtype=float, 

553 doc="The read noise to use if no Detector is present in the Exposure", 

554 default=0.0, 

555 ) 

556 doEmpiricalReadNoise = pexConfig.Field( 

557 dtype=bool, 

558 default=False, 

559 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

560 ) 

561 

562 # Linearization. 

563 doLinearize = pexConfig.Field( 

564 dtype=bool, 

565 doc="Correct for nonlinearity of the detector's response?", 

566 default=True, 

567 ) 

568 

569 # Crosstalk. 

570 doCrosstalk = pexConfig.Field( 

571 dtype=bool, 

572 doc="Apply intra-CCD crosstalk correction?", 

573 default=False, 

574 ) 

575 doCrosstalkBeforeAssemble = pexConfig.Field( 

576 dtype=bool, 

577 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

578 default=False, 

579 ) 

580 crosstalk = pexConfig.ConfigurableField( 

581 target=CrosstalkTask, 

582 doc="Intra-CCD crosstalk correction", 

583 ) 

584 

585 # Masking options. 

586 doDefect = pexConfig.Field( 

587 dtype=bool, 

588 doc="Apply correction for CCD defects, e.g. hot pixels?", 

589 default=True, 

590 ) 

591 doNanMasking = pexConfig.Field( 

592 dtype=bool, 

593 doc="Mask NAN pixels?", 

594 default=True, 

595 ) 

596 doWidenSaturationTrails = pexConfig.Field( 

597 dtype=bool, 

598 doc="Widen bleed trails based on their width?", 

599 default=True 

600 ) 

601 

602 # Brighter-Fatter correction. 

603 doBrighterFatter = pexConfig.Field( 

604 dtype=bool, 

605 default=False, 

606 doc="Apply the brighter fatter correction" 

607 ) 

608 brighterFatterLevel = pexConfig.ChoiceField( 

609 dtype=str, 

610 default="DETECTOR", 

611 doc="The level at which to correct for brighter-fatter.", 

612 allowed={ 

613 "AMP": "Every amplifier treated separately.", 

614 "DETECTOR": "One kernel per detector", 

615 } 

616 ) 

617 brighterFatterMaxIter = pexConfig.Field( 

618 dtype=int, 

619 default=10, 

620 doc="Maximum number of iterations for the brighter fatter correction" 

621 ) 

622 brighterFatterThreshold = pexConfig.Field( 

623 dtype=float, 

624 default=1000, 

625 doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

626 " absolute value of the difference between the current corrected image and the one" 

627 " from the previous iteration summed over all the pixels." 

628 ) 

629 brighterFatterApplyGain = pexConfig.Field( 

630 dtype=bool, 

631 default=True, 

632 doc="Should the gain be applied when applying the brighter fatter correction?" 

633 ) 

634 brighterFatterMaskGrowSize = pexConfig.Field( 

635 dtype=int, 

636 default=0, 

637 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate " 

638 " when brighter-fatter correction is applied." 

639 ) 

640 

641 # Dark subtraction. 

642 doDark = pexConfig.Field( 

643 dtype=bool, 

644 doc="Apply dark frame correction?", 

645 default=True, 

646 ) 

647 darkDataProductName = pexConfig.Field( 

648 dtype=str, 

649 doc="Name of the dark data product", 

650 default="dark", 

651 ) 

652 

653 # Camera-specific stray light removal. 

654 doStrayLight = pexConfig.Field( 

655 dtype=bool, 

656 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

657 default=False, 

658 ) 

659 strayLight = pexConfig.ConfigurableField( 

660 target=StrayLightTask, 

661 doc="y-band stray light correction" 

662 ) 

663 

664 # Flat correction. 

665 doFlat = pexConfig.Field( 

666 dtype=bool, 

667 doc="Apply flat field correction?", 

668 default=True, 

669 ) 

670 flatDataProductName = pexConfig.Field( 

671 dtype=str, 

672 doc="Name of the flat data product", 

673 default="flat", 

674 ) 

675 flatScalingType = pexConfig.ChoiceField( 

676 dtype=str, 

677 doc="The method for scaling the flat on the fly.", 

678 default='USER', 

679 allowed={ 

680 "USER": "Scale by flatUserScale", 

681 "MEAN": "Scale by the inverse of the mean", 

682 "MEDIAN": "Scale by the inverse of the median", 

683 }, 

684 ) 

685 flatUserScale = pexConfig.Field( 

686 dtype=float, 

687 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

688 default=1.0, 

689 ) 

690 doTweakFlat = pexConfig.Field( 

691 dtype=bool, 

692 doc="Tweak flats to match observed amplifier ratios?", 

693 default=False 

694 ) 

695 

696 # Amplifier normalization based on gains instead of using flats configuration. 

697 doApplyGains = pexConfig.Field( 

698 dtype=bool, 

699 doc="Correct the amplifiers for their gains instead of applying flat correction", 

700 default=False, 

701 ) 

702 normalizeGains = pexConfig.Field( 

703 dtype=bool, 

704 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

705 default=False, 

706 ) 

707 

708 # Fringe correction. 

709 doFringe = pexConfig.Field( 

710 dtype=bool, 

711 doc="Apply fringe correction?", 

712 default=True, 

713 ) 

714 fringe = pexConfig.ConfigurableField( 

715 target=FringeTask, 

716 doc="Fringe subtraction task", 

717 ) 

718 fringeAfterFlat = pexConfig.Field( 

719 dtype=bool, 

720 doc="Do fringe subtraction after flat-fielding?", 

721 default=True, 

722 ) 

723 

724 # Initial CCD-level background statistics options. 

725 doMeasureBackground = pexConfig.Field( 

726 dtype=bool, 

727 doc="Measure the background level on the reduced image?", 

728 default=False, 

729 ) 

730 

731 # Camera-specific masking configuration. 

732 doCameraSpecificMasking = pexConfig.Field( 

733 dtype=bool, 

734 doc="Mask camera-specific bad regions?", 

735 default=False, 

736 ) 

737 masking = pexConfig.ConfigurableField( 

738 target=MaskingTask, 

739 doc="Masking task." 

740 ) 

741 

742 # Interpolation options. 

743 

744 doInterpolate = pexConfig.Field( 

745 dtype=bool, 

746 doc="Interpolate masked pixels?", 

747 default=True, 

748 ) 

749 doSaturationInterpolation = pexConfig.Field( 

750 dtype=bool, 

751 doc="Perform interpolation over pixels masked as saturated?" 

752 " NB: This is independent of doSaturation; if that is False this plane" 

753 " will likely be blank, resulting in a no-op here.", 

754 default=True, 

755 ) 

756 doNanInterpolation = pexConfig.Field( 

757 dtype=bool, 

758 doc="Perform interpolation over pixels masked as NaN?" 

759 " NB: This is independent of doNanMasking; if that is False this plane" 

760 " will likely be blank, resulting in a no-op here.", 

761 default=True, 

762 ) 

763 doNanInterpAfterFlat = pexConfig.Field( 

764 dtype=bool, 

765 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

766 "also have to interpolate them before flat-fielding."), 

767 default=False, 

768 ) 

769 maskListToInterpolate = pexConfig.ListField( 

770 dtype=str, 

771 doc="List of mask planes that should be interpolated.", 

772 default=['SAT', 'BAD', 'UNMASKEDNAN'], 

773 ) 

774 doSaveInterpPixels = pexConfig.Field( 

775 dtype=bool, 

776 doc="Save a copy of the pre-interpolated pixel values?", 

777 default=False, 

778 ) 

779 

780 # Default photometric calibration options. 

781 fluxMag0T1 = pexConfig.DictField( 

782 keytype=str, 

783 itemtype=float, 

784 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

785 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

786 )) 

787 ) 

788 defaultFluxMag0T1 = pexConfig.Field( 

789 dtype=float, 

790 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

791 default=pow(10.0, 0.4*28.0) 

792 ) 

793 

794 # Vignette correction configuration. 

795 doVignette = pexConfig.Field( 

796 dtype=bool, 

797 doc="Apply vignetting parameters?", 

798 default=False, 

799 ) 

800 vignette = pexConfig.ConfigurableField( 

801 target=VignetteTask, 

802 doc="Vignetting task.", 

803 ) 

804 

805 # Transmission curve configuration. 

806 doAttachTransmissionCurve = pexConfig.Field( 

807 dtype=bool, 

808 default=False, 

809 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

810 ) 

811 doUseOpticsTransmission = pexConfig.Field( 

812 dtype=bool, 

813 default=True, 

814 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

815 ) 

816 doUseFilterTransmission = pexConfig.Field( 

817 dtype=bool, 

818 default=True, 

819 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

820 ) 

821 doUseSensorTransmission = pexConfig.Field( 

822 dtype=bool, 

823 default=True, 

824 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

825 ) 

826 doUseAtmosphereTransmission = pexConfig.Field( 

827 dtype=bool, 

828 default=True, 

829 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

830 ) 

831 

832 # Illumination correction. 

833 doIlluminationCorrection = pexConfig.Field( 

834 dtype=bool, 

835 default=False, 

836 doc="Perform illumination correction?" 

837 ) 

838 illuminationCorrectionDataProductName = pexConfig.Field( 

839 dtype=str, 

840 doc="Name of the illumination correction data product.", 

841 default="illumcor", 

842 ) 

843 illumScale = pexConfig.Field( 

844 dtype=float, 

845 doc="Scale factor for the illumination correction.", 

846 default=1.0, 

847 ) 

848 illumFilters = pexConfig.ListField( 

849 dtype=str, 

850 default=[], 

851 doc="Only perform illumination correction for these filters." 

852 ) 

853 

854 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

855 doWrite = pexConfig.Field( 

856 dtype=bool, 

857 doc="Persist postISRCCD?", 

858 default=True, 

859 ) 

860 

861 def validate(self): 

862 super().validate() 

863 if self.doFlat and self.doApplyGains: 

864 raise ValueError("You may not specify both doFlat and doApplyGains") 

865 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

866 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

867 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate: 

868 self.config.maskListToInterpolate.append("SAT") 

869 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

870 self.config.maskListToInterpolate.append("UNMASKEDNAN") 

871 

872 

873class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

874 """Apply common instrument signature correction algorithms to a raw frame. 

875 

876 The process for correcting imaging data is very similar from 

877 camera to camera. This task provides a vanilla implementation of 

878 doing these corrections, including the ability to turn certain 

879 corrections off if they are not needed. The inputs to the primary 

880 method, `run()`, are a raw exposure to be corrected and the 

881 calibration data products. The raw input is a single chip sized 

882 mosaic of all amps including overscans and other non-science 

883 pixels. The method `runDataRef()` identifies and defines the 

884 calibration data products, and is intended for use by a 

885 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

886 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

887 subclassed for different camera, although the most camera specific 

888 methods have been split into subtasks that can be redirected 

889 appropriately. 

890 

891 The __init__ method sets up the subtasks for ISR processing, using 

892 the defaults from `lsst.ip.isr`. 

893 

894 Parameters 

895 ---------- 

896 args : `list` 

897 Positional arguments passed to the Task constructor. None used at this time. 

898 kwargs : `dict`, optional 

899 Keyword arguments passed on to the Task constructor. None used at this time. 

900 """ 

901 ConfigClass = IsrTaskConfig 

902 _DefaultName = "isr" 

903 

904 def __init__(self, **kwargs): 

905 super().__init__(**kwargs) 

906 self.makeSubtask("assembleCcd") 

907 self.makeSubtask("crosstalk") 

908 self.makeSubtask("strayLight") 

909 self.makeSubtask("fringe") 

910 self.makeSubtask("masking") 

911 self.makeSubtask("overscan") 

912 self.makeSubtask("vignette") 

913 

914 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

915 inputs = butlerQC.get(inputRefs) 

916 

917 try: 

918 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

919 except Exception as e: 

920 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

921 (inputRefs, e)) 

922 

923 inputs['isGen3'] = True 

924 

925 detector = inputs['ccdExposure'].getDetector() 

926 

927 if self.config.doCrosstalk is True: 

928 # Crosstalk sources need to be defined by the pipeline 

929 # yaml if they exist. 

930 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

931 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

932 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

933 else: 

934 coeffVector = (self.config.crosstalk.crosstalkValues 

935 if self.config.crosstalk.useConfigCoefficients else None) 

936 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

937 inputs['crosstalk'] = crosstalkCalib 

938 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

939 if 'crosstalkSources' not in inputs: 

940 self.log.warn("No crosstalkSources found for chip with interChip terms!") 

941 

942 if self.doLinearize(detector) is True: 

943 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict): 

944 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

945 linearizer.fromYaml(inputs['linearizer']) 

946 else: 

947 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector, 

948 log=self.log) 

949 inputs['linearizer'] = linearizer 

950 

951 if self.config.doDefect is True: 

952 if "defects" in inputs and inputs['defects'] is not None: 

953 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

954 # masking expects a list of defects defined by their bounding box 

955 if not isinstance(inputs["defects"], Defects): 

956 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

957 

958 # Load the correct style of brighter fatter kernel, and repack 

959 # the information as a numpy array. 

960 if self.config.doBrighterFatter: 

961 brighterFatterKernel = inputs.pop('newBFKernel', None) 

962 if brighterFatterKernel is None: 

963 brighterFatterKernel = inputs.get('bfKernel', None) 

964 

965 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

966 detId = detector.getId() 

967 inputs['bfGains'] = brighterFatterKernel.gain 

968 # If the kernel is not an ndarray, it's the cp_pipe version 

969 # so extract the kernel for this detector, or raise an error 

970 if self.config.brighterFatterLevel == 'DETECTOR': 

971 if brighterFatterKernel.detectorKernel: 

972 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId] 

973 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

974 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId] 

975 else: 

976 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

977 else: 

978 # TODO DM-15631 for implementing this 

979 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

980 

981 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

982 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

983 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

984 expId=expId, 

985 assembler=self.assembleCcd 

986 if self.config.doAssembleIsrExposures else None) 

987 else: 

988 inputs['fringes'] = pipeBase.Struct(fringes=None) 

989 

990 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

991 if 'strayLightData' not in inputs: 

992 inputs['strayLightData'] = None 

993 

994 outputs = self.run(**inputs) 

995 butlerQC.put(outputs, outputRefs) 

996 

997 def readIsrData(self, dataRef, rawExposure): 

998 """Retrieve necessary frames for instrument signature removal. 

999 

1000 Pre-fetching all required ISR data products limits the IO 

1001 required by the ISR. Any conflict between the calibration data 

1002 available and that needed for ISR is also detected prior to 

1003 doing processing, allowing it to fail quickly. 

1004 

1005 Parameters 

1006 ---------- 

1007 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1008 Butler reference of the detector data to be processed 

1009 rawExposure : `afw.image.Exposure` 

1010 The raw exposure that will later be corrected with the 

1011 retrieved calibration data; should not be modified in this 

1012 method. 

1013 

1014 Returns 

1015 ------- 

1016 result : `lsst.pipe.base.Struct` 

1017 Result struct with components (which may be `None`): 

1018 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1019 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1020 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1021 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1022 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1023 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1024 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 

1025 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1026 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1027 - ``seed``: random seed derived from the ccdExposureId for random 

1028 number generator (`uint32`). 

1029 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1030 A ``TransmissionCurve`` that represents the throughput of the optics, 

1031 to be evaluated in focal-plane coordinates. 

1032 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1033 A ``TransmissionCurve`` that represents the throughput of the filter 

1034 itself, to be evaluated in focal-plane coordinates. 

1035 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1036 A ``TransmissionCurve`` that represents the throughput of the sensor 

1037 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1038 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1039 A ``TransmissionCurve`` that represents the throughput of the 

1040 atmosphere, assumed to be spatially constant. 

1041 - ``strayLightData`` : `object` 

1042 An opaque object containing calibration information for 

1043 stray-light correction. If `None`, no correction will be 

1044 performed. 

1045 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1046 

1047 Raises 

1048 ------ 

1049 NotImplementedError : 

1050 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1051 """ 

1052 try: 

1053 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1054 dateObs = dateObs.toPython().isoformat() 

1055 except RuntimeError: 

1056 self.log.warn("Unable to identify dateObs for rawExposure.") 

1057 dateObs = None 

1058 

1059 ccd = rawExposure.getDetector() 

1060 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter 

1061 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1062 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1063 if self.config.doBias else None) 

1064 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1065 linearizer = (dataRef.get("linearizer", immediate=True) 

1066 if self.doLinearize(ccd) else None) 

1067 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1068 linearizer.log = self.log 

1069 if isinstance(linearizer, numpy.ndarray): 

1070 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1071 

1072 crosstalkCalib = None 

1073 if self.config.doCrosstalk: 

1074 try: 

1075 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1076 except NoResults: 

1077 coeffVector = (self.config.crosstalk.crosstalkValues 

1078 if self.config.crosstalk.useConfigCoefficients else None) 

1079 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1080 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1081 if self.config.doCrosstalk else None) 

1082 

1083 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1084 if self.config.doDark else None) 

1085 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1086 dateObs=dateObs) 

1087 if self.config.doFlat else None) 

1088 

1089 brighterFatterKernel = None 

1090 brighterFatterGains = None 

1091 if self.config.doBrighterFatter is True: 

1092 try: 

1093 # Use the new-style cp_pipe version of the kernel if it exists 

1094 # If using a new-style kernel, always use the self-consistent 

1095 # gains, i.e. the ones inside the kernel object itself 

1096 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1097 brighterFatterGains = brighterFatterKernel.gain 

1098 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded") 

1099 except NoResults: 

1100 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1101 brighterFatterKernel = dataRef.get("bfKernel") 

1102 self.log.info("Old style bright-fatter kernel (np.array) loaded") 

1103 except NoResults: 

1104 brighterFatterKernel = None 

1105 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1106 # If the kernel is not an ndarray, it's the cp_pipe version 

1107 # so extract the kernel for this detector, or raise an error 

1108 if self.config.brighterFatterLevel == 'DETECTOR': 

1109 if brighterFatterKernel.detectorKernel: 

1110 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()] 

1111 elif brighterFatterKernel.detectorKernelFromAmpKernels: 

1112 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()] 

1113 else: 

1114 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1115 else: 

1116 # TODO DM-15631 for implementing this 

1117 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1118 

1119 defectList = (dataRef.get("defects") 

1120 if self.config.doDefect else None) 

1121 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

1122 if self.config.doAssembleIsrExposures else None) 

1123 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1124 else pipeBase.Struct(fringes=None)) 

1125 

1126 if self.config.doAttachTransmissionCurve: 

1127 opticsTransmission = (dataRef.get("transmission_optics") 

1128 if self.config.doUseOpticsTransmission else None) 

1129 filterTransmission = (dataRef.get("transmission_filter") 

1130 if self.config.doUseFilterTransmission else None) 

1131 sensorTransmission = (dataRef.get("transmission_sensor") 

1132 if self.config.doUseSensorTransmission else None) 

1133 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1134 if self.config.doUseAtmosphereTransmission else None) 

1135 else: 

1136 opticsTransmission = None 

1137 filterTransmission = None 

1138 sensorTransmission = None 

1139 atmosphereTransmission = None 

1140 

1141 if self.config.doStrayLight: 

1142 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1143 else: 

1144 strayLightData = None 

1145 

1146 illumMaskedImage = (self.getIsrExposure(dataRef, 

1147 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1148 if (self.config.doIlluminationCorrection 

1149 and filterName in self.config.illumFilters) 

1150 else None) 

1151 

1152 # Struct should include only kwargs to run() 

1153 return pipeBase.Struct(bias=biasExposure, 

1154 linearizer=linearizer, 

1155 crosstalk=crosstalkCalib, 

1156 crosstalkSources=crosstalkSources, 

1157 dark=darkExposure, 

1158 flat=flatExposure, 

1159 bfKernel=brighterFatterKernel, 

1160 bfGains=brighterFatterGains, 

1161 defects=defectList, 

1162 fringes=fringeStruct, 

1163 opticsTransmission=opticsTransmission, 

1164 filterTransmission=filterTransmission, 

1165 sensorTransmission=sensorTransmission, 

1166 atmosphereTransmission=atmosphereTransmission, 

1167 strayLightData=strayLightData, 

1168 illumMaskedImage=illumMaskedImage 

1169 ) 

1170 

1171 @pipeBase.timeMethod 

1172 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, 

1173 crosstalk=None, crosstalkSources=None, 

1174 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, 

1175 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1176 sensorTransmission=None, atmosphereTransmission=None, 

1177 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1178 isGen3=False, 

1179 ): 

1180 """Perform instrument signature removal on an exposure. 

1181 

1182 Steps included in the ISR processing, in order performed, are: 

1183 - saturation and suspect pixel masking 

1184 - overscan subtraction 

1185 - CCD assembly of individual amplifiers 

1186 - bias subtraction 

1187 - variance image construction 

1188 - linearization of non-linear response 

1189 - crosstalk masking 

1190 - brighter-fatter correction 

1191 - dark subtraction 

1192 - fringe correction 

1193 - stray light subtraction 

1194 - flat correction 

1195 - masking of known defects and camera specific features 

1196 - vignette calculation 

1197 - appending transmission curve and distortion model 

1198 

1199 Parameters 

1200 ---------- 

1201 ccdExposure : `lsst.afw.image.Exposure` 

1202 The raw exposure that is to be run through ISR. The 

1203 exposure is modified by this method. 

1204 camera : `lsst.afw.cameraGeom.Camera`, optional 

1205 The camera geometry for this exposure. Required if ``isGen3`` is 

1206 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1207 ``flat`` does not have an associated detector. 

1208 bias : `lsst.afw.image.Exposure`, optional 

1209 Bias calibration frame. 

1210 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1211 Functor for linearization. 

1212 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1213 Calibration for crosstalk. 

1214 crosstalkSources : `list`, optional 

1215 List of possible crosstalk sources. 

1216 dark : `lsst.afw.image.Exposure`, optional 

1217 Dark calibration frame. 

1218 flat : `lsst.afw.image.Exposure`, optional 

1219 Flat calibration frame. 

1220 bfKernel : `numpy.ndarray`, optional 

1221 Brighter-fatter kernel. 

1222 bfGains : `dict` of `float`, optional 

1223 Gains used to override the detector's nominal gains for the 

1224 brighter-fatter correction. A dict keyed by amplifier name for 

1225 the detector in question. 

1226 defects : `lsst.meas.algorithms.Defects`, optional 

1227 List of defects. 

1228 fringes : `lsst.pipe.base.Struct`, optional 

1229 Struct containing the fringe correction data, with 

1230 elements: 

1231 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1232 - ``seed``: random seed derived from the ccdExposureId for random 

1233 number generator (`uint32`) 

1234 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1235 A ``TransmissionCurve`` that represents the throughput of the optics, 

1236 to be evaluated in focal-plane coordinates. 

1237 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1238 A ``TransmissionCurve`` that represents the throughput of the filter 

1239 itself, to be evaluated in focal-plane coordinates. 

1240 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1241 A ``TransmissionCurve`` that represents the throughput of the sensor 

1242 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1243 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1244 A ``TransmissionCurve`` that represents the throughput of the 

1245 atmosphere, assumed to be spatially constant. 

1246 detectorNum : `int`, optional 

1247 The integer number for the detector to process. 

1248 isGen3 : bool, optional 

1249 Flag this call to run() as using the Gen3 butler environment. 

1250 strayLightData : `object`, optional 

1251 Opaque object containing calibration information for stray-light 

1252 correction. If `None`, no correction will be performed. 

1253 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1254 Illumination correction image. 

1255 

1256 Returns 

1257 ------- 

1258 result : `lsst.pipe.base.Struct` 

1259 Result struct with component: 

1260 - ``exposure`` : `afw.image.Exposure` 

1261 The fully ISR corrected exposure. 

1262 - ``outputExposure`` : `afw.image.Exposure` 

1263 An alias for `exposure` 

1264 - ``ossThumb`` : `numpy.ndarray` 

1265 Thumbnail image of the exposure after overscan subtraction. 

1266 - ``flattenedThumb`` : `numpy.ndarray` 

1267 Thumbnail image of the exposure after flat-field correction. 

1268 

1269 Raises 

1270 ------ 

1271 RuntimeError 

1272 Raised if a configuration option is set to True, but the 

1273 required calibration data has not been specified. 

1274 

1275 Notes 

1276 ----- 

1277 The current processed exposure can be viewed by setting the 

1278 appropriate lsstDebug entries in the `debug.display` 

1279 dictionary. The names of these entries correspond to some of 

1280 the IsrTaskConfig Boolean options, with the value denoting the 

1281 frame to use. The exposure is shown inside the matching 

1282 option check and after the processing of that step has 

1283 finished. The steps with debug points are: 

1284 

1285 doAssembleCcd 

1286 doBias 

1287 doCrosstalk 

1288 doBrighterFatter 

1289 doDark 

1290 doFringe 

1291 doStrayLight 

1292 doFlat 

1293 

1294 In addition, setting the "postISRCCD" entry displays the 

1295 exposure after all ISR processing has finished. 

1296 

1297 """ 

1298 

1299 if isGen3 is True: 

1300 # Gen3 currently cannot automatically do configuration overrides. 

1301 # DM-15257 looks to discuss this issue. 

1302 # Configure input exposures; 

1303 if detectorNum is None: 

1304 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1305 

1306 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1307 bias = self.ensureExposure(bias, camera, detectorNum) 

1308 dark = self.ensureExposure(dark, camera, detectorNum) 

1309 flat = self.ensureExposure(flat, camera, detectorNum) 

1310 else: 

1311 if isinstance(ccdExposure, ButlerDataRef): 

1312 return self.runDataRef(ccdExposure) 

1313 

1314 ccd = ccdExposure.getDetector() 

1315 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter 

1316 

1317 if not ccd: 

1318 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1319 ccd = [FakeAmp(ccdExposure, self.config)] 

1320 

1321 # Validate Input 

1322 if self.config.doBias and bias is None: 

1323 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1324 if self.doLinearize(ccd) and linearizer is None: 

1325 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1326 if self.config.doBrighterFatter and bfKernel is None: 

1327 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1328 if self.config.doDark and dark is None: 

1329 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1330 if self.config.doFlat and flat is None: 

1331 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1332 if self.config.doDefect and defects is None: 

1333 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1334 if (self.config.doFringe and filterName in self.fringe.config.filters 

1335 and fringes.fringes is None): 

1336 # The `fringes` object needs to be a pipeBase.Struct, as 

1337 # we use it as a `dict` for the parameters of 

1338 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1339 # not be `None` if `doFringe=True`. Otherwise, raise. 

1340 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1341 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters 

1342 and illumMaskedImage is None): 

1343 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1344 

1345 # Begin ISR processing. 

1346 if self.config.doConvertIntToFloat: 

1347 self.log.info("Converting exposure to floating point values.") 

1348 ccdExposure = self.convertIntToFloat(ccdExposure) 

1349 

1350 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1351 self.log.info("Applying bias correction.") 

1352 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1353 trimToFit=self.config.doTrimToMatchCalib) 

1354 self.debugView(ccdExposure, "doBias") 

1355 

1356 # Amplifier level processing. 

1357 overscans = [] 

1358 for amp in ccd: 

1359 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1360 if ccdExposure.getBBox().contains(amp.getBBox()): 

1361 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1362 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1363 

1364 if self.config.doOverscan and not badAmp: 

1365 # Overscan correction on amp-by-amp basis. 

1366 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1367 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1368 if overscanResults is not None and \ 

1369 self.config.qa is not None and self.config.qa.saveStats is True: 

1370 if isinstance(overscanResults.overscanFit, float): 

1371 qaMedian = overscanResults.overscanFit 

1372 qaStdev = float("NaN") 

1373 else: 

1374 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1375 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1376 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1377 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1378 

1379 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian) 

1380 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev) 

1381 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1382 amp.getName(), qaMedian, qaStdev) 

1383 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1384 else: 

1385 if badAmp: 

1386 self.log.warn("Amplifier %s is bad.", amp.getName()) 

1387 overscanResults = None 

1388 

1389 overscans.append(overscanResults if overscanResults is not None else None) 

1390 else: 

1391 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1392 

1393 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1394 self.log.info("Applying crosstalk correction.") 

1395 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1396 crosstalkSources=crosstalkSources) 

1397 self.debugView(ccdExposure, "doCrosstalk") 

1398 

1399 if self.config.doAssembleCcd: 

1400 self.log.info("Assembling CCD from amplifiers.") 

1401 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1402 

1403 if self.config.expectWcs and not ccdExposure.getWcs(): 

1404 self.log.warn("No WCS found in input exposure.") 

1405 self.debugView(ccdExposure, "doAssembleCcd") 

1406 

1407 ossThumb = None 

1408 if self.config.qa.doThumbnailOss: 

1409 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1410 

1411 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1412 self.log.info("Applying bias correction.") 

1413 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1414 trimToFit=self.config.doTrimToMatchCalib) 

1415 self.debugView(ccdExposure, "doBias") 

1416 

1417 if self.config.doVariance: 

1418 for amp, overscanResults in zip(ccd, overscans): 

1419 if ccdExposure.getBBox().contains(amp.getBBox()): 

1420 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1421 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1422 if overscanResults is not None: 

1423 self.updateVariance(ampExposure, amp, 

1424 overscanImage=overscanResults.overscanImage) 

1425 else: 

1426 self.updateVariance(ampExposure, amp, 

1427 overscanImage=None) 

1428 if self.config.qa is not None and self.config.qa.saveStats is True: 

1429 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1430 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1431 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1432 qaStats.getValue(afwMath.MEDIAN)) 

1433 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1434 qaStats.getValue(afwMath.STDEVCLIP)) 

1435 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1436 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1437 qaStats.getValue(afwMath.STDEVCLIP)) 

1438 

1439 if self.doLinearize(ccd): 

1440 self.log.info("Applying linearizer.") 

1441 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1442 detector=ccd, log=self.log) 

1443 

1444 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1445 self.log.info("Applying crosstalk correction.") 

1446 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1447 crosstalkSources=crosstalkSources, isTrimmed=True) 

1448 self.debugView(ccdExposure, "doCrosstalk") 

1449 

1450 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do 

1451 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1452 if self.config.doDefect: 

1453 self.log.info("Masking defects.") 

1454 self.maskDefect(ccdExposure, defects) 

1455 

1456 if self.config.numEdgeSuspect > 0: 

1457 self.log.info("Masking edges as SUSPECT.") 

1458 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1459 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1460 

1461 if self.config.doNanMasking: 

1462 self.log.info("Masking NAN value pixels.") 

1463 self.maskNan(ccdExposure) 

1464 

1465 if self.config.doWidenSaturationTrails: 

1466 self.log.info("Widening saturation trails.") 

1467 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1468 

1469 if self.config.doCameraSpecificMasking: 

1470 self.log.info("Masking regions for camera specific reasons.") 

1471 self.masking.run(ccdExposure) 

1472 

1473 if self.config.doBrighterFatter: 

1474 # We need to apply flats and darks before we can interpolate, and we 

1475 # need to interpolate before we do B-F, but we do B-F without the 

1476 # flats and darks applied so we can work in units of electrons or holes. 

1477 # This context manager applies and then removes the darks and flats. 

1478 # 

1479 # We also do not want to interpolate values here, so operate on temporary 

1480 # images so we can apply only the BF-correction and roll back the 

1481 # interpolation. 

1482 interpExp = ccdExposure.clone() 

1483 with self.flatContext(interpExp, flat, dark): 

1484 isrFunctions.interpolateFromMask( 

1485 maskedImage=interpExp.getMaskedImage(), 

1486 fwhm=self.config.fwhm, 

1487 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1488 maskNameList=self.config.maskListToInterpolate 

1489 ) 

1490 bfExp = interpExp.clone() 

1491 

1492 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.", 

1493 type(bfKernel), type(bfGains)) 

1494 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1495 self.config.brighterFatterMaxIter, 

1496 self.config.brighterFatterThreshold, 

1497 self.config.brighterFatterApplyGain, 

1498 bfGains) 

1499 if bfResults[1] == self.config.brighterFatterMaxIter: 

1500 self.log.warn("Brighter fatter correction did not converge, final difference %f.", 

1501 bfResults[0]) 

1502 else: 

1503 self.log.info("Finished brighter fatter correction in %d iterations.", 

1504 bfResults[1]) 

1505 image = ccdExposure.getMaskedImage().getImage() 

1506 bfCorr = bfExp.getMaskedImage().getImage() 

1507 bfCorr -= interpExp.getMaskedImage().getImage() 

1508 image += bfCorr 

1509 

1510 # Applying the brighter-fatter correction applies a 

1511 # convolution to the science image. At the edges this 

1512 # convolution may not have sufficient valid pixels to 

1513 # produce a valid correction. Mark pixels within the size 

1514 # of the brighter-fatter kernel as EDGE to warn of this 

1515 # fact. 

1516 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.") 

1517 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1518 maskPlane="EDGE") 

1519 

1520 if self.config.brighterFatterMaskGrowSize > 0: 

1521 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1522 for maskPlane in self.config.maskListToInterpolate: 

1523 isrFunctions.growMasks(ccdExposure.getMask(), 

1524 radius=self.config.brighterFatterMaskGrowSize, 

1525 maskNameList=maskPlane, 

1526 maskValue=maskPlane) 

1527 

1528 self.debugView(ccdExposure, "doBrighterFatter") 

1529 

1530 if self.config.doDark: 

1531 self.log.info("Applying dark correction.") 

1532 self.darkCorrection(ccdExposure, dark) 

1533 self.debugView(ccdExposure, "doDark") 

1534 

1535 if self.config.doFringe and not self.config.fringeAfterFlat: 

1536 self.log.info("Applying fringe correction before flat.") 

1537 self.fringe.run(ccdExposure, **fringes.getDict()) 

1538 self.debugView(ccdExposure, "doFringe") 

1539 

1540 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1541 self.log.info("Checking strayLight correction.") 

1542 self.strayLight.run(ccdExposure, strayLightData) 

1543 self.debugView(ccdExposure, "doStrayLight") 

1544 

1545 if self.config.doFlat: 

1546 self.log.info("Applying flat correction.") 

1547 self.flatCorrection(ccdExposure, flat) 

1548 self.debugView(ccdExposure, "doFlat") 

1549 

1550 if self.config.doApplyGains: 

1551 self.log.info("Applying gain correction instead of flat.") 

1552 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1553 

1554 if self.config.doFringe and self.config.fringeAfterFlat: 

1555 self.log.info("Applying fringe correction after flat.") 

1556 self.fringe.run(ccdExposure, **fringes.getDict()) 

1557 

1558 if self.config.doVignette: 

1559 self.log.info("Constructing Vignette polygon.") 

1560 self.vignettePolygon = self.vignette.run(ccdExposure) 

1561 

1562 if self.config.vignette.doWriteVignettePolygon: 

1563 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1564 

1565 if self.config.doAttachTransmissionCurve: 

1566 self.log.info("Adding transmission curves.") 

1567 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1568 filterTransmission=filterTransmission, 

1569 sensorTransmission=sensorTransmission, 

1570 atmosphereTransmission=atmosphereTransmission) 

1571 

1572 flattenedThumb = None 

1573 if self.config.qa.doThumbnailFlattened: 

1574 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1575 

1576 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters: 

1577 self.log.info("Performing illumination correction.") 

1578 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1579 illumMaskedImage, illumScale=self.config.illumScale, 

1580 trimToFit=self.config.doTrimToMatchCalib) 

1581 

1582 preInterpExp = None 

1583 if self.config.doSaveInterpPixels: 

1584 preInterpExp = ccdExposure.clone() 

1585 

1586 # Reset and interpolate bad pixels. 

1587 # 

1588 # Large contiguous bad regions (which should have the BAD mask 

1589 # bit set) should have their values set to the image median. 

1590 # This group should include defects and bad amplifiers. As the 

1591 # area covered by these defects are large, there's little 

1592 # reason to expect that interpolation would provide a more 

1593 # useful value. 

1594 # 

1595 # Smaller defects can be safely interpolated after the larger 

1596 # regions have had their pixel values reset. This ensures 

1597 # that the remaining defects adjacent to bad amplifiers (as an 

1598 # example) do not attempt to interpolate extreme values. 

1599 if self.config.doSetBadRegions: 

1600 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1601 if badPixelCount > 0: 

1602 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1603 

1604 if self.config.doInterpolate: 

1605 self.log.info("Interpolating masked pixels.") 

1606 isrFunctions.interpolateFromMask( 

1607 maskedImage=ccdExposure.getMaskedImage(), 

1608 fwhm=self.config.fwhm, 

1609 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1610 maskNameList=list(self.config.maskListToInterpolate) 

1611 ) 

1612 

1613 self.roughZeroPoint(ccdExposure) 

1614 

1615 if self.config.doMeasureBackground: 

1616 self.log.info("Measuring background level.") 

1617 self.measureBackground(ccdExposure, self.config.qa) 

1618 

1619 if self.config.qa is not None and self.config.qa.saveStats is True: 

1620 for amp in ccd: 

1621 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1622 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1623 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1624 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1625 qaStats.getValue(afwMath.MEDIAN)) 

1626 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1627 qaStats.getValue(afwMath.STDEVCLIP)) 

1628 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1629 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1630 qaStats.getValue(afwMath.STDEVCLIP)) 

1631 

1632 self.debugView(ccdExposure, "postISRCCD") 

1633 

1634 return pipeBase.Struct( 

1635 exposure=ccdExposure, 

1636 ossThumb=ossThumb, 

1637 flattenedThumb=flattenedThumb, 

1638 

1639 preInterpolatedExposure=preInterpExp, 

1640 outputExposure=ccdExposure, 

1641 outputOssThumbnail=ossThumb, 

1642 outputFlattenedThumbnail=flattenedThumb, 

1643 ) 

1644 

1645 @pipeBase.timeMethod 

1646 def runDataRef(self, sensorRef): 

1647 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1648 

1649 This method contains the `CmdLineTask` interface to the ISR 

1650 processing. All IO is handled here, freeing the `run()` method 

1651 to manage only pixel-level calculations. The steps performed 

1652 are: 

1653 - Read in necessary detrending/isr/calibration data. 

1654 - Process raw exposure in `run()`. 

1655 - Persist the ISR-corrected exposure as "postISRCCD" if 

1656 config.doWrite=True. 

1657 

1658 Parameters 

1659 ---------- 

1660 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1661 DataRef of the detector data to be processed 

1662 

1663 Returns 

1664 ------- 

1665 result : `lsst.pipe.base.Struct` 

1666 Result struct with component: 

1667 - ``exposure`` : `afw.image.Exposure` 

1668 The fully ISR corrected exposure. 

1669 

1670 Raises 

1671 ------ 

1672 RuntimeError 

1673 Raised if a configuration option is set to True, but the 

1674 required calibration data does not exist. 

1675 

1676 """ 

1677 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1678 

1679 ccdExposure = sensorRef.get(self.config.datasetType) 

1680 

1681 camera = sensorRef.get("camera") 

1682 isrData = self.readIsrData(sensorRef, ccdExposure) 

1683 

1684 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1685 

1686 if self.config.doWrite: 

1687 sensorRef.put(result.exposure, "postISRCCD") 

1688 if result.preInterpolatedExposure is not None: 

1689 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated") 

1690 if result.ossThumb is not None: 

1691 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1692 if result.flattenedThumb is not None: 

1693 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1694 

1695 return result 

1696 

1697 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1698 """Retrieve a calibration dataset for removing instrument signature. 

1699 

1700 Parameters 

1701 ---------- 

1702 

1703 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1704 DataRef of the detector data to find calibration datasets 

1705 for. 

1706 datasetType : `str` 

1707 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1708 dateObs : `str`, optional 

1709 Date of the observation. Used to correct butler failures 

1710 when using fallback filters. 

1711 immediate : `Bool` 

1712 If True, disable butler proxies to enable error handling 

1713 within this routine. 

1714 

1715 Returns 

1716 ------- 

1717 exposure : `lsst.afw.image.Exposure` 

1718 Requested calibration frame. 

1719 

1720 Raises 

1721 ------ 

1722 RuntimeError 

1723 Raised if no matching calibration frame can be found. 

1724 """ 

1725 try: 

1726 exp = dataRef.get(datasetType, immediate=immediate) 

1727 except Exception as exc1: 

1728 if not self.config.fallbackFilterName: 

1729 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1730 try: 

1731 if self.config.useFallbackDate and dateObs: 

1732 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1733 dateObs=dateObs, immediate=immediate) 

1734 else: 

1735 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1736 except Exception as exc2: 

1737 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1738 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1739 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1740 

1741 if self.config.doAssembleIsrExposures: 

1742 exp = self.assembleCcd.assembleCcd(exp) 

1743 return exp 

1744 

1745 def ensureExposure(self, inputExp, camera, detectorNum): 

1746 """Ensure that the data returned by Butler is a fully constructed exposure. 

1747 

1748 ISR requires exposure-level image data for historical reasons, so if we did 

1749 not recieve that from Butler, construct it from what we have, modifying the 

1750 input in place. 

1751 

1752 Parameters 

1753 ---------- 

1754 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1755 `lsst.afw.image.ImageF` 

1756 The input data structure obtained from Butler. 

1757 camera : `lsst.afw.cameraGeom.camera` 

1758 The camera associated with the image. Used to find the appropriate 

1759 detector. 

1760 detectorNum : `int` 

1761 The detector this exposure should match. 

1762 

1763 Returns 

1764 ------- 

1765 inputExp : `lsst.afw.image.Exposure` 

1766 The re-constructed exposure, with appropriate detector parameters. 

1767 

1768 Raises 

1769 ------ 

1770 TypeError 

1771 Raised if the input data cannot be used to construct an exposure. 

1772 """ 

1773 if isinstance(inputExp, afwImage.DecoratedImageU): 

1774 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1775 elif isinstance(inputExp, afwImage.ImageF): 

1776 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1777 elif isinstance(inputExp, afwImage.MaskedImageF): 

1778 inputExp = afwImage.makeExposure(inputExp) 

1779 elif isinstance(inputExp, afwImage.Exposure): 

1780 pass 

1781 elif inputExp is None: 

1782 # Assume this will be caught by the setup if it is a problem. 

1783 return inputExp 

1784 else: 

1785 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1786 (type(inputExp), )) 

1787 

1788 if inputExp.getDetector() is None: 

1789 inputExp.setDetector(camera[detectorNum]) 

1790 

1791 return inputExp 

1792 

1793 def convertIntToFloat(self, exposure): 

1794 """Convert exposure image from uint16 to float. 

1795 

1796 If the exposure does not need to be converted, the input is 

1797 immediately returned. For exposures that are converted to use 

1798 floating point pixels, the variance is set to unity and the 

1799 mask to zero. 

1800 

1801 Parameters 

1802 ---------- 

1803 exposure : `lsst.afw.image.Exposure` 

1804 The raw exposure to be converted. 

1805 

1806 Returns 

1807 ------- 

1808 newexposure : `lsst.afw.image.Exposure` 

1809 The input ``exposure``, converted to floating point pixels. 

1810 

1811 Raises 

1812 ------ 

1813 RuntimeError 

1814 Raised if the exposure type cannot be converted to float. 

1815 

1816 """ 

1817 if isinstance(exposure, afwImage.ExposureF): 

1818 # Nothing to be done 

1819 self.log.debug("Exposure already of type float.") 

1820 return exposure 

1821 if not hasattr(exposure, "convertF"): 

1822 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1823 

1824 newexposure = exposure.convertF() 

1825 newexposure.variance[:] = 1 

1826 newexposure.mask[:] = 0x0 

1827 

1828 return newexposure 

1829 

1830 def maskAmplifier(self, ccdExposure, amp, defects): 

1831 """Identify bad amplifiers, saturated and suspect pixels. 

1832 

1833 Parameters 

1834 ---------- 

1835 ccdExposure : `lsst.afw.image.Exposure` 

1836 Input exposure to be masked. 

1837 amp : `lsst.afw.table.AmpInfoCatalog` 

1838 Catalog of parameters defining the amplifier on this 

1839 exposure to mask. 

1840 defects : `lsst.meas.algorithms.Defects` 

1841 List of defects. Used to determine if the entire 

1842 amplifier is bad. 

1843 

1844 Returns 

1845 ------- 

1846 badAmp : `Bool` 

1847 If this is true, the entire amplifier area is covered by 

1848 defects and unusable. 

1849 

1850 """ 

1851 maskedImage = ccdExposure.getMaskedImage() 

1852 

1853 badAmp = False 

1854 

1855 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1856 # comparison with current defects definition. 

1857 if defects is not None: 

1858 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1859 

1860 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1861 # association with pixels in current ccdExposure). 

1862 if badAmp: 

1863 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1864 afwImage.PARENT) 

1865 maskView = dataView.getMask() 

1866 maskView |= maskView.getPlaneBitMask("BAD") 

1867 del maskView 

1868 return badAmp 

1869 

1870 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1871 # Saturation and suspect pixels can be masked now, though. 

1872 limits = dict() 

1873 if self.config.doSaturation and not badAmp: 

1874 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1875 if self.config.doSuspect and not badAmp: 

1876 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1877 if math.isfinite(self.config.saturation): 

1878 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1879 

1880 for maskName, maskThreshold in limits.items(): 

1881 if not math.isnan(maskThreshold): 

1882 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1883 isrFunctions.makeThresholdMask( 

1884 maskedImage=dataView, 

1885 threshold=maskThreshold, 

1886 growFootprints=0, 

1887 maskName=maskName 

1888 ) 

1889 

1890 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1891 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1892 afwImage.PARENT) 

1893 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1894 self.config.suspectMaskName]) 

1895 if numpy.all(maskView.getArray() & maskVal > 0): 

1896 badAmp = True 

1897 maskView |= maskView.getPlaneBitMask("BAD") 

1898 

1899 return badAmp 

1900 

1901 def overscanCorrection(self, ccdExposure, amp): 

1902 """Apply overscan correction in place. 

1903 

1904 This method does initial pixel rejection of the overscan 

1905 region. The overscan can also be optionally segmented to 

1906 allow for discontinuous overscan responses to be fit 

1907 separately. The actual overscan subtraction is performed by 

1908 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

1909 which is called here after the amplifier is preprocessed. 

1910 

1911 Parameters 

1912 ---------- 

1913 ccdExposure : `lsst.afw.image.Exposure` 

1914 Exposure to have overscan correction performed. 

1915 amp : `lsst.afw.cameraGeom.Amplifer` 

1916 The amplifier to consider while correcting the overscan. 

1917 

1918 Returns 

1919 ------- 

1920 overscanResults : `lsst.pipe.base.Struct` 

1921 Result struct with components: 

1922 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

1923 Value or fit subtracted from the amplifier image data. 

1924 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

1925 Value or fit subtracted from the overscan image data. 

1926 - ``overscanImage`` : `lsst.afw.image.Image` 

1927 Image of the overscan region with the overscan 

1928 correction applied. This quantity is used to estimate 

1929 the amplifier read noise empirically. 

1930 

1931 Raises 

1932 ------ 

1933 RuntimeError 

1934 Raised if the ``amp`` does not contain raw pixel information. 

1935 

1936 See Also 

1937 -------- 

1938 lsst.ip.isr.isrFunctions.overscanCorrection 

1939 """ 

1940 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

1941 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

1942 return None 

1943 

1944 statControl = afwMath.StatisticsControl() 

1945 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

1946 

1947 # Determine the bounding boxes 

1948 dataBBox = amp.getRawDataBBox() 

1949 oscanBBox = amp.getRawHorizontalOverscanBBox() 

1950 dx0 = 0 

1951 dx1 = 0 

1952 

1953 prescanBBox = amp.getRawPrescanBBox() 

1954 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

1955 dx0 += self.config.overscanNumLeadingColumnsToSkip 

1956 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

1957 else: 

1958 dx0 += self.config.overscanNumTrailingColumnsToSkip 

1959 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

1960 

1961 # Determine if we need to work on subregions of the amplifier and overscan. 

1962 imageBBoxes = [] 

1963 overscanBBoxes = [] 

1964 

1965 if ((self.config.overscanBiasJump 

1966 and self.config.overscanBiasJumpLocation) 

1967 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

1968 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

1969 self.config.overscanBiasJumpDevices)): 

1970 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

1971 yLower = self.config.overscanBiasJumpLocation 

1972 yUpper = dataBBox.getHeight() - yLower 

1973 else: 

1974 yUpper = self.config.overscanBiasJumpLocation 

1975 yLower = dataBBox.getHeight() - yUpper 

1976 

1977 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1978 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

1979 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1980 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1981 yLower))) 

1982 

1983 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

1984 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

1985 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

1986 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1987 yUpper))) 

1988 else: 

1989 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

1990 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

1991 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

1992 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

1993 oscanBBox.getHeight()))) 

1994 

1995 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

1996 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

1997 ampImage = ccdExposure.maskedImage[imageBBox] 

1998 overscanImage = ccdExposure.maskedImage[overscanBBox] 

1999 

2000 overscanArray = overscanImage.image.array 

2001 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2002 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2003 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2004 

2005 statControl = afwMath.StatisticsControl() 

2006 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2007 

2008 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2009 

2010 # Measure average overscan levels and record them in the metadata. 

2011 levelStat = afwMath.MEDIAN 

2012 sigmaStat = afwMath.STDEVCLIP 

2013 

2014 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2015 self.config.qa.flatness.nIter) 

2016 metadata = ccdExposure.getMetadata() 

2017 ampNum = amp.getName() 

2018 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2019 if isinstance(overscanResults.overscanFit, float): 

2020 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2021 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2022 else: 

2023 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2024 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2025 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2026 

2027 return overscanResults 

2028 

2029 def updateVariance(self, ampExposure, amp, overscanImage=None): 

2030 """Set the variance plane using the amplifier gain and read noise 

2031 

2032 The read noise is calculated from the ``overscanImage`` if the 

2033 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2034 the value from the amplifier data is used. 

2035 

2036 Parameters 

2037 ---------- 

2038 ampExposure : `lsst.afw.image.Exposure` 

2039 Exposure to process. 

2040 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2041 Amplifier detector data. 

2042 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2043 Image of overscan, required only for empirical read noise. 

2044 

2045 See also 

2046 -------- 

2047 lsst.ip.isr.isrFunctions.updateVariance 

2048 """ 

2049 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2050 gain = amp.getGain() 

2051 

2052 if math.isnan(gain): 

2053 gain = 1.0 

2054 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2055 elif gain <= 0: 

2056 patchedGain = 1.0 

2057 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.", 

2058 amp.getName(), gain, patchedGain) 

2059 gain = patchedGain 

2060 

2061 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2062 self.log.info("Overscan is none for EmpiricalReadNoise.") 

2063 

2064 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2065 stats = afwMath.StatisticsControl() 

2066 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2067 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2068 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2069 amp.getName(), readNoise) 

2070 else: 

2071 readNoise = amp.getReadNoise() 

2072 

2073 isrFunctions.updateVariance( 

2074 maskedImage=ampExposure.getMaskedImage(), 

2075 gain=gain, 

2076 readNoise=readNoise, 

2077 ) 

2078 

2079 def darkCorrection(self, exposure, darkExposure, invert=False): 

2080 """Apply dark correction in place. 

2081 

2082 Parameters 

2083 ---------- 

2084 exposure : `lsst.afw.image.Exposure` 

2085 Exposure to process. 

2086 darkExposure : `lsst.afw.image.Exposure` 

2087 Dark exposure of the same size as ``exposure``. 

2088 invert : `Bool`, optional 

2089 If True, re-add the dark to an already corrected image. 

2090 

2091 Raises 

2092 ------ 

2093 RuntimeError 

2094 Raised if either ``exposure`` or ``darkExposure`` do not 

2095 have their dark time defined. 

2096 

2097 See Also 

2098 -------- 

2099 lsst.ip.isr.isrFunctions.darkCorrection 

2100 """ 

2101 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2102 if math.isnan(expScale): 

2103 raise RuntimeError("Exposure darktime is NAN.") 

2104 if darkExposure.getInfo().getVisitInfo() is not None \ 

2105 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2106 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2107 else: 

2108 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2109 # so getDarkTime() does not exist. 

2110 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2111 darkScale = 1.0 

2112 

2113 isrFunctions.darkCorrection( 

2114 maskedImage=exposure.getMaskedImage(), 

2115 darkMaskedImage=darkExposure.getMaskedImage(), 

2116 expScale=expScale, 

2117 darkScale=darkScale, 

2118 invert=invert, 

2119 trimToFit=self.config.doTrimToMatchCalib 

2120 ) 

2121 

2122 def doLinearize(self, detector): 

2123 """Check if linearization is needed for the detector cameraGeom. 

2124 

2125 Checks config.doLinearize and the linearity type of the first 

2126 amplifier. 

2127 

2128 Parameters 

2129 ---------- 

2130 detector : `lsst.afw.cameraGeom.Detector` 

2131 Detector to get linearity type from. 

2132 

2133 Returns 

2134 ------- 

2135 doLinearize : `Bool` 

2136 If True, linearization should be performed. 

2137 """ 

2138 return self.config.doLinearize and \ 

2139 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2140 

2141 def flatCorrection(self, exposure, flatExposure, invert=False): 

2142 """Apply flat correction in place. 

2143 

2144 Parameters 

2145 ---------- 

2146 exposure : `lsst.afw.image.Exposure` 

2147 Exposure to process. 

2148 flatExposure : `lsst.afw.image.Exposure` 

2149 Flat exposure of the same size as ``exposure``. 

2150 invert : `Bool`, optional 

2151 If True, unflatten an already flattened image. 

2152 

2153 See Also 

2154 -------- 

2155 lsst.ip.isr.isrFunctions.flatCorrection 

2156 """ 

2157 isrFunctions.flatCorrection( 

2158 maskedImage=exposure.getMaskedImage(), 

2159 flatMaskedImage=flatExposure.getMaskedImage(), 

2160 scalingType=self.config.flatScalingType, 

2161 userScale=self.config.flatUserScale, 

2162 invert=invert, 

2163 trimToFit=self.config.doTrimToMatchCalib 

2164 ) 

2165 

2166 def saturationDetection(self, exposure, amp): 

2167 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2168 

2169 Parameters 

2170 ---------- 

2171 exposure : `lsst.afw.image.Exposure` 

2172 Exposure to process. Only the amplifier DataSec is processed. 

2173 amp : `lsst.afw.table.AmpInfoCatalog` 

2174 Amplifier detector data. 

2175 

2176 See Also 

2177 -------- 

2178 lsst.ip.isr.isrFunctions.makeThresholdMask 

2179 """ 

2180 if not math.isnan(amp.getSaturation()): 

2181 maskedImage = exposure.getMaskedImage() 

2182 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2183 isrFunctions.makeThresholdMask( 

2184 maskedImage=dataView, 

2185 threshold=amp.getSaturation(), 

2186 growFootprints=0, 

2187 maskName=self.config.saturatedMaskName, 

2188 ) 

2189 

2190 def saturationInterpolation(self, exposure): 

2191 """Interpolate over saturated pixels, in place. 

2192 

2193 This method should be called after `saturationDetection`, to 

2194 ensure that the saturated pixels have been identified in the 

2195 SAT mask. It should also be called after `assembleCcd`, since 

2196 saturated regions may cross amplifier boundaries. 

2197 

2198 Parameters 

2199 ---------- 

2200 exposure : `lsst.afw.image.Exposure` 

2201 Exposure to process. 

2202 

2203 See Also 

2204 -------- 

2205 lsst.ip.isr.isrTask.saturationDetection 

2206 lsst.ip.isr.isrFunctions.interpolateFromMask 

2207 """ 

2208 isrFunctions.interpolateFromMask( 

2209 maskedImage=exposure.getMaskedImage(), 

2210 fwhm=self.config.fwhm, 

2211 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2212 maskNameList=list(self.config.saturatedMaskName), 

2213 ) 

2214 

2215 def suspectDetection(self, exposure, amp): 

2216 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2217 

2218 Parameters 

2219 ---------- 

2220 exposure : `lsst.afw.image.Exposure` 

2221 Exposure to process. Only the amplifier DataSec is processed. 

2222 amp : `lsst.afw.table.AmpInfoCatalog` 

2223 Amplifier detector data. 

2224 

2225 See Also 

2226 -------- 

2227 lsst.ip.isr.isrFunctions.makeThresholdMask 

2228 

2229 Notes 

2230 ----- 

2231 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2232 This is intended to indicate pixels that may be affected by unknown systematics; 

2233 for example if non-linearity corrections above a certain level are unstable 

2234 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2235 that no such level exists and no pixels are to be masked as suspicious. 

2236 """ 

2237 suspectLevel = amp.getSuspectLevel() 

2238 if math.isnan(suspectLevel): 

2239 return 

2240 

2241 maskedImage = exposure.getMaskedImage() 

2242 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2243 isrFunctions.makeThresholdMask( 

2244 maskedImage=dataView, 

2245 threshold=suspectLevel, 

2246 growFootprints=0, 

2247 maskName=self.config.suspectMaskName, 

2248 ) 

2249 

2250 def maskDefect(self, exposure, defectBaseList): 

2251 """Mask defects using mask plane "BAD", in place. 

2252 

2253 Parameters 

2254 ---------- 

2255 exposure : `lsst.afw.image.Exposure` 

2256 Exposure to process. 

2257 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2258 `lsst.afw.image.DefectBase`. 

2259 List of defects to mask. 

2260 

2261 Notes 

2262 ----- 

2263 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2264 """ 

2265 maskedImage = exposure.getMaskedImage() 

2266 if not isinstance(defectBaseList, Defects): 

2267 # Promotes DefectBase to Defect 

2268 defectList = Defects(defectBaseList) 

2269 else: 

2270 defectList = defectBaseList 

2271 defectList.maskPixels(maskedImage, maskName="BAD") 

2272 

2273 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2274 """Mask edge pixels with applicable mask plane. 

2275 

2276 Parameters 

2277 ---------- 

2278 exposure : `lsst.afw.image.Exposure` 

2279 Exposure to process. 

2280 numEdgePixels : `int`, optional 

2281 Number of edge pixels to mask. 

2282 maskPlane : `str`, optional 

2283 Mask plane name to use. 

2284 level : `str`, optional 

2285 Level at which to mask edges. 

2286 """ 

2287 maskedImage = exposure.getMaskedImage() 

2288 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2289 

2290 if numEdgePixels > 0: 

2291 if level == 'DETECTOR': 

2292 boxes = [maskedImage.getBBox()] 

2293 elif level == 'AMP': 

2294 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2295 

2296 for box in boxes: 

2297 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2298 subImage = maskedImage[box] 

2299 box.grow(-numEdgePixels) 

2300 # Mask pixels outside box 

2301 SourceDetectionTask.setEdgeBits( 

2302 subImage, 

2303 box, 

2304 maskBitMask) 

2305 

2306 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2307 """Mask and interpolate defects using mask plane "BAD", in place. 

2308 

2309 Parameters 

2310 ---------- 

2311 exposure : `lsst.afw.image.Exposure` 

2312 Exposure to process. 

2313 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 

2314 `lsst.afw.image.DefectBase`. 

2315 List of defects to mask and interpolate. 

2316 

2317 See Also 

2318 -------- 

2319 lsst.ip.isr.isrTask.maskDefect 

2320 """ 

2321 self.maskDefect(exposure, defectBaseList) 

2322 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2323 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2324 isrFunctions.interpolateFromMask( 

2325 maskedImage=exposure.getMaskedImage(), 

2326 fwhm=self.config.fwhm, 

2327 growSaturatedFootprints=0, 

2328 maskNameList=["BAD"], 

2329 ) 

2330 

2331 def maskNan(self, exposure): 

2332 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2333 

2334 Parameters 

2335 ---------- 

2336 exposure : `lsst.afw.image.Exposure` 

2337 Exposure to process. 

2338 

2339 Notes 

2340 ----- 

2341 We mask over all NaNs, including those that are masked with 

2342 other bits (because those may or may not be interpolated over 

2343 later, and we want to remove all NaNs). Despite this 

2344 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 

2345 the historical name. 

2346 """ 

2347 maskedImage = exposure.getMaskedImage() 

2348 

2349 # Find and mask NaNs 

2350 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2351 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2352 numNans = maskNans(maskedImage, maskVal) 

2353 self.metadata.set("NUMNANS", numNans) 

2354 if numNans > 0: 

2355 self.log.warn("There were %d unmasked NaNs.", numNans) 

2356 

2357 def maskAndInterpolateNan(self, exposure): 

2358 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 

2359 

2360 Parameters 

2361 ---------- 

2362 exposure : `lsst.afw.image.Exposure` 

2363 Exposure to process. 

2364 

2365 See Also 

2366 -------- 

2367 lsst.ip.isr.isrTask.maskNan 

2368 """ 

2369 self.maskNan(exposure) 

2370 isrFunctions.interpolateFromMask( 

2371 maskedImage=exposure.getMaskedImage(), 

2372 fwhm=self.config.fwhm, 

2373 growSaturatedFootprints=0, 

2374 maskNameList=["UNMASKEDNAN"], 

2375 ) 

2376 

2377 def measureBackground(self, exposure, IsrQaConfig=None): 

2378 """Measure the image background in subgrids, for quality control purposes. 

2379 

2380 Parameters 

2381 ---------- 

2382 exposure : `lsst.afw.image.Exposure` 

2383 Exposure to process. 

2384 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2385 Configuration object containing parameters on which background 

2386 statistics and subgrids to use. 

2387 """ 

2388 if IsrQaConfig is not None: 

2389 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2390 IsrQaConfig.flatness.nIter) 

2391 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2392 statsControl.setAndMask(maskVal) 

2393 maskedImage = exposure.getMaskedImage() 

2394 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2395 skyLevel = stats.getValue(afwMath.MEDIAN) 

2396 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2397 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2398 metadata = exposure.getMetadata() 

2399 metadata.set('SKYLEVEL', skyLevel) 

2400 metadata.set('SKYSIGMA', skySigma) 

2401 

2402 # calcluating flatlevel over the subgrids 

2403 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2404 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2405 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2406 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2407 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2408 skyLevels = numpy.zeros((nX, nY)) 

2409 

2410 for j in range(nY): 

2411 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2412 for i in range(nX): 

2413 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2414 

2415 xLLC = xc - meshXHalf 

2416 yLLC = yc - meshYHalf 

2417 xURC = xc + meshXHalf - 1 

2418 yURC = yc + meshYHalf - 1 

2419 

2420 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2421 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2422 

2423 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2424 

2425 good = numpy.where(numpy.isfinite(skyLevels)) 

2426 skyMedian = numpy.median(skyLevels[good]) 

2427 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2428 flatness_rms = numpy.std(flatness) 

2429 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2430 

2431 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2432 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2433 nX, nY, flatness_pp, flatness_rms) 

2434 

2435 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2436 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2437 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2438 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2439 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2440 

2441 def roughZeroPoint(self, exposure): 

2442 """Set an approximate magnitude zero point for the exposure. 

2443 

2444 Parameters 

2445 ---------- 

2446 exposure : `lsst.afw.image.Exposure` 

2447 Exposure to process. 

2448 """ 

2449 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter 

2450 if filterName in self.config.fluxMag0T1: 

2451 fluxMag0 = self.config.fluxMag0T1[filterName] 

2452 else: 

2453 self.log.warn("No rough magnitude zero point set for filter %s.", filterName) 

2454 fluxMag0 = self.config.defaultFluxMag0T1 

2455 

2456 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2457 if not expTime > 0: # handle NaN as well as <= 0 

2458 self.log.warn("Non-positive exposure time; skipping rough zero point.") 

2459 return 

2460 

2461 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime)) 

2462 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2463 

2464 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2465 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2466 

2467 Parameters 

2468 ---------- 

2469 ccdExposure : `lsst.afw.image.Exposure` 

2470 Exposure to process. 

2471 fpPolygon : `lsst.afw.geom.Polygon` 

2472 Polygon in focal plane coordinates. 

2473 """ 

2474 # Get ccd corners in focal plane coordinates 

2475 ccd = ccdExposure.getDetector() 

2476 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2477 ccdPolygon = Polygon(fpCorners) 

2478 

2479 # Get intersection of ccd corners with fpPolygon 

2480 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2481 

2482 # Transform back to pixel positions and build new polygon 

2483 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2484 validPolygon = Polygon(ccdPoints) 

2485 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2486 

2487 @contextmanager 

2488 def flatContext(self, exp, flat, dark=None): 

2489 """Context manager that applies and removes flats and darks, 

2490 if the task is configured to apply them. 

2491 

2492 Parameters 

2493 ---------- 

2494 exp : `lsst.afw.image.Exposure` 

2495 Exposure to process. 

2496 flat : `lsst.afw.image.Exposure` 

2497 Flat exposure the same size as ``exp``. 

2498 dark : `lsst.afw.image.Exposure`, optional 

2499 Dark exposure the same size as ``exp``. 

2500 

2501 Yields 

2502 ------ 

2503 exp : `lsst.afw.image.Exposure` 

2504 The flat and dark corrected exposure. 

2505 """ 

2506 if self.config.doDark and dark is not None: 

2507 self.darkCorrection(exp, dark) 

2508 if self.config.doFlat: 

2509 self.flatCorrection(exp, flat) 

2510 try: 

2511 yield exp 

2512 finally: 

2513 if self.config.doFlat: 

2514 self.flatCorrection(exp, flat, invert=True) 

2515 if self.config.doDark and dark is not None: 

2516 self.darkCorrection(exp, dark, invert=True) 

2517 

2518 def debugView(self, exposure, stepname): 

2519 """Utility function to examine ISR exposure at different stages. 

2520 

2521 Parameters 

2522 ---------- 

2523 exposure : `lsst.afw.image.Exposure` 

2524 Exposure to view. 

2525 stepname : `str` 

2526 State of processing to view. 

2527 """ 

2528 frame = getDebugFrame(self._display, stepname) 

2529 if frame: 

2530 display = getDisplay(frame) 

2531 display.scale('asinh', 'zscale') 

2532 display.mtv(exposure) 

2533 prompt = "Press Enter to continue [c]... " 

2534 while True: 

2535 ans = input(prompt).lower() 

2536 if ans in ("", "c",): 

2537 break 

2538 

2539 

2540class FakeAmp(object): 

2541 """A Detector-like object that supports returning gain and saturation level 

2542 

2543 This is used when the input exposure does not have a detector. 

2544 

2545 Parameters 

2546 ---------- 

2547 exposure : `lsst.afw.image.Exposure` 

2548 Exposure to generate a fake amplifier for. 

2549 config : `lsst.ip.isr.isrTaskConfig` 

2550 Configuration to apply to the fake amplifier. 

2551 """ 

2552 

2553 def __init__(self, exposure, config): 

2554 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2555 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2556 self._gain = config.gain 

2557 self._readNoise = config.readNoise 

2558 self._saturation = config.saturation 

2559 

2560 def getBBox(self): 

2561 return self._bbox 

2562 

2563 def getRawBBox(self): 

2564 return self._bbox 

2565 

2566 def getRawHorizontalOverscanBBox(self): 

2567 return self._RawHorizontalOverscanBBox 

2568 

2569 def getGain(self): 

2570 return self._gain 

2571 

2572 def getReadNoise(self): 

2573 return self._readNoise 

2574 

2575 def getSaturation(self): 

2576 return self._saturation 

2577 

2578 def getSuspectLevel(self): 

2579 return float("NaN") 

2580 

2581 

2582class RunIsrConfig(pexConfig.Config): 

2583 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2584 

2585 

2586class RunIsrTask(pipeBase.CmdLineTask): 

2587 """Task to wrap the default IsrTask to allow it to be retargeted. 

2588 

2589 The standard IsrTask can be called directly from a command line 

2590 program, but doing so removes the ability of the task to be 

2591 retargeted. As most cameras override some set of the IsrTask 

2592 methods, this would remove those data-specific methods in the 

2593 output post-ISR images. This wrapping class fixes the issue, 

2594 allowing identical post-ISR images to be generated by both the 

2595 processCcd and isrTask code. 

2596 """ 

2597 ConfigClass = RunIsrConfig 

2598 _DefaultName = "runIsr" 

2599 

2600 def __init__(self, *args, **kwargs): 

2601 super().__init__(*args, **kwargs) 

2602 self.makeSubtask("isr") 

2603 

2604 def runDataRef(self, dataRef): 

2605 """ 

2606 Parameters 

2607 ---------- 

2608 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2609 data reference of the detector data to be processed 

2610 

2611 Returns 

2612 ------- 

2613 result : `pipeBase.Struct` 

2614 Result struct with component: 

2615 

2616 - exposure : `lsst.afw.image.Exposure` 

2617 Post-ISR processed exposure. 

2618 """ 

2619 return self.isr.runDataRef(dataRef)