Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42 

43from . import isrFunctions 

44from . import isrQa 

45from . import linearize 

46from .defects import Defects 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 minimum=0, # can fall back to cameraGeom 

123 ) 

124 # TODO: DM-25348. This does not work yet to correctly load 

125 # possible crosstalk sources. 

126 crosstalkSources = cT.PrerequisiteInput( 

127 name="isrOverscanCorrected", 

128 doc="Overscan corrected input images.", 

129 storageClass="Exposure", 

130 dimensions=["instrument", "exposure", "detector"], 

131 deferLoad=True, 

132 multiple=True, 

133 lookupFunction=crosstalkSourceLookup, 

134 minimum=0, # not needed for all instruments, no config to control this 

135 ) 

136 bias = cT.PrerequisiteInput( 

137 name="bias", 

138 doc="Input bias calibration.", 

139 storageClass="ExposureF", 

140 dimensions=["instrument", "detector"], 

141 isCalibration=True, 

142 ) 

143 dark = cT.PrerequisiteInput( 

144 name='dark', 

145 doc="Input dark calibration.", 

146 storageClass="ExposureF", 

147 dimensions=["instrument", "detector"], 

148 isCalibration=True, 

149 ) 

150 flat = cT.PrerequisiteInput( 

151 name="flat", 

152 doc="Input flat calibration.", 

153 storageClass="ExposureF", 

154 dimensions=["instrument", "physical_filter", "detector"], 

155 isCalibration=True, 

156 ) 

157 ptc = cT.PrerequisiteInput( 

158 name="ptc", 

159 doc="Input Photon Transfer Curve dataset", 

160 storageClass="PhotonTransferCurveDataset", 

161 dimensions=["instrument", "detector"], 

162 isCalibration=True, 

163 ) 

164 fringes = cT.PrerequisiteInput( 

165 name="fringe", 

166 doc="Input fringe calibration.", 

167 storageClass="ExposureF", 

168 dimensions=["instrument", "physical_filter", "detector"], 

169 isCalibration=True, 

170 minimum=0, # only needed for some bands, even when enabled 

171 ) 

172 strayLightData = cT.PrerequisiteInput( 

173 name='yBackground', 

174 doc="Input stray light calibration.", 

175 storageClass="StrayLightData", 

176 dimensions=["instrument", "physical_filter", "detector"], 

177 deferLoad=True, 

178 isCalibration=True, 

179 minimum=0, # only needed for some bands, even when enabled 

180 ) 

181 bfKernel = cT.PrerequisiteInput( 

182 name='bfKernel', 

183 doc="Input brighter-fatter kernel.", 

184 storageClass="NumpyArray", 

185 dimensions=["instrument"], 

186 isCalibration=True, 

187 minimum=0, # can use either bfKernel or newBFKernel 

188 ) 

189 newBFKernel = cT.PrerequisiteInput( 

190 name='brighterFatterKernel', 

191 doc="Newer complete kernel + gain solutions.", 

192 storageClass="BrighterFatterKernel", 

193 dimensions=["instrument", "detector"], 

194 isCalibration=True, 

195 minimum=0, # can use either bfKernel or newBFKernel 

196 ) 

197 defects = cT.PrerequisiteInput( 

198 name='defects', 

199 doc="Input defect tables.", 

200 storageClass="Defects", 

201 dimensions=["instrument", "detector"], 

202 isCalibration=True, 

203 ) 

204 linearizer = cT.PrerequisiteInput( 

205 name='linearizer', 

206 storageClass="Linearizer", 

207 doc="Linearity correction calibration.", 

208 dimensions=["instrument", "detector"], 

209 isCalibration=True, 

210 minimum=0, # can fall back to cameraGeom 

211 ) 

212 opticsTransmission = cT.PrerequisiteInput( 

213 name="transmission_optics", 

214 storageClass="TransmissionCurve", 

215 doc="Transmission curve due to the optics.", 

216 dimensions=["instrument"], 

217 isCalibration=True, 

218 ) 

219 filterTransmission = cT.PrerequisiteInput( 

220 name="transmission_filter", 

221 storageClass="TransmissionCurve", 

222 doc="Transmission curve due to the filter.", 

223 dimensions=["instrument", "physical_filter"], 

224 isCalibration=True, 

225 ) 

226 sensorTransmission = cT.PrerequisiteInput( 

227 name="transmission_sensor", 

228 storageClass="TransmissionCurve", 

229 doc="Transmission curve due to the sensor.", 

230 dimensions=["instrument", "detector"], 

231 isCalibration=True, 

232 ) 

233 atmosphereTransmission = cT.PrerequisiteInput( 

234 name="transmission_atmosphere", 

235 storageClass="TransmissionCurve", 

236 doc="Transmission curve due to the atmosphere.", 

237 dimensions=["instrument"], 

238 isCalibration=True, 

239 ) 

240 illumMaskedImage = cT.PrerequisiteInput( 

241 name="illum", 

242 doc="Input illumination correction.", 

243 storageClass="MaskedImageF", 

244 dimensions=["instrument", "physical_filter", "detector"], 

245 isCalibration=True, 

246 ) 

247 

248 outputExposure = cT.Output( 

249 name='postISRCCD', 

250 doc="Output ISR processed exposure.", 

251 storageClass="Exposure", 

252 dimensions=["instrument", "exposure", "detector"], 

253 ) 

254 preInterpExposure = cT.Output( 

255 name='preInterpISRCCD', 

256 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

257 storageClass="ExposureF", 

258 dimensions=["instrument", "exposure", "detector"], 

259 ) 

260 outputOssThumbnail = cT.Output( 

261 name="OssThumb", 

262 doc="Output Overscan-subtracted thumbnail image.", 

263 storageClass="Thumbnail", 

264 dimensions=["instrument", "exposure", "detector"], 

265 ) 

266 outputFlattenedThumbnail = cT.Output( 

267 name="FlattenedThumb", 

268 doc="Output flat-corrected thumbnail image.", 

269 storageClass="Thumbnail", 

270 dimensions=["instrument", "exposure", "detector"], 

271 ) 

272 

273 def __init__(self, *, config=None): 

274 super().__init__(config=config) 

275 

276 if config.doBias is not True: 

277 self.prerequisiteInputs.discard("bias") 

278 if config.doLinearize is not True: 

279 self.prerequisiteInputs.discard("linearizer") 

280 if config.doCrosstalk is not True: 

281 self.inputs.discard("crosstalkSources") 

282 self.prerequisiteInputs.discard("crosstalk") 

283 if config.doBrighterFatter is not True: 

284 self.prerequisiteInputs.discard("bfKernel") 

285 self.prerequisiteInputs.discard("newBFKernel") 

286 if config.doDefect is not True: 

287 self.prerequisiteInputs.discard("defects") 

288 if config.doDark is not True: 

289 self.prerequisiteInputs.discard("dark") 

290 if config.doFlat is not True: 

291 self.prerequisiteInputs.discard("flat") 

292 if config.doFringe is not True: 

293 self.prerequisiteInputs.discard("fringe") 

294 if config.doStrayLight is not True: 

295 self.prerequisiteInputs.discard("strayLightData") 

296 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

297 self.prerequisiteInputs.discard("ptc") 

298 if config.doAttachTransmissionCurve is not True: 

299 self.prerequisiteInputs.discard("opticsTransmission") 

300 self.prerequisiteInputs.discard("filterTransmission") 

301 self.prerequisiteInputs.discard("sensorTransmission") 

302 self.prerequisiteInputs.discard("atmosphereTransmission") 

303 if config.doUseOpticsTransmission is not True: 

304 self.prerequisiteInputs.discard("opticsTransmission") 

305 if config.doUseFilterTransmission is not True: 

306 self.prerequisiteInputs.discard("filterTransmission") 

307 if config.doUseSensorTransmission is not True: 

308 self.prerequisiteInputs.discard("sensorTransmission") 

309 if config.doUseAtmosphereTransmission is not True: 

310 self.prerequisiteInputs.discard("atmosphereTransmission") 

311 if config.doIlluminationCorrection is not True: 

312 self.prerequisiteInputs.discard("illumMaskedImage") 

313 

314 if config.doWrite is not True: 

315 self.outputs.discard("outputExposure") 

316 self.outputs.discard("preInterpExposure") 

317 self.outputs.discard("outputFlattenedThumbnail") 

318 self.outputs.discard("outputOssThumbnail") 

319 if config.doSaveInterpPixels is not True: 

320 self.outputs.discard("preInterpExposure") 

321 if config.qa.doThumbnailOss is not True: 

322 self.outputs.discard("outputOssThumbnail") 

323 if config.qa.doThumbnailFlattened is not True: 

324 self.outputs.discard("outputFlattenedThumbnail") 

325 

326 

327class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

328 pipelineConnections=IsrTaskConnections): 

329 """Configuration parameters for IsrTask. 

330 

331 Items are grouped in the order in which they are executed by the task. 

332 """ 

333 datasetType = pexConfig.Field( 

334 dtype=str, 

335 doc="Dataset type for input data; users will typically leave this alone, " 

336 "but camera-specific ISR tasks will override it", 

337 default="raw", 

338 ) 

339 

340 fallbackFilterName = pexConfig.Field( 

341 dtype=str, 

342 doc="Fallback default filter name for calibrations.", 

343 optional=True 

344 ) 

345 useFallbackDate = pexConfig.Field( 

346 dtype=bool, 

347 doc="Pass observation date when using fallback filter.", 

348 default=False, 

349 ) 

350 expectWcs = pexConfig.Field( 

351 dtype=bool, 

352 default=True, 

353 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

354 ) 

355 fwhm = pexConfig.Field( 

356 dtype=float, 

357 doc="FWHM of PSF in arcseconds.", 

358 default=1.0, 

359 ) 

360 qa = pexConfig.ConfigField( 

361 dtype=isrQa.IsrQaConfig, 

362 doc="QA related configuration options.", 

363 ) 

364 

365 # Image conversion configuration 

366 doConvertIntToFloat = pexConfig.Field( 

367 dtype=bool, 

368 doc="Convert integer raw images to floating point values?", 

369 default=True, 

370 ) 

371 

372 # Saturated pixel handling. 

373 doSaturation = pexConfig.Field( 

374 dtype=bool, 

375 doc="Mask saturated pixels? NB: this is totally independent of the" 

376 " interpolation option - this is ONLY setting the bits in the mask." 

377 " To have them interpolated make sure doSaturationInterpolation=True", 

378 default=True, 

379 ) 

380 saturatedMaskName = pexConfig.Field( 

381 dtype=str, 

382 doc="Name of mask plane to use in saturation detection and interpolation", 

383 default="SAT", 

384 ) 

385 saturation = pexConfig.Field( 

386 dtype=float, 

387 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

388 default=float("NaN"), 

389 ) 

390 growSaturationFootprintSize = pexConfig.Field( 

391 dtype=int, 

392 doc="Number of pixels by which to grow the saturation footprints", 

393 default=1, 

394 ) 

395 

396 # Suspect pixel handling. 

397 doSuspect = pexConfig.Field( 

398 dtype=bool, 

399 doc="Mask suspect pixels?", 

400 default=False, 

401 ) 

402 suspectMaskName = pexConfig.Field( 

403 dtype=str, 

404 doc="Name of mask plane to use for suspect pixels", 

405 default="SUSPECT", 

406 ) 

407 numEdgeSuspect = pexConfig.Field( 

408 dtype=int, 

409 doc="Number of edge pixels to be flagged as untrustworthy.", 

410 default=0, 

411 ) 

412 edgeMaskLevel = pexConfig.ChoiceField( 

413 dtype=str, 

414 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

415 default="DETECTOR", 

416 allowed={ 

417 'DETECTOR': 'Mask only the edges of the full detector.', 

418 'AMP': 'Mask edges of each amplifier.', 

419 }, 

420 ) 

421 

422 # Initial masking options. 

423 doSetBadRegions = pexConfig.Field( 

424 dtype=bool, 

425 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

426 default=True, 

427 ) 

428 badStatistic = pexConfig.ChoiceField( 

429 dtype=str, 

430 doc="How to estimate the average value for BAD regions.", 

431 default='MEANCLIP', 

432 allowed={ 

433 "MEANCLIP": "Correct using the (clipped) mean of good data", 

434 "MEDIAN": "Correct using the median of the good data", 

435 }, 

436 ) 

437 

438 # Overscan subtraction configuration. 

439 doOverscan = pexConfig.Field( 

440 dtype=bool, 

441 doc="Do overscan subtraction?", 

442 default=True, 

443 ) 

444 overscan = pexConfig.ConfigurableField( 

445 target=OverscanCorrectionTask, 

446 doc="Overscan subtraction task for image segments.", 

447 ) 

448 

449 overscanFitType = pexConfig.ChoiceField( 

450 dtype=str, 

451 doc="The method for fitting the overscan bias level.", 

452 default='MEDIAN', 

453 allowed={ 

454 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

455 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

456 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

457 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

458 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

459 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

460 "MEAN": "Correct using the mean of the overscan region", 

461 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

462 "MEDIAN": "Correct using the median of the overscan region", 

463 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

464 }, 

465 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

466 " This option will no longer be used, and will be removed after v20.") 

467 ) 

468 overscanOrder = pexConfig.Field( 

469 dtype=int, 

470 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

471 "or number of spline knots if overscan fit type is a spline."), 

472 default=1, 

473 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

474 " This option will no longer be used, and will be removed after v20.") 

475 ) 

476 overscanNumSigmaClip = pexConfig.Field( 

477 dtype=float, 

478 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

479 default=3.0, 

480 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

481 " This option will no longer be used, and will be removed after v20.") 

482 ) 

483 overscanIsInt = pexConfig.Field( 

484 dtype=bool, 

485 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

486 " and overscan.FitType=MEDIAN_PER_ROW.", 

487 default=True, 

488 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

489 " This option will no longer be used, and will be removed after v20.") 

490 ) 

491 # These options do not get deprecated, as they define how we slice up the image data. 

492 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

493 dtype=int, 

494 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

495 default=0, 

496 ) 

497 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

498 dtype=int, 

499 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

500 default=0, 

501 ) 

502 overscanMaxDev = pexConfig.Field( 502 ↛ exitline 502 didn't jump to the function exit

503 dtype=float, 

504 doc="Maximum deviation from the median for overscan", 

505 default=1000.0, check=lambda x: x > 0 

506 ) 

507 overscanBiasJump = pexConfig.Field( 

508 dtype=bool, 

509 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

510 default=False, 

511 ) 

512 overscanBiasJumpKeyword = pexConfig.Field( 

513 dtype=str, 

514 doc="Header keyword containing information about devices.", 

515 default="NO_SUCH_KEY", 

516 ) 

517 overscanBiasJumpDevices = pexConfig.ListField( 

518 dtype=str, 

519 doc="List of devices that need piecewise overscan correction.", 

520 default=(), 

521 ) 

522 overscanBiasJumpLocation = pexConfig.Field( 

523 dtype=int, 

524 doc="Location of bias jump along y-axis.", 

525 default=0, 

526 ) 

527 

528 # Amplifier to CCD assembly configuration 

529 doAssembleCcd = pexConfig.Field( 

530 dtype=bool, 

531 default=True, 

532 doc="Assemble amp-level exposures into a ccd-level exposure?" 

533 ) 

534 assembleCcd = pexConfig.ConfigurableField( 

535 target=AssembleCcdTask, 

536 doc="CCD assembly task", 

537 ) 

538 

539 # General calibration configuration. 

540 doAssembleIsrExposures = pexConfig.Field( 

541 dtype=bool, 

542 default=False, 

543 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

544 ) 

545 doTrimToMatchCalib = pexConfig.Field( 

546 dtype=bool, 

547 default=False, 

548 doc="Trim raw data to match calibration bounding boxes?" 

549 ) 

550 

551 # Bias subtraction. 

552 doBias = pexConfig.Field( 

553 dtype=bool, 

554 doc="Apply bias frame correction?", 

555 default=True, 

556 ) 

557 biasDataProductName = pexConfig.Field( 

558 dtype=str, 

559 doc="Name of the bias data product", 

560 default="bias", 

561 ) 

562 doBiasBeforeOverscan = pexConfig.Field( 

563 dtype=bool, 

564 doc="Reverse order of overscan and bias correction.", 

565 default=False 

566 ) 

567 

568 # Variance construction 

569 doVariance = pexConfig.Field( 

570 dtype=bool, 

571 doc="Calculate variance?", 

572 default=True 

573 ) 

574 gain = pexConfig.Field( 

575 dtype=float, 

576 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

577 default=float("NaN"), 

578 ) 

579 readNoise = pexConfig.Field( 

580 dtype=float, 

581 doc="The read noise to use if no Detector is present in the Exposure", 

582 default=0.0, 

583 ) 

584 doEmpiricalReadNoise = pexConfig.Field( 

585 dtype=bool, 

586 default=False, 

587 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

588 ) 

589 usePtcReadNoise = pexConfig.Field( 

590 dtype=bool, 

591 default=False, 

592 doc="Use readnoise values from the Photon Transfer Curve?" 

593 ) 

594 maskNegativeVariance = pexConfig.Field( 

595 dtype=bool, 

596 default=True, 

597 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

598 "in the measurement of the overscan at an edge due to the data falling off faster " 

599 "than the overscan model can account for it." 

600 ) 

601 negativeVarianceMaskName = pexConfig.Field( 

602 dtype=str, 

603 default="BAD", 

604 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

605 ) 

606 # Linearization. 

607 doLinearize = pexConfig.Field( 

608 dtype=bool, 

609 doc="Correct for nonlinearity of the detector's response?", 

610 default=True, 

611 ) 

612 

613 # Crosstalk. 

614 doCrosstalk = pexConfig.Field( 

615 dtype=bool, 

616 doc="Apply intra-CCD crosstalk correction?", 

617 default=False, 

618 ) 

619 doCrosstalkBeforeAssemble = pexConfig.Field( 

620 dtype=bool, 

621 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

622 default=False, 

623 ) 

624 crosstalk = pexConfig.ConfigurableField( 

625 target=CrosstalkTask, 

626 doc="Intra-CCD crosstalk correction", 

627 ) 

628 

629 # Masking options. 

630 doDefect = pexConfig.Field( 

631 dtype=bool, 

632 doc="Apply correction for CCD defects, e.g. hot pixels?", 

633 default=True, 

634 ) 

635 doNanMasking = pexConfig.Field( 

636 dtype=bool, 

637 doc="Mask non-finite (NAN, inf) pixels?", 

638 default=True, 

639 ) 

640 doWidenSaturationTrails = pexConfig.Field( 

641 dtype=bool, 

642 doc="Widen bleed trails based on their width?", 

643 default=True 

644 ) 

645 

646 # Brighter-Fatter correction. 

647 doBrighterFatter = pexConfig.Field( 

648 dtype=bool, 

649 default=False, 

650 doc="Apply the brighter-fatter correction?" 

651 ) 

652 brighterFatterLevel = pexConfig.ChoiceField( 

653 dtype=str, 

654 default="DETECTOR", 

655 doc="The level at which to correct for brighter-fatter.", 

656 allowed={ 

657 "AMP": "Every amplifier treated separately.", 

658 "DETECTOR": "One kernel per detector", 

659 } 

660 ) 

661 brighterFatterMaxIter = pexConfig.Field( 

662 dtype=int, 

663 default=10, 

664 doc="Maximum number of iterations for the brighter-fatter correction" 

665 ) 

666 brighterFatterThreshold = pexConfig.Field( 

667 dtype=float, 

668 default=1000, 

669 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

670 "absolute value of the difference between the current corrected image and the one " 

671 "from the previous iteration summed over all the pixels." 

672 ) 

673 brighterFatterApplyGain = pexConfig.Field( 

674 dtype=bool, 

675 default=True, 

676 doc="Should the gain be applied when applying the brighter-fatter correction?" 

677 ) 

678 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

679 dtype=str, 

680 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

681 "correction.", 

682 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

683 ) 

684 brighterFatterMaskGrowSize = pexConfig.Field( 

685 dtype=int, 

686 default=0, 

687 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

688 "when brighter-fatter correction is applied." 

689 ) 

690 

691 # Dark subtraction. 

692 doDark = pexConfig.Field( 

693 dtype=bool, 

694 doc="Apply dark frame correction?", 

695 default=True, 

696 ) 

697 darkDataProductName = pexConfig.Field( 

698 dtype=str, 

699 doc="Name of the dark data product", 

700 default="dark", 

701 ) 

702 

703 # Camera-specific stray light removal. 

704 doStrayLight = pexConfig.Field( 

705 dtype=bool, 

706 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

707 default=False, 

708 ) 

709 strayLight = pexConfig.ConfigurableField( 

710 target=StrayLightTask, 

711 doc="y-band stray light correction" 

712 ) 

713 

714 # Flat correction. 

715 doFlat = pexConfig.Field( 

716 dtype=bool, 

717 doc="Apply flat field correction?", 

718 default=True, 

719 ) 

720 flatDataProductName = pexConfig.Field( 

721 dtype=str, 

722 doc="Name of the flat data product", 

723 default="flat", 

724 ) 

725 flatScalingType = pexConfig.ChoiceField( 

726 dtype=str, 

727 doc="The method for scaling the flat on the fly.", 

728 default='USER', 

729 allowed={ 

730 "USER": "Scale by flatUserScale", 

731 "MEAN": "Scale by the inverse of the mean", 

732 "MEDIAN": "Scale by the inverse of the median", 

733 }, 

734 ) 

735 flatUserScale = pexConfig.Field( 

736 dtype=float, 

737 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

738 default=1.0, 

739 ) 

740 doTweakFlat = pexConfig.Field( 

741 dtype=bool, 

742 doc="Tweak flats to match observed amplifier ratios?", 

743 default=False 

744 ) 

745 

746 # Amplifier normalization based on gains instead of using flats configuration. 

747 doApplyGains = pexConfig.Field( 

748 dtype=bool, 

749 doc="Correct the amplifiers for their gains instead of applying flat correction", 

750 default=False, 

751 ) 

752 usePtcGains = pexConfig.Field( 

753 dtype=bool, 

754 doc="Use the gain values from the Photon Transfer Curve?", 

755 default=False, 

756 ) 

757 normalizeGains = pexConfig.Field( 

758 dtype=bool, 

759 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

760 default=False, 

761 ) 

762 

763 # Fringe correction. 

764 doFringe = pexConfig.Field( 

765 dtype=bool, 

766 doc="Apply fringe correction?", 

767 default=True, 

768 ) 

769 fringe = pexConfig.ConfigurableField( 

770 target=FringeTask, 

771 doc="Fringe subtraction task", 

772 ) 

773 fringeAfterFlat = pexConfig.Field( 

774 dtype=bool, 

775 doc="Do fringe subtraction after flat-fielding?", 

776 default=True, 

777 ) 

778 

779 # Initial CCD-level background statistics options. 

780 doMeasureBackground = pexConfig.Field( 

781 dtype=bool, 

782 doc="Measure the background level on the reduced image?", 

783 default=False, 

784 ) 

785 

786 # Camera-specific masking configuration. 

787 doCameraSpecificMasking = pexConfig.Field( 

788 dtype=bool, 

789 doc="Mask camera-specific bad regions?", 

790 default=False, 

791 ) 

792 masking = pexConfig.ConfigurableField( 

793 target=MaskingTask, 

794 doc="Masking task." 

795 ) 

796 

797 # Interpolation options. 

798 

799 doInterpolate = pexConfig.Field( 

800 dtype=bool, 

801 doc="Interpolate masked pixels?", 

802 default=True, 

803 ) 

804 doSaturationInterpolation = pexConfig.Field( 

805 dtype=bool, 

806 doc="Perform interpolation over pixels masked as saturated?" 

807 " NB: This is independent of doSaturation; if that is False this plane" 

808 " will likely be blank, resulting in a no-op here.", 

809 default=True, 

810 ) 

811 doNanInterpolation = pexConfig.Field( 

812 dtype=bool, 

813 doc="Perform interpolation over pixels masked as NaN?" 

814 " NB: This is independent of doNanMasking; if that is False this plane" 

815 " will likely be blank, resulting in a no-op here.", 

816 default=True, 

817 ) 

818 doNanInterpAfterFlat = pexConfig.Field( 

819 dtype=bool, 

820 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

821 "also have to interpolate them before flat-fielding."), 

822 default=False, 

823 ) 

824 maskListToInterpolate = pexConfig.ListField( 

825 dtype=str, 

826 doc="List of mask planes that should be interpolated.", 

827 default=['SAT', 'BAD'], 

828 ) 

829 doSaveInterpPixels = pexConfig.Field( 

830 dtype=bool, 

831 doc="Save a copy of the pre-interpolated pixel values?", 

832 default=False, 

833 ) 

834 

835 # Default photometric calibration options. 

836 fluxMag0T1 = pexConfig.DictField( 

837 keytype=str, 

838 itemtype=float, 

839 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

840 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

841 )) 

842 ) 

843 defaultFluxMag0T1 = pexConfig.Field( 

844 dtype=float, 

845 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

846 default=pow(10.0, 0.4*28.0) 

847 ) 

848 

849 # Vignette correction configuration. 

850 doVignette = pexConfig.Field( 

851 dtype=bool, 

852 doc="Apply vignetting parameters?", 

853 default=False, 

854 ) 

855 vignette = pexConfig.ConfigurableField( 

856 target=VignetteTask, 

857 doc="Vignetting task.", 

858 ) 

859 

860 # Transmission curve configuration. 

861 doAttachTransmissionCurve = pexConfig.Field( 

862 dtype=bool, 

863 default=False, 

864 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

865 ) 

866 doUseOpticsTransmission = pexConfig.Field( 

867 dtype=bool, 

868 default=True, 

869 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

870 ) 

871 doUseFilterTransmission = pexConfig.Field( 

872 dtype=bool, 

873 default=True, 

874 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

875 ) 

876 doUseSensorTransmission = pexConfig.Field( 

877 dtype=bool, 

878 default=True, 

879 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

880 ) 

881 doUseAtmosphereTransmission = pexConfig.Field( 

882 dtype=bool, 

883 default=True, 

884 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

885 ) 

886 

887 # Illumination correction. 

888 doIlluminationCorrection = pexConfig.Field( 

889 dtype=bool, 

890 default=False, 

891 doc="Perform illumination correction?" 

892 ) 

893 illuminationCorrectionDataProductName = pexConfig.Field( 

894 dtype=str, 

895 doc="Name of the illumination correction data product.", 

896 default="illumcor", 

897 ) 

898 illumScale = pexConfig.Field( 

899 dtype=float, 

900 doc="Scale factor for the illumination correction.", 

901 default=1.0, 

902 ) 

903 illumFilters = pexConfig.ListField( 

904 dtype=str, 

905 default=[], 

906 doc="Only perform illumination correction for these filters." 

907 ) 

908 

909 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

910 doWrite = pexConfig.Field( 

911 dtype=bool, 

912 doc="Persist postISRCCD?", 

913 default=True, 

914 ) 

915 

916 def validate(self): 

917 super().validate() 

918 if self.doFlat and self.doApplyGains: 

919 raise ValueError("You may not specify both doFlat and doApplyGains") 

920 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

921 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

922 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

923 self.maskListToInterpolate.append(self.saturatedMaskName) 

924 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

925 self.maskListToInterpolate.remove(self.saturatedMaskName) 

926 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

927 self.maskListToInterpolate.append("UNMASKEDNAN") 

928 

929 

930class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

931 """Apply common instrument signature correction algorithms to a raw frame. 

932 

933 The process for correcting imaging data is very similar from 

934 camera to camera. This task provides a vanilla implementation of 

935 doing these corrections, including the ability to turn certain 

936 corrections off if they are not needed. The inputs to the primary 

937 method, `run()`, are a raw exposure to be corrected and the 

938 calibration data products. The raw input is a single chip sized 

939 mosaic of all amps including overscans and other non-science 

940 pixels. The method `runDataRef()` identifies and defines the 

941 calibration data products, and is intended for use by a 

942 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

943 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

944 subclassed for different camera, although the most camera specific 

945 methods have been split into subtasks that can be redirected 

946 appropriately. 

947 

948 The __init__ method sets up the subtasks for ISR processing, using 

949 the defaults from `lsst.ip.isr`. 

950 

951 Parameters 

952 ---------- 

953 args : `list` 

954 Positional arguments passed to the Task constructor. None used at this time. 

955 kwargs : `dict`, optional 

956 Keyword arguments passed on to the Task constructor. None used at this time. 

957 """ 

958 ConfigClass = IsrTaskConfig 

959 _DefaultName = "isr" 

960 

961 def __init__(self, **kwargs): 

962 super().__init__(**kwargs) 

963 self.makeSubtask("assembleCcd") 

964 self.makeSubtask("crosstalk") 

965 self.makeSubtask("strayLight") 

966 self.makeSubtask("fringe") 

967 self.makeSubtask("masking") 

968 self.makeSubtask("overscan") 

969 self.makeSubtask("vignette") 

970 

971 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

972 inputs = butlerQC.get(inputRefs) 

973 

974 try: 

975 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

976 except Exception as e: 

977 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

978 (inputRefs, e)) 

979 

980 inputs['isGen3'] = True 

981 

982 detector = inputs['ccdExposure'].getDetector() 

983 

984 if self.config.doCrosstalk is True: 

985 # Crosstalk sources need to be defined by the pipeline 

986 # yaml if they exist. 

987 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

988 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

989 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

990 else: 

991 coeffVector = (self.config.crosstalk.crosstalkValues 

992 if self.config.crosstalk.useConfigCoefficients else None) 

993 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

994 inputs['crosstalk'] = crosstalkCalib 

995 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

996 if 'crosstalkSources' not in inputs: 

997 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

998 

999 if self.doLinearize(detector) is True: 

1000 if 'linearizer' in inputs: 

1001 if isinstance(inputs['linearizer'], dict): 

1002 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1003 linearizer.fromYaml(inputs['linearizer']) 

1004 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1005 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1006 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1007 detector=detector, 

1008 log=self.log) 

1009 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1010 else: 

1011 linearizer = inputs['linearizer'] 

1012 linearizer.log = self.log 

1013 inputs['linearizer'] = linearizer 

1014 else: 

1015 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1016 self.log.warning("Constructing linearizer from cameraGeom information.") 

1017 

1018 if self.config.doDefect is True: 

1019 if "defects" in inputs and inputs['defects'] is not None: 

1020 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

1021 # masking expects a list of defects defined by their bounding box 

1022 if not isinstance(inputs["defects"], Defects): 

1023 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1024 

1025 # Load the correct style of brighter-fatter kernel, and repack 

1026 # the information as a numpy array. 

1027 if self.config.doBrighterFatter: 

1028 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1029 if brighterFatterKernel is None: 

1030 brighterFatterKernel = inputs.get('bfKernel', None) 

1031 

1032 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1033 # This is a ISR calib kernel 

1034 detName = detector.getName() 

1035 level = brighterFatterKernel.level 

1036 

1037 # This is expected to be a dictionary of amp-wise gains. 

1038 inputs['bfGains'] = brighterFatterKernel.gain 

1039 if self.config.brighterFatterLevel == 'DETECTOR': 

1040 if level == 'DETECTOR': 

1041 if detName in brighterFatterKernel.detKernels: 

1042 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1043 else: 

1044 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1045 elif level == 'AMP': 

1046 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1047 "fatter kernels.") 

1048 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1049 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1050 elif self.config.brighterFatterLevel == 'AMP': 

1051 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1052 

1053 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1054 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

1055 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1056 expId=expId, 

1057 assembler=self.assembleCcd 

1058 if self.config.doAssembleIsrExposures else None) 

1059 else: 

1060 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1061 

1062 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1063 if 'strayLightData' not in inputs: 

1064 inputs['strayLightData'] = None 

1065 

1066 outputs = self.run(**inputs) 

1067 butlerQC.put(outputs, outputRefs) 

1068 

1069 def readIsrData(self, dataRef, rawExposure): 

1070 """Retrieve necessary frames for instrument signature removal. 

1071 

1072 Pre-fetching all required ISR data products limits the IO 

1073 required by the ISR. Any conflict between the calibration data 

1074 available and that needed for ISR is also detected prior to 

1075 doing processing, allowing it to fail quickly. 

1076 

1077 Parameters 

1078 ---------- 

1079 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1080 Butler reference of the detector data to be processed 

1081 rawExposure : `afw.image.Exposure` 

1082 The raw exposure that will later be corrected with the 

1083 retrieved calibration data; should not be modified in this 

1084 method. 

1085 

1086 Returns 

1087 ------- 

1088 result : `lsst.pipe.base.Struct` 

1089 Result struct with components (which may be `None`): 

1090 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1091 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1092 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1093 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1094 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1095 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1096 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1097 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1098 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1099 - ``seed``: random seed derived from the ccdExposureId for random 

1100 number generator (`uint32`). 

1101 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1102 A ``TransmissionCurve`` that represents the throughput of the optics, 

1103 to be evaluated in focal-plane coordinates. 

1104 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1105 A ``TransmissionCurve`` that represents the throughput of the filter 

1106 itself, to be evaluated in focal-plane coordinates. 

1107 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1108 A ``TransmissionCurve`` that represents the throughput of the sensor 

1109 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1110 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1111 A ``TransmissionCurve`` that represents the throughput of the 

1112 atmosphere, assumed to be spatially constant. 

1113 - ``strayLightData`` : `object` 

1114 An opaque object containing calibration information for 

1115 stray-light correction. If `None`, no correction will be 

1116 performed. 

1117 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1118 

1119 Raises 

1120 ------ 

1121 NotImplementedError : 

1122 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1123 """ 

1124 try: 

1125 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1126 dateObs = dateObs.toPython().isoformat() 

1127 except RuntimeError: 

1128 self.log.warning("Unable to identify dateObs for rawExposure.") 

1129 dateObs = None 

1130 

1131 ccd = rawExposure.getDetector() 

1132 filterLabel = rawExposure.getFilterLabel() 

1133 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1134 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1135 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1136 if self.config.doBias else None) 

1137 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1138 linearizer = (dataRef.get("linearizer", immediate=True) 

1139 if self.doLinearize(ccd) else None) 

1140 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1141 linearizer.log = self.log 

1142 if isinstance(linearizer, numpy.ndarray): 

1143 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1144 

1145 crosstalkCalib = None 

1146 if self.config.doCrosstalk: 

1147 try: 

1148 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1149 except NoResults: 

1150 coeffVector = (self.config.crosstalk.crosstalkValues 

1151 if self.config.crosstalk.useConfigCoefficients else None) 

1152 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1153 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1154 if self.config.doCrosstalk else None) 

1155 

1156 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1157 if self.config.doDark else None) 

1158 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1159 dateObs=dateObs) 

1160 if self.config.doFlat else None) 

1161 

1162 brighterFatterKernel = None 

1163 brighterFatterGains = None 

1164 if self.config.doBrighterFatter is True: 

1165 try: 

1166 # Use the new-style cp_pipe version of the kernel if it exists 

1167 # If using a new-style kernel, always use the self-consistent 

1168 # gains, i.e. the ones inside the kernel object itself 

1169 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1170 brighterFatterGains = brighterFatterKernel.gain 

1171 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded") 

1172 except NoResults: 

1173 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1174 brighterFatterKernel = dataRef.get("bfKernel") 

1175 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded") 

1176 except NoResults: 

1177 brighterFatterKernel = None 

1178 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1179 # If the kernel is not an ndarray, it's the cp_pipe version 

1180 # so extract the kernel for this detector, or raise an error 

1181 if self.config.brighterFatterLevel == 'DETECTOR': 

1182 if brighterFatterKernel.detKernels: 

1183 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()] 

1184 else: 

1185 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1186 else: 

1187 # TODO DM-15631 for implementing this 

1188 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1189 

1190 defectList = (dataRef.get("defects") 

1191 if self.config.doDefect else None) 

1192 expId = rawExposure.getInfo().getVisitInfo().getExposureId() 

1193 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd 

1194 if self.config.doAssembleIsrExposures else None) 

1195 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1196 else pipeBase.Struct(fringes=None)) 

1197 

1198 if self.config.doAttachTransmissionCurve: 

1199 opticsTransmission = (dataRef.get("transmission_optics") 

1200 if self.config.doUseOpticsTransmission else None) 

1201 filterTransmission = (dataRef.get("transmission_filter") 

1202 if self.config.doUseFilterTransmission else None) 

1203 sensorTransmission = (dataRef.get("transmission_sensor") 

1204 if self.config.doUseSensorTransmission else None) 

1205 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1206 if self.config.doUseAtmosphereTransmission else None) 

1207 else: 

1208 opticsTransmission = None 

1209 filterTransmission = None 

1210 sensorTransmission = None 

1211 atmosphereTransmission = None 

1212 

1213 if self.config.doStrayLight: 

1214 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1215 else: 

1216 strayLightData = None 

1217 

1218 illumMaskedImage = (self.getIsrExposure(dataRef, 

1219 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1220 if (self.config.doIlluminationCorrection 

1221 and physicalFilter in self.config.illumFilters) 

1222 else None) 

1223 

1224 # Struct should include only kwargs to run() 

1225 return pipeBase.Struct(bias=biasExposure, 

1226 linearizer=linearizer, 

1227 crosstalk=crosstalkCalib, 

1228 crosstalkSources=crosstalkSources, 

1229 dark=darkExposure, 

1230 flat=flatExposure, 

1231 bfKernel=brighterFatterKernel, 

1232 bfGains=brighterFatterGains, 

1233 defects=defectList, 

1234 fringes=fringeStruct, 

1235 opticsTransmission=opticsTransmission, 

1236 filterTransmission=filterTransmission, 

1237 sensorTransmission=sensorTransmission, 

1238 atmosphereTransmission=atmosphereTransmission, 

1239 strayLightData=strayLightData, 

1240 illumMaskedImage=illumMaskedImage 

1241 ) 

1242 

1243 @pipeBase.timeMethod 

1244 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1245 crosstalk=None, crosstalkSources=None, 

1246 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1247 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1248 sensorTransmission=None, atmosphereTransmission=None, 

1249 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1250 isGen3=False, 

1251 ): 

1252 """Perform instrument signature removal on an exposure. 

1253 

1254 Steps included in the ISR processing, in order performed, are: 

1255 - saturation and suspect pixel masking 

1256 - overscan subtraction 

1257 - CCD assembly of individual amplifiers 

1258 - bias subtraction 

1259 - variance image construction 

1260 - linearization of non-linear response 

1261 - crosstalk masking 

1262 - brighter-fatter correction 

1263 - dark subtraction 

1264 - fringe correction 

1265 - stray light subtraction 

1266 - flat correction 

1267 - masking of known defects and camera specific features 

1268 - vignette calculation 

1269 - appending transmission curve and distortion model 

1270 

1271 Parameters 

1272 ---------- 

1273 ccdExposure : `lsst.afw.image.Exposure` 

1274 The raw exposure that is to be run through ISR. The 

1275 exposure is modified by this method. 

1276 camera : `lsst.afw.cameraGeom.Camera`, optional 

1277 The camera geometry for this exposure. Required if ``isGen3`` is 

1278 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1279 ``flat`` does not have an associated detector. 

1280 bias : `lsst.afw.image.Exposure`, optional 

1281 Bias calibration frame. 

1282 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1283 Functor for linearization. 

1284 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1285 Calibration for crosstalk. 

1286 crosstalkSources : `list`, optional 

1287 List of possible crosstalk sources. 

1288 dark : `lsst.afw.image.Exposure`, optional 

1289 Dark calibration frame. 

1290 flat : `lsst.afw.image.Exposure`, optional 

1291 Flat calibration frame. 

1292 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1293 Photon transfer curve dataset, with, e.g., gains 

1294 and read noise. 

1295 bfKernel : `numpy.ndarray`, optional 

1296 Brighter-fatter kernel. 

1297 bfGains : `dict` of `float`, optional 

1298 Gains used to override the detector's nominal gains for the 

1299 brighter-fatter correction. A dict keyed by amplifier name for 

1300 the detector in question. 

1301 defects : `lsst.ip.isr.Defects`, optional 

1302 List of defects. 

1303 fringes : `lsst.pipe.base.Struct`, optional 

1304 Struct containing the fringe correction data, with 

1305 elements: 

1306 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1307 - ``seed``: random seed derived from the ccdExposureId for random 

1308 number generator (`uint32`) 

1309 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1310 A ``TransmissionCurve`` that represents the throughput of the optics, 

1311 to be evaluated in focal-plane coordinates. 

1312 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1313 A ``TransmissionCurve`` that represents the throughput of the filter 

1314 itself, to be evaluated in focal-plane coordinates. 

1315 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1316 A ``TransmissionCurve`` that represents the throughput of the sensor 

1317 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1318 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1319 A ``TransmissionCurve`` that represents the throughput of the 

1320 atmosphere, assumed to be spatially constant. 

1321 detectorNum : `int`, optional 

1322 The integer number for the detector to process. 

1323 isGen3 : bool, optional 

1324 Flag this call to run() as using the Gen3 butler environment. 

1325 strayLightData : `object`, optional 

1326 Opaque object containing calibration information for stray-light 

1327 correction. If `None`, no correction will be performed. 

1328 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1329 Illumination correction image. 

1330 

1331 Returns 

1332 ------- 

1333 result : `lsst.pipe.base.Struct` 

1334 Result struct with component: 

1335 - ``exposure`` : `afw.image.Exposure` 

1336 The fully ISR corrected exposure. 

1337 - ``outputExposure`` : `afw.image.Exposure` 

1338 An alias for `exposure` 

1339 - ``ossThumb`` : `numpy.ndarray` 

1340 Thumbnail image of the exposure after overscan subtraction. 

1341 - ``flattenedThumb`` : `numpy.ndarray` 

1342 Thumbnail image of the exposure after flat-field correction. 

1343 

1344 Raises 

1345 ------ 

1346 RuntimeError 

1347 Raised if a configuration option is set to True, but the 

1348 required calibration data has not been specified. 

1349 

1350 Notes 

1351 ----- 

1352 The current processed exposure can be viewed by setting the 

1353 appropriate lsstDebug entries in the `debug.display` 

1354 dictionary. The names of these entries correspond to some of 

1355 the IsrTaskConfig Boolean options, with the value denoting the 

1356 frame to use. The exposure is shown inside the matching 

1357 option check and after the processing of that step has 

1358 finished. The steps with debug points are: 

1359 

1360 doAssembleCcd 

1361 doBias 

1362 doCrosstalk 

1363 doBrighterFatter 

1364 doDark 

1365 doFringe 

1366 doStrayLight 

1367 doFlat 

1368 

1369 In addition, setting the "postISRCCD" entry displays the 

1370 exposure after all ISR processing has finished. 

1371 

1372 """ 

1373 

1374 if isGen3 is True: 

1375 # Gen3 currently cannot automatically do configuration overrides. 

1376 # DM-15257 looks to discuss this issue. 

1377 # Configure input exposures; 

1378 if detectorNum is None: 

1379 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1380 

1381 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1382 bias = self.ensureExposure(bias, camera, detectorNum) 

1383 dark = self.ensureExposure(dark, camera, detectorNum) 

1384 flat = self.ensureExposure(flat, camera, detectorNum) 

1385 else: 

1386 if isinstance(ccdExposure, ButlerDataRef): 

1387 return self.runDataRef(ccdExposure) 

1388 

1389 ccd = ccdExposure.getDetector() 

1390 filterLabel = ccdExposure.getFilterLabel() 

1391 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1392 

1393 if not ccd: 

1394 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1395 ccd = [FakeAmp(ccdExposure, self.config)] 

1396 

1397 # Validate Input 

1398 if self.config.doBias and bias is None: 

1399 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1400 if self.doLinearize(ccd) and linearizer is None: 

1401 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1402 if self.config.doBrighterFatter and bfKernel is None: 

1403 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1404 if self.config.doDark and dark is None: 

1405 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1406 if self.config.doFlat and flat is None: 

1407 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1408 if self.config.doDefect and defects is None: 

1409 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1410 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1411 and fringes.fringes is None): 

1412 # The `fringes` object needs to be a pipeBase.Struct, as 

1413 # we use it as a `dict` for the parameters of 

1414 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1415 # not be `None` if `doFringe=True`. Otherwise, raise. 

1416 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1417 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1418 and illumMaskedImage is None): 

1419 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1420 

1421 # Begin ISR processing. 

1422 if self.config.doConvertIntToFloat: 

1423 self.log.info("Converting exposure to floating point values.") 

1424 ccdExposure = self.convertIntToFloat(ccdExposure) 

1425 

1426 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1427 self.log.info("Applying bias correction.") 

1428 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1429 trimToFit=self.config.doTrimToMatchCalib) 

1430 self.debugView(ccdExposure, "doBias") 

1431 

1432 # Amplifier level processing. 

1433 overscans = [] 

1434 for amp in ccd: 

1435 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1436 if ccdExposure.getBBox().contains(amp.getBBox()): 

1437 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1438 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1439 

1440 if self.config.doOverscan and not badAmp: 

1441 # Overscan correction on amp-by-amp basis. 

1442 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1443 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1444 if overscanResults is not None and \ 

1445 self.config.qa is not None and self.config.qa.saveStats is True: 

1446 if isinstance(overscanResults.overscanFit, float): 

1447 qaMedian = overscanResults.overscanFit 

1448 qaStdev = float("NaN") 

1449 else: 

1450 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1451 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1452 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1453 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1454 

1455 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian) 

1456 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev) 

1457 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1458 amp.getName(), qaMedian, qaStdev) 

1459 

1460 # Residuals after overscan correction 

1461 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1462 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1463 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1464 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1465 

1466 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter) 

1467 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter) 

1468 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1469 amp.getName(), qaMedianAfter, qaStdevAfter) 

1470 

1471 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1472 else: 

1473 if badAmp: 

1474 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1475 overscanResults = None 

1476 

1477 overscans.append(overscanResults if overscanResults is not None else None) 

1478 else: 

1479 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1480 

1481 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1482 self.log.info("Applying crosstalk correction.") 

1483 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1484 crosstalkSources=crosstalkSources, camera=camera) 

1485 self.debugView(ccdExposure, "doCrosstalk") 

1486 

1487 if self.config.doAssembleCcd: 

1488 self.log.info("Assembling CCD from amplifiers.") 

1489 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1490 

1491 if self.config.expectWcs and not ccdExposure.getWcs(): 

1492 self.log.warning("No WCS found in input exposure.") 

1493 self.debugView(ccdExposure, "doAssembleCcd") 

1494 

1495 ossThumb = None 

1496 if self.config.qa.doThumbnailOss: 

1497 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1498 

1499 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1500 self.log.info("Applying bias correction.") 

1501 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1502 trimToFit=self.config.doTrimToMatchCalib) 

1503 self.debugView(ccdExposure, "doBias") 

1504 

1505 if self.config.doVariance: 

1506 for amp, overscanResults in zip(ccd, overscans): 

1507 if ccdExposure.getBBox().contains(amp.getBBox()): 

1508 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1509 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1510 if overscanResults is not None: 

1511 self.updateVariance(ampExposure, amp, 

1512 overscanImage=overscanResults.overscanImage, 

1513 ptcDataset=ptc) 

1514 else: 

1515 self.updateVariance(ampExposure, amp, 

1516 overscanImage=None, 

1517 ptcDataset=ptc) 

1518 if self.config.qa is not None and self.config.qa.saveStats is True: 

1519 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1520 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1521 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1522 qaStats.getValue(afwMath.MEDIAN)) 

1523 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1524 qaStats.getValue(afwMath.STDEVCLIP)) 

1525 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1526 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1527 qaStats.getValue(afwMath.STDEVCLIP)) 

1528 if self.config.maskNegativeVariance: 

1529 self.maskNegativeVariance(ccdExposure) 

1530 

1531 if self.doLinearize(ccd): 

1532 self.log.info("Applying linearizer.") 

1533 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1534 detector=ccd, log=self.log) 

1535 

1536 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1537 self.log.info("Applying crosstalk correction.") 

1538 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1539 crosstalkSources=crosstalkSources, isTrimmed=True) 

1540 self.debugView(ccdExposure, "doCrosstalk") 

1541 

1542 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do 

1543 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1544 if self.config.doDefect: 

1545 self.log.info("Masking defects.") 

1546 self.maskDefect(ccdExposure, defects) 

1547 

1548 if self.config.numEdgeSuspect > 0: 

1549 self.log.info("Masking edges as SUSPECT.") 

1550 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1551 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1552 

1553 if self.config.doNanMasking: 

1554 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1555 self.maskNan(ccdExposure) 

1556 

1557 if self.config.doWidenSaturationTrails: 

1558 self.log.info("Widening saturation trails.") 

1559 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1560 

1561 if self.config.doCameraSpecificMasking: 

1562 self.log.info("Masking regions for camera specific reasons.") 

1563 self.masking.run(ccdExposure) 

1564 

1565 if self.config.doBrighterFatter: 

1566 # We need to apply flats and darks before we can interpolate, and we 

1567 # need to interpolate before we do B-F, but we do B-F without the 

1568 # flats and darks applied so we can work in units of electrons or holes. 

1569 # This context manager applies and then removes the darks and flats. 

1570 # 

1571 # We also do not want to interpolate values here, so operate on temporary 

1572 # images so we can apply only the BF-correction and roll back the 

1573 # interpolation. 

1574 interpExp = ccdExposure.clone() 

1575 with self.flatContext(interpExp, flat, dark): 

1576 isrFunctions.interpolateFromMask( 

1577 maskedImage=interpExp.getMaskedImage(), 

1578 fwhm=self.config.fwhm, 

1579 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1580 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1581 ) 

1582 bfExp = interpExp.clone() 

1583 

1584 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1585 type(bfKernel), type(bfGains)) 

1586 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1587 self.config.brighterFatterMaxIter, 

1588 self.config.brighterFatterThreshold, 

1589 self.config.brighterFatterApplyGain, 

1590 bfGains) 

1591 if bfResults[1] == self.config.brighterFatterMaxIter: 

1592 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1593 bfResults[0]) 

1594 else: 

1595 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1596 bfResults[1]) 

1597 image = ccdExposure.getMaskedImage().getImage() 

1598 bfCorr = bfExp.getMaskedImage().getImage() 

1599 bfCorr -= interpExp.getMaskedImage().getImage() 

1600 image += bfCorr 

1601 

1602 # Applying the brighter-fatter correction applies a 

1603 # convolution to the science image. At the edges this 

1604 # convolution may not have sufficient valid pixels to 

1605 # produce a valid correction. Mark pixels within the size 

1606 # of the brighter-fatter kernel as EDGE to warn of this 

1607 # fact. 

1608 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1609 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1610 maskPlane="EDGE") 

1611 

1612 if self.config.brighterFatterMaskGrowSize > 0: 

1613 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1614 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1615 isrFunctions.growMasks(ccdExposure.getMask(), 

1616 radius=self.config.brighterFatterMaskGrowSize, 

1617 maskNameList=maskPlane, 

1618 maskValue=maskPlane) 

1619 

1620 self.debugView(ccdExposure, "doBrighterFatter") 

1621 

1622 if self.config.doDark: 

1623 self.log.info("Applying dark correction.") 

1624 self.darkCorrection(ccdExposure, dark) 

1625 self.debugView(ccdExposure, "doDark") 

1626 

1627 if self.config.doFringe and not self.config.fringeAfterFlat: 

1628 self.log.info("Applying fringe correction before flat.") 

1629 self.fringe.run(ccdExposure, **fringes.getDict()) 

1630 self.debugView(ccdExposure, "doFringe") 

1631 

1632 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1633 self.log.info("Checking strayLight correction.") 

1634 self.strayLight.run(ccdExposure, strayLightData) 

1635 self.debugView(ccdExposure, "doStrayLight") 

1636 

1637 if self.config.doFlat: 

1638 self.log.info("Applying flat correction.") 

1639 self.flatCorrection(ccdExposure, flat) 

1640 self.debugView(ccdExposure, "doFlat") 

1641 

1642 if self.config.doApplyGains: 

1643 self.log.info("Applying gain correction instead of flat.") 

1644 if self.config.usePtcGains: 

1645 self.log.info("Using gains from the Photon Transfer Curve.") 

1646 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1647 ptcGains=ptc.gain) 

1648 else: 

1649 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1650 

1651 if self.config.doFringe and self.config.fringeAfterFlat: 

1652 self.log.info("Applying fringe correction after flat.") 

1653 self.fringe.run(ccdExposure, **fringes.getDict()) 

1654 

1655 if self.config.doVignette: 

1656 self.log.info("Constructing Vignette polygon.") 

1657 self.vignettePolygon = self.vignette.run(ccdExposure) 

1658 

1659 if self.config.vignette.doWriteVignettePolygon: 

1660 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1661 

1662 if self.config.doAttachTransmissionCurve: 

1663 self.log.info("Adding transmission curves.") 

1664 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1665 filterTransmission=filterTransmission, 

1666 sensorTransmission=sensorTransmission, 

1667 atmosphereTransmission=atmosphereTransmission) 

1668 

1669 flattenedThumb = None 

1670 if self.config.qa.doThumbnailFlattened: 

1671 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1672 

1673 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1674 self.log.info("Performing illumination correction.") 

1675 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1676 illumMaskedImage, illumScale=self.config.illumScale, 

1677 trimToFit=self.config.doTrimToMatchCalib) 

1678 

1679 preInterpExp = None 

1680 if self.config.doSaveInterpPixels: 

1681 preInterpExp = ccdExposure.clone() 

1682 

1683 # Reset and interpolate bad pixels. 

1684 # 

1685 # Large contiguous bad regions (which should have the BAD mask 

1686 # bit set) should have their values set to the image median. 

1687 # This group should include defects and bad amplifiers. As the 

1688 # area covered by these defects are large, there's little 

1689 # reason to expect that interpolation would provide a more 

1690 # useful value. 

1691 # 

1692 # Smaller defects can be safely interpolated after the larger 

1693 # regions have had their pixel values reset. This ensures 

1694 # that the remaining defects adjacent to bad amplifiers (as an 

1695 # example) do not attempt to interpolate extreme values. 

1696 if self.config.doSetBadRegions: 

1697 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1698 if badPixelCount > 0: 

1699 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1700 

1701 if self.config.doInterpolate: 

1702 self.log.info("Interpolating masked pixels.") 

1703 isrFunctions.interpolateFromMask( 

1704 maskedImage=ccdExposure.getMaskedImage(), 

1705 fwhm=self.config.fwhm, 

1706 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1707 maskNameList=list(self.config.maskListToInterpolate) 

1708 ) 

1709 

1710 self.roughZeroPoint(ccdExposure) 

1711 

1712 if self.config.doMeasureBackground: 

1713 self.log.info("Measuring background level.") 

1714 self.measureBackground(ccdExposure, self.config.qa) 

1715 

1716 if self.config.qa is not None and self.config.qa.saveStats is True: 

1717 for amp in ccd: 

1718 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1719 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1720 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1721 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1722 qaStats.getValue(afwMath.MEDIAN)) 

1723 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1724 qaStats.getValue(afwMath.STDEVCLIP)) 

1725 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1726 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1727 qaStats.getValue(afwMath.STDEVCLIP)) 

1728 

1729 self.debugView(ccdExposure, "postISRCCD") 

1730 

1731 return pipeBase.Struct( 

1732 exposure=ccdExposure, 

1733 ossThumb=ossThumb, 

1734 flattenedThumb=flattenedThumb, 

1735 

1736 preInterpExposure=preInterpExp, 

1737 outputExposure=ccdExposure, 

1738 outputOssThumbnail=ossThumb, 

1739 outputFlattenedThumbnail=flattenedThumb, 

1740 ) 

1741 

1742 @pipeBase.timeMethod 

1743 def runDataRef(self, sensorRef): 

1744 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1745 

1746 This method contains the `CmdLineTask` interface to the ISR 

1747 processing. All IO is handled here, freeing the `run()` method 

1748 to manage only pixel-level calculations. The steps performed 

1749 are: 

1750 - Read in necessary detrending/isr/calibration data. 

1751 - Process raw exposure in `run()`. 

1752 - Persist the ISR-corrected exposure as "postISRCCD" if 

1753 config.doWrite=True. 

1754 

1755 Parameters 

1756 ---------- 

1757 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1758 DataRef of the detector data to be processed 

1759 

1760 Returns 

1761 ------- 

1762 result : `lsst.pipe.base.Struct` 

1763 Result struct with component: 

1764 - ``exposure`` : `afw.image.Exposure` 

1765 The fully ISR corrected exposure. 

1766 

1767 Raises 

1768 ------ 

1769 RuntimeError 

1770 Raised if a configuration option is set to True, but the 

1771 required calibration data does not exist. 

1772 

1773 """ 

1774 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1775 

1776 ccdExposure = sensorRef.get(self.config.datasetType) 

1777 

1778 camera = sensorRef.get("camera") 

1779 isrData = self.readIsrData(sensorRef, ccdExposure) 

1780 

1781 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1782 

1783 if self.config.doWrite: 

1784 sensorRef.put(result.exposure, "postISRCCD") 

1785 if result.preInterpExposure is not None: 

1786 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated") 

1787 if result.ossThumb is not None: 

1788 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1789 if result.flattenedThumb is not None: 

1790 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1791 

1792 return result 

1793 

1794 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1795 """Retrieve a calibration dataset for removing instrument signature. 

1796 

1797 Parameters 

1798 ---------- 

1799 

1800 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1801 DataRef of the detector data to find calibration datasets 

1802 for. 

1803 datasetType : `str` 

1804 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1805 dateObs : `str`, optional 

1806 Date of the observation. Used to correct butler failures 

1807 when using fallback filters. 

1808 immediate : `Bool` 

1809 If True, disable butler proxies to enable error handling 

1810 within this routine. 

1811 

1812 Returns 

1813 ------- 

1814 exposure : `lsst.afw.image.Exposure` 

1815 Requested calibration frame. 

1816 

1817 Raises 

1818 ------ 

1819 RuntimeError 

1820 Raised if no matching calibration frame can be found. 

1821 """ 

1822 try: 

1823 exp = dataRef.get(datasetType, immediate=immediate) 

1824 except Exception as exc1: 

1825 if not self.config.fallbackFilterName: 

1826 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1827 try: 

1828 if self.config.useFallbackDate and dateObs: 

1829 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1830 dateObs=dateObs, immediate=immediate) 

1831 else: 

1832 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1833 except Exception as exc2: 

1834 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1835 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1836 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1837 

1838 if self.config.doAssembleIsrExposures: 

1839 exp = self.assembleCcd.assembleCcd(exp) 

1840 return exp 

1841 

1842 def ensureExposure(self, inputExp, camera, detectorNum): 

1843 """Ensure that the data returned by Butler is a fully constructed exposure. 

1844 

1845 ISR requires exposure-level image data for historical reasons, so if we did 

1846 not recieve that from Butler, construct it from what we have, modifying the 

1847 input in place. 

1848 

1849 Parameters 

1850 ---------- 

1851 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1852 `lsst.afw.image.ImageF` 

1853 The input data structure obtained from Butler. 

1854 camera : `lsst.afw.cameraGeom.camera` 

1855 The camera associated with the image. Used to find the appropriate 

1856 detector. 

1857 detectorNum : `int` 

1858 The detector this exposure should match. 

1859 

1860 Returns 

1861 ------- 

1862 inputExp : `lsst.afw.image.Exposure` 

1863 The re-constructed exposure, with appropriate detector parameters. 

1864 

1865 Raises 

1866 ------ 

1867 TypeError 

1868 Raised if the input data cannot be used to construct an exposure. 

1869 """ 

1870 if isinstance(inputExp, afwImage.DecoratedImageU): 

1871 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1872 elif isinstance(inputExp, afwImage.ImageF): 

1873 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1874 elif isinstance(inputExp, afwImage.MaskedImageF): 

1875 inputExp = afwImage.makeExposure(inputExp) 

1876 elif isinstance(inputExp, afwImage.Exposure): 

1877 pass 

1878 elif inputExp is None: 

1879 # Assume this will be caught by the setup if it is a problem. 

1880 return inputExp 

1881 else: 

1882 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1883 (type(inputExp), )) 

1884 

1885 if inputExp.getDetector() is None: 

1886 inputExp.setDetector(camera[detectorNum]) 

1887 

1888 return inputExp 

1889 

1890 def convertIntToFloat(self, exposure): 

1891 """Convert exposure image from uint16 to float. 

1892 

1893 If the exposure does not need to be converted, the input is 

1894 immediately returned. For exposures that are converted to use 

1895 floating point pixels, the variance is set to unity and the 

1896 mask to zero. 

1897 

1898 Parameters 

1899 ---------- 

1900 exposure : `lsst.afw.image.Exposure` 

1901 The raw exposure to be converted. 

1902 

1903 Returns 

1904 ------- 

1905 newexposure : `lsst.afw.image.Exposure` 

1906 The input ``exposure``, converted to floating point pixels. 

1907 

1908 Raises 

1909 ------ 

1910 RuntimeError 

1911 Raised if the exposure type cannot be converted to float. 

1912 

1913 """ 

1914 if isinstance(exposure, afwImage.ExposureF): 

1915 # Nothing to be done 

1916 self.log.debug("Exposure already of type float.") 

1917 return exposure 

1918 if not hasattr(exposure, "convertF"): 

1919 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1920 

1921 newexposure = exposure.convertF() 

1922 newexposure.variance[:] = 1 

1923 newexposure.mask[:] = 0x0 

1924 

1925 return newexposure 

1926 

1927 def maskAmplifier(self, ccdExposure, amp, defects): 

1928 """Identify bad amplifiers, saturated and suspect pixels. 

1929 

1930 Parameters 

1931 ---------- 

1932 ccdExposure : `lsst.afw.image.Exposure` 

1933 Input exposure to be masked. 

1934 amp : `lsst.afw.table.AmpInfoCatalog` 

1935 Catalog of parameters defining the amplifier on this 

1936 exposure to mask. 

1937 defects : `lsst.ip.isr.Defects` 

1938 List of defects. Used to determine if the entire 

1939 amplifier is bad. 

1940 

1941 Returns 

1942 ------- 

1943 badAmp : `Bool` 

1944 If this is true, the entire amplifier area is covered by 

1945 defects and unusable. 

1946 

1947 """ 

1948 maskedImage = ccdExposure.getMaskedImage() 

1949 

1950 badAmp = False 

1951 

1952 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1953 # comparison with current defects definition. 

1954 if defects is not None: 

1955 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1956 

1957 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1958 # association with pixels in current ccdExposure). 

1959 if badAmp: 

1960 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1961 afwImage.PARENT) 

1962 maskView = dataView.getMask() 

1963 maskView |= maskView.getPlaneBitMask("BAD") 

1964 del maskView 

1965 return badAmp 

1966 

1967 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1968 # Saturation and suspect pixels can be masked now, though. 

1969 limits = dict() 

1970 if self.config.doSaturation and not badAmp: 

1971 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1972 if self.config.doSuspect and not badAmp: 

1973 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1974 if math.isfinite(self.config.saturation): 

1975 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1976 

1977 for maskName, maskThreshold in limits.items(): 

1978 if not math.isnan(maskThreshold): 

1979 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1980 isrFunctions.makeThresholdMask( 

1981 maskedImage=dataView, 

1982 threshold=maskThreshold, 

1983 growFootprints=0, 

1984 maskName=maskName 

1985 ) 

1986 

1987 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1988 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1989 afwImage.PARENT) 

1990 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1991 self.config.suspectMaskName]) 

1992 if numpy.all(maskView.getArray() & maskVal > 0): 

1993 badAmp = True 

1994 maskView |= maskView.getPlaneBitMask("BAD") 

1995 

1996 return badAmp 

1997 

1998 def overscanCorrection(self, ccdExposure, amp): 

1999 """Apply overscan correction in place. 

2000 

2001 This method does initial pixel rejection of the overscan 

2002 region. The overscan can also be optionally segmented to 

2003 allow for discontinuous overscan responses to be fit 

2004 separately. The actual overscan subtraction is performed by 

2005 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

2006 which is called here after the amplifier is preprocessed. 

2007 

2008 Parameters 

2009 ---------- 

2010 ccdExposure : `lsst.afw.image.Exposure` 

2011 Exposure to have overscan correction performed. 

2012 amp : `lsst.afw.cameraGeom.Amplifer` 

2013 The amplifier to consider while correcting the overscan. 

2014 

2015 Returns 

2016 ------- 

2017 overscanResults : `lsst.pipe.base.Struct` 

2018 Result struct with components: 

2019 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

2020 Value or fit subtracted from the amplifier image data. 

2021 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

2022 Value or fit subtracted from the overscan image data. 

2023 - ``overscanImage`` : `lsst.afw.image.Image` 

2024 Image of the overscan region with the overscan 

2025 correction applied. This quantity is used to estimate 

2026 the amplifier read noise empirically. 

2027 

2028 Raises 

2029 ------ 

2030 RuntimeError 

2031 Raised if the ``amp`` does not contain raw pixel information. 

2032 

2033 See Also 

2034 -------- 

2035 lsst.ip.isr.isrFunctions.overscanCorrection 

2036 """ 

2037 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2038 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2039 return None 

2040 

2041 statControl = afwMath.StatisticsControl() 

2042 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2043 

2044 # Determine the bounding boxes 

2045 dataBBox = amp.getRawDataBBox() 

2046 oscanBBox = amp.getRawHorizontalOverscanBBox() 

2047 dx0 = 0 

2048 dx1 = 0 

2049 

2050 prescanBBox = amp.getRawPrescanBBox() 

2051 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

2052 dx0 += self.config.overscanNumLeadingColumnsToSkip 

2053 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

2054 else: 

2055 dx0 += self.config.overscanNumTrailingColumnsToSkip 

2056 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

2057 

2058 # Determine if we need to work on subregions of the amplifier and overscan. 

2059 imageBBoxes = [] 

2060 overscanBBoxes = [] 

2061 

2062 if ((self.config.overscanBiasJump 

2063 and self.config.overscanBiasJumpLocation) 

2064 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

2065 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

2066 self.config.overscanBiasJumpDevices)): 

2067 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

2068 yLower = self.config.overscanBiasJumpLocation 

2069 yUpper = dataBBox.getHeight() - yLower 

2070 else: 

2071 yUpper = self.config.overscanBiasJumpLocation 

2072 yLower = dataBBox.getHeight() - yUpper 

2073 

2074 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2075 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

2076 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2077 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2078 yLower))) 

2079 

2080 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

2081 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

2082 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

2083 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2084 yUpper))) 

2085 else: 

2086 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2087 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

2088 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2089 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2090 oscanBBox.getHeight()))) 

2091 

2092 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

2093 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

2094 ampImage = ccdExposure.maskedImage[imageBBox] 

2095 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2096 

2097 overscanArray = overscanImage.image.array 

2098 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2099 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2100 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2101 

2102 statControl = afwMath.StatisticsControl() 

2103 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2104 

2105 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2106 

2107 # Measure average overscan levels and record them in the metadata. 

2108 levelStat = afwMath.MEDIAN 

2109 sigmaStat = afwMath.STDEVCLIP 

2110 

2111 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2112 self.config.qa.flatness.nIter) 

2113 metadata = ccdExposure.getMetadata() 

2114 ampNum = amp.getName() 

2115 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2116 if isinstance(overscanResults.overscanFit, float): 

2117 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2118 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2119 else: 

2120 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2121 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2122 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2123 

2124 return overscanResults 

2125 

2126 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2127 """Set the variance plane using the gain and read noise 

2128 

2129 The read noise is calculated from the ``overscanImage`` if the 

2130 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2131 the value from the amplifier data is used. 

2132 

2133 Parameters 

2134 ---------- 

2135 ampExposure : `lsst.afw.image.Exposure` 

2136 Exposure to process. 

2137 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2138 Amplifier detector data. 

2139 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2140 Image of overscan, required only for empirical read noise. 

2141 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2142 PTC dataset containing the gains and read noise. 

2143 

2144 

2145 Raises 

2146 ------ 

2147 RuntimeError 

2148 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2149 are ``True``, but ptcDataset is not provided. 

2150 

2151 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2152 ``overscanImage`` is ``None``. 

2153 

2154 See also 

2155 -------- 

2156 lsst.ip.isr.isrFunctions.updateVariance 

2157 """ 

2158 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2159 if self.config.usePtcGains: 

2160 if ptcDataset is None: 

2161 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2162 else: 

2163 gain = ptcDataset.gain[amp.getName()] 

2164 self.log.info("Using gain from Photon Transfer Curve.") 

2165 else: 

2166 gain = amp.getGain() 

2167 

2168 if math.isnan(gain): 

2169 gain = 1.0 

2170 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2171 elif gain <= 0: 

2172 patchedGain = 1.0 

2173 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2174 amp.getName(), gain, patchedGain) 

2175 gain = patchedGain 

2176 

2177 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2178 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2179 

2180 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2181 stats = afwMath.StatisticsControl() 

2182 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2183 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2184 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2185 amp.getName(), readNoise) 

2186 elif self.config.usePtcReadNoise: 

2187 if ptcDataset is None: 

2188 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2189 else: 

2190 readNoise = ptcDataset.noise[amp.getName()] 

2191 self.log.info("Using read noise from Photon Transfer Curve.") 

2192 else: 

2193 readNoise = amp.getReadNoise() 

2194 

2195 isrFunctions.updateVariance( 

2196 maskedImage=ampExposure.getMaskedImage(), 

2197 gain=gain, 

2198 readNoise=readNoise, 

2199 ) 

2200 

2201 def maskNegativeVariance(self, exposure): 

2202 """Identify and mask pixels with negative variance values. 

2203 

2204 Parameters 

2205 ---------- 

2206 exposure : `lsst.afw.image.Exposure` 

2207 Exposure to process. 

2208 

2209 See Also 

2210 -------- 

2211 lsst.ip.isr.isrFunctions.updateVariance 

2212 """ 

2213 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2214 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2215 exposure.mask.array[bad] |= maskPlane 

2216 

2217 def darkCorrection(self, exposure, darkExposure, invert=False): 

2218 """Apply dark correction in place. 

2219 

2220 Parameters 

2221 ---------- 

2222 exposure : `lsst.afw.image.Exposure` 

2223 Exposure to process. 

2224 darkExposure : `lsst.afw.image.Exposure` 

2225 Dark exposure of the same size as ``exposure``. 

2226 invert : `Bool`, optional 

2227 If True, re-add the dark to an already corrected image. 

2228 

2229 Raises 

2230 ------ 

2231 RuntimeError 

2232 Raised if either ``exposure`` or ``darkExposure`` do not 

2233 have their dark time defined. 

2234 

2235 See Also 

2236 -------- 

2237 lsst.ip.isr.isrFunctions.darkCorrection 

2238 """ 

2239 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2240 if math.isnan(expScale): 

2241 raise RuntimeError("Exposure darktime is NAN.") 

2242 if darkExposure.getInfo().getVisitInfo() is not None \ 

2243 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2244 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2245 else: 

2246 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2247 # so getDarkTime() does not exist. 

2248 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2249 darkScale = 1.0 

2250 

2251 isrFunctions.darkCorrection( 

2252 maskedImage=exposure.getMaskedImage(), 

2253 darkMaskedImage=darkExposure.getMaskedImage(), 

2254 expScale=expScale, 

2255 darkScale=darkScale, 

2256 invert=invert, 

2257 trimToFit=self.config.doTrimToMatchCalib 

2258 ) 

2259 

2260 def doLinearize(self, detector): 

2261 """Check if linearization is needed for the detector cameraGeom. 

2262 

2263 Checks config.doLinearize and the linearity type of the first 

2264 amplifier. 

2265 

2266 Parameters 

2267 ---------- 

2268 detector : `lsst.afw.cameraGeom.Detector` 

2269 Detector to get linearity type from. 

2270 

2271 Returns 

2272 ------- 

2273 doLinearize : `Bool` 

2274 If True, linearization should be performed. 

2275 """ 

2276 return self.config.doLinearize and \ 

2277 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2278 

2279 def flatCorrection(self, exposure, flatExposure, invert=False): 

2280 """Apply flat correction in place. 

2281 

2282 Parameters 

2283 ---------- 

2284 exposure : `lsst.afw.image.Exposure` 

2285 Exposure to process. 

2286 flatExposure : `lsst.afw.image.Exposure` 

2287 Flat exposure of the same size as ``exposure``. 

2288 invert : `Bool`, optional 

2289 If True, unflatten an already flattened image. 

2290 

2291 See Also 

2292 -------- 

2293 lsst.ip.isr.isrFunctions.flatCorrection 

2294 """ 

2295 isrFunctions.flatCorrection( 

2296 maskedImage=exposure.getMaskedImage(), 

2297 flatMaskedImage=flatExposure.getMaskedImage(), 

2298 scalingType=self.config.flatScalingType, 

2299 userScale=self.config.flatUserScale, 

2300 invert=invert, 

2301 trimToFit=self.config.doTrimToMatchCalib 

2302 ) 

2303 

2304 def saturationDetection(self, exposure, amp): 

2305 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2306 

2307 Parameters 

2308 ---------- 

2309 exposure : `lsst.afw.image.Exposure` 

2310 Exposure to process. Only the amplifier DataSec is processed. 

2311 amp : `lsst.afw.table.AmpInfoCatalog` 

2312 Amplifier detector data. 

2313 

2314 See Also 

2315 -------- 

2316 lsst.ip.isr.isrFunctions.makeThresholdMask 

2317 """ 

2318 if not math.isnan(amp.getSaturation()): 

2319 maskedImage = exposure.getMaskedImage() 

2320 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2321 isrFunctions.makeThresholdMask( 

2322 maskedImage=dataView, 

2323 threshold=amp.getSaturation(), 

2324 growFootprints=0, 

2325 maskName=self.config.saturatedMaskName, 

2326 ) 

2327 

2328 def saturationInterpolation(self, exposure): 

2329 """Interpolate over saturated pixels, in place. 

2330 

2331 This method should be called after `saturationDetection`, to 

2332 ensure that the saturated pixels have been identified in the 

2333 SAT mask. It should also be called after `assembleCcd`, since 

2334 saturated regions may cross amplifier boundaries. 

2335 

2336 Parameters 

2337 ---------- 

2338 exposure : `lsst.afw.image.Exposure` 

2339 Exposure to process. 

2340 

2341 See Also 

2342 -------- 

2343 lsst.ip.isr.isrTask.saturationDetection 

2344 lsst.ip.isr.isrFunctions.interpolateFromMask 

2345 """ 

2346 isrFunctions.interpolateFromMask( 

2347 maskedImage=exposure.getMaskedImage(), 

2348 fwhm=self.config.fwhm, 

2349 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2350 maskNameList=list(self.config.saturatedMaskName), 

2351 ) 

2352 

2353 def suspectDetection(self, exposure, amp): 

2354 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2355 

2356 Parameters 

2357 ---------- 

2358 exposure : `lsst.afw.image.Exposure` 

2359 Exposure to process. Only the amplifier DataSec is processed. 

2360 amp : `lsst.afw.table.AmpInfoCatalog` 

2361 Amplifier detector data. 

2362 

2363 See Also 

2364 -------- 

2365 lsst.ip.isr.isrFunctions.makeThresholdMask 

2366 

2367 Notes 

2368 ----- 

2369 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2370 This is intended to indicate pixels that may be affected by unknown systematics; 

2371 for example if non-linearity corrections above a certain level are unstable 

2372 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2373 that no such level exists and no pixels are to be masked as suspicious. 

2374 """ 

2375 suspectLevel = amp.getSuspectLevel() 

2376 if math.isnan(suspectLevel): 

2377 return 

2378 

2379 maskedImage = exposure.getMaskedImage() 

2380 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2381 isrFunctions.makeThresholdMask( 

2382 maskedImage=dataView, 

2383 threshold=suspectLevel, 

2384 growFootprints=0, 

2385 maskName=self.config.suspectMaskName, 

2386 ) 

2387 

2388 def maskDefect(self, exposure, defectBaseList): 

2389 """Mask defects using mask plane "BAD", in place. 

2390 

2391 Parameters 

2392 ---------- 

2393 exposure : `lsst.afw.image.Exposure` 

2394 Exposure to process. 

2395 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2396 `lsst.afw.image.DefectBase`. 

2397 List of defects to mask. 

2398 

2399 Notes 

2400 ----- 

2401 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2402 """ 

2403 maskedImage = exposure.getMaskedImage() 

2404 if not isinstance(defectBaseList, Defects): 

2405 # Promotes DefectBase to Defect 

2406 defectList = Defects(defectBaseList) 

2407 else: 

2408 defectList = defectBaseList 

2409 defectList.maskPixels(maskedImage, maskName="BAD") 

2410 

2411 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2412 """Mask edge pixels with applicable mask plane. 

2413 

2414 Parameters 

2415 ---------- 

2416 exposure : `lsst.afw.image.Exposure` 

2417 Exposure to process. 

2418 numEdgePixels : `int`, optional 

2419 Number of edge pixels to mask. 

2420 maskPlane : `str`, optional 

2421 Mask plane name to use. 

2422 level : `str`, optional 

2423 Level at which to mask edges. 

2424 """ 

2425 maskedImage = exposure.getMaskedImage() 

2426 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2427 

2428 if numEdgePixels > 0: 

2429 if level == 'DETECTOR': 

2430 boxes = [maskedImage.getBBox()] 

2431 elif level == 'AMP': 

2432 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2433 

2434 for box in boxes: 

2435 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2436 subImage = maskedImage[box] 

2437 box.grow(-numEdgePixels) 

2438 # Mask pixels outside box 

2439 SourceDetectionTask.setEdgeBits( 

2440 subImage, 

2441 box, 

2442 maskBitMask) 

2443 

2444 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2445 """Mask and interpolate defects using mask plane "BAD", in place. 

2446 

2447 Parameters 

2448 ---------- 

2449 exposure : `lsst.afw.image.Exposure` 

2450 Exposure to process. 

2451 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2452 `lsst.afw.image.DefectBase`. 

2453 List of defects to mask and interpolate. 

2454 

2455 See Also 

2456 -------- 

2457 lsst.ip.isr.isrTask.maskDefect 

2458 """ 

2459 self.maskDefect(exposure, defectBaseList) 

2460 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2461 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2462 isrFunctions.interpolateFromMask( 

2463 maskedImage=exposure.getMaskedImage(), 

2464 fwhm=self.config.fwhm, 

2465 growSaturatedFootprints=0, 

2466 maskNameList=["BAD"], 

2467 ) 

2468 

2469 def maskNan(self, exposure): 

2470 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2471 

2472 Parameters 

2473 ---------- 

2474 exposure : `lsst.afw.image.Exposure` 

2475 Exposure to process. 

2476 

2477 Notes 

2478 ----- 

2479 We mask over all non-finite values (NaN, inf), including those 

2480 that are masked with other bits (because those may or may not be 

2481 interpolated over later, and we want to remove all NaN/infs). 

2482 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2483 preserve the historical name. 

2484 """ 

2485 maskedImage = exposure.getMaskedImage() 

2486 

2487 # Find and mask NaNs 

2488 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2489 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2490 numNans = maskNans(maskedImage, maskVal) 

2491 self.metadata.set("NUMNANS", numNans) 

2492 if numNans > 0: 

2493 self.log.warning("There were %d unmasked NaNs.", numNans) 

2494 

2495 def maskAndInterpolateNan(self, exposure): 

2496 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2497 in place. 

2498 

2499 Parameters 

2500 ---------- 

2501 exposure : `lsst.afw.image.Exposure` 

2502 Exposure to process. 

2503 

2504 See Also 

2505 -------- 

2506 lsst.ip.isr.isrTask.maskNan 

2507 """ 

2508 self.maskNan(exposure) 

2509 isrFunctions.interpolateFromMask( 

2510 maskedImage=exposure.getMaskedImage(), 

2511 fwhm=self.config.fwhm, 

2512 growSaturatedFootprints=0, 

2513 maskNameList=["UNMASKEDNAN"], 

2514 ) 

2515 

2516 def measureBackground(self, exposure, IsrQaConfig=None): 

2517 """Measure the image background in subgrids, for quality control purposes. 

2518 

2519 Parameters 

2520 ---------- 

2521 exposure : `lsst.afw.image.Exposure` 

2522 Exposure to process. 

2523 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2524 Configuration object containing parameters on which background 

2525 statistics and subgrids to use. 

2526 """ 

2527 if IsrQaConfig is not None: 

2528 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2529 IsrQaConfig.flatness.nIter) 

2530 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2531 statsControl.setAndMask(maskVal) 

2532 maskedImage = exposure.getMaskedImage() 

2533 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2534 skyLevel = stats.getValue(afwMath.MEDIAN) 

2535 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2536 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2537 metadata = exposure.getMetadata() 

2538 metadata.set('SKYLEVEL', skyLevel) 

2539 metadata.set('SKYSIGMA', skySigma) 

2540 

2541 # calcluating flatlevel over the subgrids 

2542 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2543 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2544 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2545 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2546 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2547 skyLevels = numpy.zeros((nX, nY)) 

2548 

2549 for j in range(nY): 

2550 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2551 for i in range(nX): 

2552 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2553 

2554 xLLC = xc - meshXHalf 

2555 yLLC = yc - meshYHalf 

2556 xURC = xc + meshXHalf - 1 

2557 yURC = yc + meshYHalf - 1 

2558 

2559 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2560 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2561 

2562 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2563 

2564 good = numpy.where(numpy.isfinite(skyLevels)) 

2565 skyMedian = numpy.median(skyLevels[good]) 

2566 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2567 flatness_rms = numpy.std(flatness) 

2568 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2569 

2570 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2571 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2572 nX, nY, flatness_pp, flatness_rms) 

2573 

2574 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2575 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2576 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2577 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2578 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2579 

2580 def roughZeroPoint(self, exposure): 

2581 """Set an approximate magnitude zero point for the exposure. 

2582 

2583 Parameters 

2584 ---------- 

2585 exposure : `lsst.afw.image.Exposure` 

2586 Exposure to process. 

2587 """ 

2588 filterLabel = exposure.getFilterLabel() 

2589 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2590 

2591 if physicalFilter in self.config.fluxMag0T1: 

2592 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2593 else: 

2594 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2595 fluxMag0 = self.config.defaultFluxMag0T1 

2596 

2597 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2598 if not expTime > 0: # handle NaN as well as <= 0 

2599 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2600 return 

2601 

2602 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2603 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2604 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2605 

2606 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2607 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2608 

2609 Parameters 

2610 ---------- 

2611 ccdExposure : `lsst.afw.image.Exposure` 

2612 Exposure to process. 

2613 fpPolygon : `lsst.afw.geom.Polygon` 

2614 Polygon in focal plane coordinates. 

2615 """ 

2616 # Get ccd corners in focal plane coordinates 

2617 ccd = ccdExposure.getDetector() 

2618 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2619 ccdPolygon = Polygon(fpCorners) 

2620 

2621 # Get intersection of ccd corners with fpPolygon 

2622 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2623 

2624 # Transform back to pixel positions and build new polygon 

2625 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2626 validPolygon = Polygon(ccdPoints) 

2627 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2628 

2629 @contextmanager 

2630 def flatContext(self, exp, flat, dark=None): 

2631 """Context manager that applies and removes flats and darks, 

2632 if the task is configured to apply them. 

2633 

2634 Parameters 

2635 ---------- 

2636 exp : `lsst.afw.image.Exposure` 

2637 Exposure to process. 

2638 flat : `lsst.afw.image.Exposure` 

2639 Flat exposure the same size as ``exp``. 

2640 dark : `lsst.afw.image.Exposure`, optional 

2641 Dark exposure the same size as ``exp``. 

2642 

2643 Yields 

2644 ------ 

2645 exp : `lsst.afw.image.Exposure` 

2646 The flat and dark corrected exposure. 

2647 """ 

2648 if self.config.doDark and dark is not None: 

2649 self.darkCorrection(exp, dark) 

2650 if self.config.doFlat: 

2651 self.flatCorrection(exp, flat) 

2652 try: 

2653 yield exp 

2654 finally: 

2655 if self.config.doFlat: 

2656 self.flatCorrection(exp, flat, invert=True) 

2657 if self.config.doDark and dark is not None: 

2658 self.darkCorrection(exp, dark, invert=True) 

2659 

2660 def debugView(self, exposure, stepname): 

2661 """Utility function to examine ISR exposure at different stages. 

2662 

2663 Parameters 

2664 ---------- 

2665 exposure : `lsst.afw.image.Exposure` 

2666 Exposure to view. 

2667 stepname : `str` 

2668 State of processing to view. 

2669 """ 

2670 frame = getDebugFrame(self._display, stepname) 

2671 if frame: 

2672 display = getDisplay(frame) 

2673 display.scale('asinh', 'zscale') 

2674 display.mtv(exposure) 

2675 prompt = "Press Enter to continue [c]... " 

2676 while True: 

2677 ans = input(prompt).lower() 

2678 if ans in ("", "c",): 

2679 break 

2680 

2681 

2682class FakeAmp(object): 

2683 """A Detector-like object that supports returning gain and saturation level 

2684 

2685 This is used when the input exposure does not have a detector. 

2686 

2687 Parameters 

2688 ---------- 

2689 exposure : `lsst.afw.image.Exposure` 

2690 Exposure to generate a fake amplifier for. 

2691 config : `lsst.ip.isr.isrTaskConfig` 

2692 Configuration to apply to the fake amplifier. 

2693 """ 

2694 

2695 def __init__(self, exposure, config): 

2696 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2697 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2698 self._gain = config.gain 

2699 self._readNoise = config.readNoise 

2700 self._saturation = config.saturation 

2701 

2702 def getBBox(self): 

2703 return self._bbox 

2704 

2705 def getRawBBox(self): 

2706 return self._bbox 

2707 

2708 def getRawHorizontalOverscanBBox(self): 

2709 return self._RawHorizontalOverscanBBox 

2710 

2711 def getGain(self): 

2712 return self._gain 

2713 

2714 def getReadNoise(self): 

2715 return self._readNoise 

2716 

2717 def getSaturation(self): 

2718 return self._saturation 

2719 

2720 def getSuspectLevel(self): 

2721 return float("NaN") 

2722 

2723 

2724class RunIsrConfig(pexConfig.Config): 

2725 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2726 

2727 

2728class RunIsrTask(pipeBase.CmdLineTask): 

2729 """Task to wrap the default IsrTask to allow it to be retargeted. 

2730 

2731 The standard IsrTask can be called directly from a command line 

2732 program, but doing so removes the ability of the task to be 

2733 retargeted. As most cameras override some set of the IsrTask 

2734 methods, this would remove those data-specific methods in the 

2735 output post-ISR images. This wrapping class fixes the issue, 

2736 allowing identical post-ISR images to be generated by both the 

2737 processCcd and isrTask code. 

2738 """ 

2739 ConfigClass = RunIsrConfig 

2740 _DefaultName = "runIsr" 

2741 

2742 def __init__(self, *args, **kwargs): 

2743 super().__init__(*args, **kwargs) 

2744 self.makeSubtask("isr") 

2745 

2746 def runDataRef(self, dataRef): 

2747 """ 

2748 Parameters 

2749 ---------- 

2750 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2751 data reference of the detector data to be processed 

2752 

2753 Returns 

2754 ------- 

2755 result : `pipeBase.Struct` 

2756 Result struct with component: 

2757 

2758 - exposure : `lsst.afw.image.Exposure` 

2759 Post-ISR processed exposure. 

2760 """ 

2761 return self.isr.runDataRef(dataRef)