Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import math 

23import numpy 

24 

25import lsst.geom 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31 

32from contextlib import contextmanager 

33from lsstDebug import getDebugFrame 

34 

35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType, 

36 ReadoutCorner) 

37from lsst.afw.display import getDisplay 

38from lsst.afw.geom import Polygon 

39from lsst.daf.persistence import ButlerDataRef 

40from lsst.daf.persistence.butler import NoResults 

41from lsst.meas.algorithms.detection import SourceDetectionTask 

42 

43from . import isrFunctions 

44from . import isrQa 

45from . import linearize 

46from .defects import Defects 

47 

48from .assembleCcdTask import AssembleCcdTask 

49from .crosstalk import CrosstalkTask, CrosstalkCalib 

50from .fringe import FringeTask 

51from .isr import maskNans 

52from .masking import MaskingTask 

53from .overscan import OverscanCorrectionTask 

54from .straylight import StrayLightTask 

55from .vignette import VignetteTask 

56from lsst.daf.butler import DimensionGraph 

57 

58 

59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"] 

60 

61 

62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections): 

63 """Lookup function to identify crosstalkSource entries. 

64 

65 This should return an empty list under most circumstances. Only 

66 when inter-chip crosstalk has been identified should this be 

67 populated. 

68 

69 This will be unused until DM-25348 resolves the quantum graph 

70 generation issue. 

71 

72 Parameters 

73 ---------- 

74 datasetType : `str` 

75 Dataset to lookup. 

76 registry : `lsst.daf.butler.Registry` 

77 Butler registry to query. 

78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate` 

79 Data id to transform to identify crosstalkSources. The 

80 ``detector`` entry will be stripped. 

81 collections : `lsst.daf.butler.CollectionSearch` 

82 Collections to search through. 

83 

84 Returns 

85 ------- 

86 results : `list` [`lsst.daf.butler.DatasetRef`] 

87 List of datasets that match the query that will be used as 

88 crosstalkSources. 

89 """ 

90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"])) 

91 results = list(registry.queryDatasets(datasetType, 

92 collections=collections, 

93 dataId=newDataId, 

94 findFirst=True, 

95 ).expanded()) 

96 return results 

97 

98 

99class IsrTaskConnections(pipeBase.PipelineTaskConnections, 

100 dimensions={"instrument", "exposure", "detector"}, 

101 defaultTemplates={}): 

102 ccdExposure = cT.Input( 

103 name="raw", 

104 doc="Input exposure to process.", 

105 storageClass="Exposure", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 camera = cT.PrerequisiteInput( 

109 name="camera", 

110 storageClass="Camera", 

111 doc="Input camera to construct complete exposures.", 

112 dimensions=["instrument"], 

113 isCalibration=True, 

114 ) 

115 

116 crosstalk = cT.PrerequisiteInput( 

117 name="crosstalk", 

118 doc="Input crosstalk object", 

119 storageClass="CrosstalkCalib", 

120 dimensions=["instrument", "detector"], 

121 isCalibration=True, 

122 minimum=0, # can fall back to cameraGeom 

123 ) 

124 # TODO: DM-25348. This does not work yet to correctly load 

125 # possible crosstalk sources. 

126 crosstalkSources = cT.PrerequisiteInput( 

127 name="isrOverscanCorrected", 

128 doc="Overscan corrected input images.", 

129 storageClass="Exposure", 

130 dimensions=["instrument", "exposure", "detector"], 

131 deferLoad=True, 

132 multiple=True, 

133 lookupFunction=crosstalkSourceLookup, 

134 minimum=0, # not needed for all instruments, no config to control this 

135 ) 

136 bias = cT.PrerequisiteInput( 

137 name="bias", 

138 doc="Input bias calibration.", 

139 storageClass="ExposureF", 

140 dimensions=["instrument", "detector"], 

141 isCalibration=True, 

142 ) 

143 dark = cT.PrerequisiteInput( 

144 name='dark', 

145 doc="Input dark calibration.", 

146 storageClass="ExposureF", 

147 dimensions=["instrument", "detector"], 

148 isCalibration=True, 

149 ) 

150 flat = cT.PrerequisiteInput( 

151 name="flat", 

152 doc="Input flat calibration.", 

153 storageClass="ExposureF", 

154 dimensions=["instrument", "physical_filter", "detector"], 

155 isCalibration=True, 

156 ) 

157 ptc = cT.PrerequisiteInput( 

158 name="ptc", 

159 doc="Input Photon Transfer Curve dataset", 

160 storageClass="PhotonTransferCurveDataset", 

161 dimensions=["instrument", "detector"], 

162 isCalibration=True, 

163 ) 

164 fringes = cT.PrerequisiteInput( 

165 name="fringe", 

166 doc="Input fringe calibration.", 

167 storageClass="ExposureF", 

168 dimensions=["instrument", "physical_filter", "detector"], 

169 isCalibration=True, 

170 minimum=0, # only needed for some bands, even when enabled 

171 ) 

172 strayLightData = cT.PrerequisiteInput( 

173 name='yBackground', 

174 doc="Input stray light calibration.", 

175 storageClass="StrayLightData", 

176 dimensions=["instrument", "physical_filter", "detector"], 

177 isCalibration=True, 

178 minimum=0, # only needed for some bands, even when enabled 

179 ) 

180 bfKernel = cT.PrerequisiteInput( 

181 name='bfKernel', 

182 doc="Input brighter-fatter kernel.", 

183 storageClass="NumpyArray", 

184 dimensions=["instrument"], 

185 isCalibration=True, 

186 minimum=0, # can use either bfKernel or newBFKernel 

187 ) 

188 newBFKernel = cT.PrerequisiteInput( 

189 name='brighterFatterKernel', 

190 doc="Newer complete kernel + gain solutions.", 

191 storageClass="BrighterFatterKernel", 

192 dimensions=["instrument", "detector"], 

193 isCalibration=True, 

194 minimum=0, # can use either bfKernel or newBFKernel 

195 ) 

196 defects = cT.PrerequisiteInput( 

197 name='defects', 

198 doc="Input defect tables.", 

199 storageClass="Defects", 

200 dimensions=["instrument", "detector"], 

201 isCalibration=True, 

202 ) 

203 linearizer = cT.PrerequisiteInput( 

204 name='linearizer', 

205 storageClass="Linearizer", 

206 doc="Linearity correction calibration.", 

207 dimensions=["instrument", "detector"], 

208 isCalibration=True, 

209 minimum=0, # can fall back to cameraGeom 

210 ) 

211 opticsTransmission = cT.PrerequisiteInput( 

212 name="transmission_optics", 

213 storageClass="TransmissionCurve", 

214 doc="Transmission curve due to the optics.", 

215 dimensions=["instrument"], 

216 isCalibration=True, 

217 ) 

218 filterTransmission = cT.PrerequisiteInput( 

219 name="transmission_filter", 

220 storageClass="TransmissionCurve", 

221 doc="Transmission curve due to the filter.", 

222 dimensions=["instrument", "physical_filter"], 

223 isCalibration=True, 

224 ) 

225 sensorTransmission = cT.PrerequisiteInput( 

226 name="transmission_sensor", 

227 storageClass="TransmissionCurve", 

228 doc="Transmission curve due to the sensor.", 

229 dimensions=["instrument", "detector"], 

230 isCalibration=True, 

231 ) 

232 atmosphereTransmission = cT.PrerequisiteInput( 

233 name="transmission_atmosphere", 

234 storageClass="TransmissionCurve", 

235 doc="Transmission curve due to the atmosphere.", 

236 dimensions=["instrument"], 

237 isCalibration=True, 

238 ) 

239 illumMaskedImage = cT.PrerequisiteInput( 

240 name="illum", 

241 doc="Input illumination correction.", 

242 storageClass="MaskedImageF", 

243 dimensions=["instrument", "physical_filter", "detector"], 

244 isCalibration=True, 

245 ) 

246 

247 outputExposure = cT.Output( 

248 name='postISRCCD', 

249 doc="Output ISR processed exposure.", 

250 storageClass="Exposure", 

251 dimensions=["instrument", "exposure", "detector"], 

252 ) 

253 preInterpExposure = cT.Output( 

254 name='preInterpISRCCD', 

255 doc="Output ISR processed exposure, with pixels left uninterpolated.", 

256 storageClass="ExposureF", 

257 dimensions=["instrument", "exposure", "detector"], 

258 ) 

259 outputOssThumbnail = cT.Output( 

260 name="OssThumb", 

261 doc="Output Overscan-subtracted thumbnail image.", 

262 storageClass="Thumbnail", 

263 dimensions=["instrument", "exposure", "detector"], 

264 ) 

265 outputFlattenedThumbnail = cT.Output( 

266 name="FlattenedThumb", 

267 doc="Output flat-corrected thumbnail image.", 

268 storageClass="Thumbnail", 

269 dimensions=["instrument", "exposure", "detector"], 

270 ) 

271 

272 def __init__(self, *, config=None): 

273 super().__init__(config=config) 

274 

275 if config.doBias is not True: 

276 self.prerequisiteInputs.discard("bias") 

277 if config.doLinearize is not True: 

278 self.prerequisiteInputs.discard("linearizer") 

279 if config.doCrosstalk is not True: 

280 self.inputs.discard("crosstalkSources") 

281 self.prerequisiteInputs.discard("crosstalk") 

282 if config.doBrighterFatter is not True: 

283 self.prerequisiteInputs.discard("bfKernel") 

284 self.prerequisiteInputs.discard("newBFKernel") 

285 if config.doDefect is not True: 

286 self.prerequisiteInputs.discard("defects") 

287 if config.doDark is not True: 

288 self.prerequisiteInputs.discard("dark") 

289 if config.doFlat is not True: 

290 self.prerequisiteInputs.discard("flat") 

291 if config.doFringe is not True: 

292 self.prerequisiteInputs.discard("fringe") 

293 if config.doStrayLight is not True: 

294 self.prerequisiteInputs.discard("strayLightData") 

295 if config.usePtcGains is not True and config.usePtcReadNoise is not True: 

296 self.prerequisiteInputs.discard("ptc") 

297 if config.doAttachTransmissionCurve is not True: 

298 self.prerequisiteInputs.discard("opticsTransmission") 

299 self.prerequisiteInputs.discard("filterTransmission") 

300 self.prerequisiteInputs.discard("sensorTransmission") 

301 self.prerequisiteInputs.discard("atmosphereTransmission") 

302 if config.doUseOpticsTransmission is not True: 

303 self.prerequisiteInputs.discard("opticsTransmission") 

304 if config.doUseFilterTransmission is not True: 

305 self.prerequisiteInputs.discard("filterTransmission") 

306 if config.doUseSensorTransmission is not True: 

307 self.prerequisiteInputs.discard("sensorTransmission") 

308 if config.doUseAtmosphereTransmission is not True: 

309 self.prerequisiteInputs.discard("atmosphereTransmission") 

310 if config.doIlluminationCorrection is not True: 

311 self.prerequisiteInputs.discard("illumMaskedImage") 

312 

313 if config.doWrite is not True: 

314 self.outputs.discard("outputExposure") 

315 self.outputs.discard("preInterpExposure") 

316 self.outputs.discard("outputFlattenedThumbnail") 

317 self.outputs.discard("outputOssThumbnail") 

318 if config.doSaveInterpPixels is not True: 

319 self.outputs.discard("preInterpExposure") 

320 if config.qa.doThumbnailOss is not True: 

321 self.outputs.discard("outputOssThumbnail") 

322 if config.qa.doThumbnailFlattened is not True: 

323 self.outputs.discard("outputFlattenedThumbnail") 

324 

325 

326class IsrTaskConfig(pipeBase.PipelineTaskConfig, 

327 pipelineConnections=IsrTaskConnections): 

328 """Configuration parameters for IsrTask. 

329 

330 Items are grouped in the order in which they are executed by the task. 

331 """ 

332 datasetType = pexConfig.Field( 

333 dtype=str, 

334 doc="Dataset type for input data; users will typically leave this alone, " 

335 "but camera-specific ISR tasks will override it", 

336 default="raw", 

337 ) 

338 

339 fallbackFilterName = pexConfig.Field( 

340 dtype=str, 

341 doc="Fallback default filter name for calibrations.", 

342 optional=True 

343 ) 

344 useFallbackDate = pexConfig.Field( 

345 dtype=bool, 

346 doc="Pass observation date when using fallback filter.", 

347 default=False, 

348 ) 

349 expectWcs = pexConfig.Field( 

350 dtype=bool, 

351 default=True, 

352 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)." 

353 ) 

354 fwhm = pexConfig.Field( 

355 dtype=float, 

356 doc="FWHM of PSF in arcseconds.", 

357 default=1.0, 

358 ) 

359 qa = pexConfig.ConfigField( 

360 dtype=isrQa.IsrQaConfig, 

361 doc="QA related configuration options.", 

362 ) 

363 

364 # Image conversion configuration 

365 doConvertIntToFloat = pexConfig.Field( 

366 dtype=bool, 

367 doc="Convert integer raw images to floating point values?", 

368 default=True, 

369 ) 

370 

371 # Saturated pixel handling. 

372 doSaturation = pexConfig.Field( 

373 dtype=bool, 

374 doc="Mask saturated pixels? NB: this is totally independent of the" 

375 " interpolation option - this is ONLY setting the bits in the mask." 

376 " To have them interpolated make sure doSaturationInterpolation=True", 

377 default=True, 

378 ) 

379 saturatedMaskName = pexConfig.Field( 

380 dtype=str, 

381 doc="Name of mask plane to use in saturation detection and interpolation", 

382 default="SAT", 

383 ) 

384 saturation = pexConfig.Field( 

385 dtype=float, 

386 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

387 default=float("NaN"), 

388 ) 

389 growSaturationFootprintSize = pexConfig.Field( 

390 dtype=int, 

391 doc="Number of pixels by which to grow the saturation footprints", 

392 default=1, 

393 ) 

394 

395 # Suspect pixel handling. 

396 doSuspect = pexConfig.Field( 

397 dtype=bool, 

398 doc="Mask suspect pixels?", 

399 default=False, 

400 ) 

401 suspectMaskName = pexConfig.Field( 

402 dtype=str, 

403 doc="Name of mask plane to use for suspect pixels", 

404 default="SUSPECT", 

405 ) 

406 numEdgeSuspect = pexConfig.Field( 

407 dtype=int, 

408 doc="Number of edge pixels to be flagged as untrustworthy.", 

409 default=0, 

410 ) 

411 edgeMaskLevel = pexConfig.ChoiceField( 

412 dtype=str, 

413 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

414 default="DETECTOR", 

415 allowed={ 

416 'DETECTOR': 'Mask only the edges of the full detector.', 

417 'AMP': 'Mask edges of each amplifier.', 

418 }, 

419 ) 

420 

421 # Initial masking options. 

422 doSetBadRegions = pexConfig.Field( 

423 dtype=bool, 

424 doc="Should we set the level of all BAD patches of the chip to the chip's average value?", 

425 default=True, 

426 ) 

427 badStatistic = pexConfig.ChoiceField( 

428 dtype=str, 

429 doc="How to estimate the average value for BAD regions.", 

430 default='MEANCLIP', 

431 allowed={ 

432 "MEANCLIP": "Correct using the (clipped) mean of good data", 

433 "MEDIAN": "Correct using the median of the good data", 

434 }, 

435 ) 

436 

437 # Overscan subtraction configuration. 

438 doOverscan = pexConfig.Field( 

439 dtype=bool, 

440 doc="Do overscan subtraction?", 

441 default=True, 

442 ) 

443 overscan = pexConfig.ConfigurableField( 

444 target=OverscanCorrectionTask, 

445 doc="Overscan subtraction task for image segments.", 

446 ) 

447 

448 overscanFitType = pexConfig.ChoiceField( 

449 dtype=str, 

450 doc="The method for fitting the overscan bias level.", 

451 default='MEDIAN', 

452 allowed={ 

453 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

454 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

455 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

456 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

457 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

458 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

459 "MEAN": "Correct using the mean of the overscan region", 

460 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

461 "MEDIAN": "Correct using the median of the overscan region", 

462 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

463 }, 

464 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

465 " This option will no longer be used, and will be removed after v20.") 

466 ) 

467 overscanOrder = pexConfig.Field( 

468 dtype=int, 

469 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " 

470 "or number of spline knots if overscan fit type is a spline."), 

471 default=1, 

472 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

473 " This option will no longer be used, and will be removed after v20.") 

474 ) 

475 overscanNumSigmaClip = pexConfig.Field( 

476 dtype=float, 

477 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

478 default=3.0, 

479 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

480 " This option will no longer be used, and will be removed after v20.") 

481 ) 

482 overscanIsInt = pexConfig.Field( 

483 dtype=bool, 

484 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" 

485 " and overscan.FitType=MEDIAN_PER_ROW.", 

486 default=True, 

487 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." 

488 " This option will no longer be used, and will be removed after v20.") 

489 ) 

490 # These options do not get deprecated, as they define how we slice up the image data. 

491 overscanNumLeadingColumnsToSkip = pexConfig.Field( 

492 dtype=int, 

493 doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

494 default=0, 

495 ) 

496 overscanNumTrailingColumnsToSkip = pexConfig.Field( 

497 dtype=int, 

498 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

499 default=0, 

500 ) 

501 overscanMaxDev = pexConfig.Field( 501 ↛ exitline 501 didn't jump to the function exit

502 dtype=float, 

503 doc="Maximum deviation from the median for overscan", 

504 default=1000.0, check=lambda x: x > 0 

505 ) 

506 overscanBiasJump = pexConfig.Field( 

507 dtype=bool, 

508 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?", 

509 default=False, 

510 ) 

511 overscanBiasJumpKeyword = pexConfig.Field( 

512 dtype=str, 

513 doc="Header keyword containing information about devices.", 

514 default="NO_SUCH_KEY", 

515 ) 

516 overscanBiasJumpDevices = pexConfig.ListField( 

517 dtype=str, 

518 doc="List of devices that need piecewise overscan correction.", 

519 default=(), 

520 ) 

521 overscanBiasJumpLocation = pexConfig.Field( 

522 dtype=int, 

523 doc="Location of bias jump along y-axis.", 

524 default=0, 

525 ) 

526 

527 # Amplifier to CCD assembly configuration 

528 doAssembleCcd = pexConfig.Field( 

529 dtype=bool, 

530 default=True, 

531 doc="Assemble amp-level exposures into a ccd-level exposure?" 

532 ) 

533 assembleCcd = pexConfig.ConfigurableField( 

534 target=AssembleCcdTask, 

535 doc="CCD assembly task", 

536 ) 

537 

538 # General calibration configuration. 

539 doAssembleIsrExposures = pexConfig.Field( 

540 dtype=bool, 

541 default=False, 

542 doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

543 ) 

544 doTrimToMatchCalib = pexConfig.Field( 

545 dtype=bool, 

546 default=False, 

547 doc="Trim raw data to match calibration bounding boxes?" 

548 ) 

549 

550 # Bias subtraction. 

551 doBias = pexConfig.Field( 

552 dtype=bool, 

553 doc="Apply bias frame correction?", 

554 default=True, 

555 ) 

556 biasDataProductName = pexConfig.Field( 

557 dtype=str, 

558 doc="Name of the bias data product", 

559 default="bias", 

560 ) 

561 doBiasBeforeOverscan = pexConfig.Field( 

562 dtype=bool, 

563 doc="Reverse order of overscan and bias correction.", 

564 default=False 

565 ) 

566 

567 # Variance construction 

568 doVariance = pexConfig.Field( 

569 dtype=bool, 

570 doc="Calculate variance?", 

571 default=True 

572 ) 

573 gain = pexConfig.Field( 

574 dtype=float, 

575 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

576 default=float("NaN"), 

577 ) 

578 readNoise = pexConfig.Field( 

579 dtype=float, 

580 doc="The read noise to use if no Detector is present in the Exposure", 

581 default=0.0, 

582 ) 

583 doEmpiricalReadNoise = pexConfig.Field( 

584 dtype=bool, 

585 default=False, 

586 doc="Calculate empirical read noise instead of value from AmpInfo data?" 

587 ) 

588 usePtcReadNoise = pexConfig.Field( 

589 dtype=bool, 

590 default=False, 

591 doc="Use readnoise values from the Photon Transfer Curve?" 

592 ) 

593 maskNegativeVariance = pexConfig.Field( 

594 dtype=bool, 

595 default=True, 

596 doc="Mask pixels that claim a negative variance? This likely indicates a failure " 

597 "in the measurement of the overscan at an edge due to the data falling off faster " 

598 "than the overscan model can account for it." 

599 ) 

600 negativeVarianceMaskName = pexConfig.Field( 

601 dtype=str, 

602 default="BAD", 

603 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.", 

604 ) 

605 # Linearization. 

606 doLinearize = pexConfig.Field( 

607 dtype=bool, 

608 doc="Correct for nonlinearity of the detector's response?", 

609 default=True, 

610 ) 

611 

612 # Crosstalk. 

613 doCrosstalk = pexConfig.Field( 

614 dtype=bool, 

615 doc="Apply intra-CCD crosstalk correction?", 

616 default=False, 

617 ) 

618 doCrosstalkBeforeAssemble = pexConfig.Field( 

619 dtype=bool, 

620 doc="Apply crosstalk correction before CCD assembly, and before trimming?", 

621 default=False, 

622 ) 

623 crosstalk = pexConfig.ConfigurableField( 

624 target=CrosstalkTask, 

625 doc="Intra-CCD crosstalk correction", 

626 ) 

627 

628 # Masking options. 

629 doDefect = pexConfig.Field( 

630 dtype=bool, 

631 doc="Apply correction for CCD defects, e.g. hot pixels?", 

632 default=True, 

633 ) 

634 doNanMasking = pexConfig.Field( 

635 dtype=bool, 

636 doc="Mask non-finite (NAN, inf) pixels?", 

637 default=True, 

638 ) 

639 doWidenSaturationTrails = pexConfig.Field( 

640 dtype=bool, 

641 doc="Widen bleed trails based on their width?", 

642 default=True 

643 ) 

644 

645 # Brighter-Fatter correction. 

646 doBrighterFatter = pexConfig.Field( 

647 dtype=bool, 

648 default=False, 

649 doc="Apply the brighter-fatter correction?" 

650 ) 

651 brighterFatterLevel = pexConfig.ChoiceField( 

652 dtype=str, 

653 default="DETECTOR", 

654 doc="The level at which to correct for brighter-fatter.", 

655 allowed={ 

656 "AMP": "Every amplifier treated separately.", 

657 "DETECTOR": "One kernel per detector", 

658 } 

659 ) 

660 brighterFatterMaxIter = pexConfig.Field( 

661 dtype=int, 

662 default=10, 

663 doc="Maximum number of iterations for the brighter-fatter correction" 

664 ) 

665 brighterFatterThreshold = pexConfig.Field( 

666 dtype=float, 

667 default=1000, 

668 doc="Threshold used to stop iterating the brighter-fatter correction. It is the " 

669 "absolute value of the difference between the current corrected image and the one " 

670 "from the previous iteration summed over all the pixels." 

671 ) 

672 brighterFatterApplyGain = pexConfig.Field( 

673 dtype=bool, 

674 default=True, 

675 doc="Should the gain be applied when applying the brighter-fatter correction?" 

676 ) 

677 brighterFatterMaskListToInterpolate = pexConfig.ListField( 

678 dtype=str, 

679 doc="List of mask planes that should be interpolated over when applying the brighter-fatter " 

680 "correction.", 

681 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"], 

682 ) 

683 brighterFatterMaskGrowSize = pexConfig.Field( 

684 dtype=int, 

685 default=0, 

686 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate " 

687 "when brighter-fatter correction is applied." 

688 ) 

689 

690 # Dark subtraction. 

691 doDark = pexConfig.Field( 

692 dtype=bool, 

693 doc="Apply dark frame correction?", 

694 default=True, 

695 ) 

696 darkDataProductName = pexConfig.Field( 

697 dtype=str, 

698 doc="Name of the dark data product", 

699 default="dark", 

700 ) 

701 

702 # Camera-specific stray light removal. 

703 doStrayLight = pexConfig.Field( 

704 dtype=bool, 

705 doc="Subtract stray light in the y-band (due to encoder LEDs)?", 

706 default=False, 

707 ) 

708 strayLight = pexConfig.ConfigurableField( 

709 target=StrayLightTask, 

710 doc="y-band stray light correction" 

711 ) 

712 

713 # Flat correction. 

714 doFlat = pexConfig.Field( 

715 dtype=bool, 

716 doc="Apply flat field correction?", 

717 default=True, 

718 ) 

719 flatDataProductName = pexConfig.Field( 

720 dtype=str, 

721 doc="Name of the flat data product", 

722 default="flat", 

723 ) 

724 flatScalingType = pexConfig.ChoiceField( 

725 dtype=str, 

726 doc="The method for scaling the flat on the fly.", 

727 default='USER', 

728 allowed={ 

729 "USER": "Scale by flatUserScale", 

730 "MEAN": "Scale by the inverse of the mean", 

731 "MEDIAN": "Scale by the inverse of the median", 

732 }, 

733 ) 

734 flatUserScale = pexConfig.Field( 

735 dtype=float, 

736 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

737 default=1.0, 

738 ) 

739 doTweakFlat = pexConfig.Field( 

740 dtype=bool, 

741 doc="Tweak flats to match observed amplifier ratios?", 

742 default=False 

743 ) 

744 

745 # Amplifier normalization based on gains instead of using flats configuration. 

746 doApplyGains = pexConfig.Field( 

747 dtype=bool, 

748 doc="Correct the amplifiers for their gains instead of applying flat correction", 

749 default=False, 

750 ) 

751 usePtcGains = pexConfig.Field( 

752 dtype=bool, 

753 doc="Use the gain values from the Photon Transfer Curve?", 

754 default=False, 

755 ) 

756 normalizeGains = pexConfig.Field( 

757 dtype=bool, 

758 doc="Normalize all the amplifiers in each CCD to have the same median value.", 

759 default=False, 

760 ) 

761 

762 # Fringe correction. 

763 doFringe = pexConfig.Field( 

764 dtype=bool, 

765 doc="Apply fringe correction?", 

766 default=True, 

767 ) 

768 fringe = pexConfig.ConfigurableField( 

769 target=FringeTask, 

770 doc="Fringe subtraction task", 

771 ) 

772 fringeAfterFlat = pexConfig.Field( 

773 dtype=bool, 

774 doc="Do fringe subtraction after flat-fielding?", 

775 default=True, 

776 ) 

777 

778 # Initial CCD-level background statistics options. 

779 doMeasureBackground = pexConfig.Field( 

780 dtype=bool, 

781 doc="Measure the background level on the reduced image?", 

782 default=False, 

783 ) 

784 

785 # Camera-specific masking configuration. 

786 doCameraSpecificMasking = pexConfig.Field( 

787 dtype=bool, 

788 doc="Mask camera-specific bad regions?", 

789 default=False, 

790 ) 

791 masking = pexConfig.ConfigurableField( 

792 target=MaskingTask, 

793 doc="Masking task." 

794 ) 

795 

796 # Interpolation options. 

797 

798 doInterpolate = pexConfig.Field( 

799 dtype=bool, 

800 doc="Interpolate masked pixels?", 

801 default=True, 

802 ) 

803 doSaturationInterpolation = pexConfig.Field( 

804 dtype=bool, 

805 doc="Perform interpolation over pixels masked as saturated?" 

806 " NB: This is independent of doSaturation; if that is False this plane" 

807 " will likely be blank, resulting in a no-op here.", 

808 default=True, 

809 ) 

810 doNanInterpolation = pexConfig.Field( 

811 dtype=bool, 

812 doc="Perform interpolation over pixels masked as NaN?" 

813 " NB: This is independent of doNanMasking; if that is False this plane" 

814 " will likely be blank, resulting in a no-op here.", 

815 default=True, 

816 ) 

817 doNanInterpAfterFlat = pexConfig.Field( 

818 dtype=bool, 

819 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

820 "also have to interpolate them before flat-fielding."), 

821 default=False, 

822 ) 

823 maskListToInterpolate = pexConfig.ListField( 

824 dtype=str, 

825 doc="List of mask planes that should be interpolated.", 

826 default=['SAT', 'BAD'], 

827 ) 

828 doSaveInterpPixels = pexConfig.Field( 

829 dtype=bool, 

830 doc="Save a copy of the pre-interpolated pixel values?", 

831 default=False, 

832 ) 

833 

834 # Default photometric calibration options. 

835 fluxMag0T1 = pexConfig.DictField( 

836 keytype=str, 

837 itemtype=float, 

838 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.", 

839 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0), 

840 )) 

841 ) 

842 defaultFluxMag0T1 = pexConfig.Field( 

843 dtype=float, 

844 doc="Default value for fluxMag0T1 (for an unrecognized filter).", 

845 default=pow(10.0, 0.4*28.0) 

846 ) 

847 

848 # Vignette correction configuration. 

849 doVignette = pexConfig.Field( 

850 dtype=bool, 

851 doc="Apply vignetting parameters?", 

852 default=False, 

853 ) 

854 vignette = pexConfig.ConfigurableField( 

855 target=VignetteTask, 

856 doc="Vignetting task.", 

857 ) 

858 

859 # Transmission curve configuration. 

860 doAttachTransmissionCurve = pexConfig.Field( 

861 dtype=bool, 

862 default=False, 

863 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

864 ) 

865 doUseOpticsTransmission = pexConfig.Field( 

866 dtype=bool, 

867 default=True, 

868 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

869 ) 

870 doUseFilterTransmission = pexConfig.Field( 

871 dtype=bool, 

872 default=True, 

873 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

874 ) 

875 doUseSensorTransmission = pexConfig.Field( 

876 dtype=bool, 

877 default=True, 

878 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

879 ) 

880 doUseAtmosphereTransmission = pexConfig.Field( 

881 dtype=bool, 

882 default=True, 

883 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

884 ) 

885 

886 # Illumination correction. 

887 doIlluminationCorrection = pexConfig.Field( 

888 dtype=bool, 

889 default=False, 

890 doc="Perform illumination correction?" 

891 ) 

892 illuminationCorrectionDataProductName = pexConfig.Field( 

893 dtype=str, 

894 doc="Name of the illumination correction data product.", 

895 default="illumcor", 

896 ) 

897 illumScale = pexConfig.Field( 

898 dtype=float, 

899 doc="Scale factor for the illumination correction.", 

900 default=1.0, 

901 ) 

902 illumFilters = pexConfig.ListField( 

903 dtype=str, 

904 default=[], 

905 doc="Only perform illumination correction for these filters." 

906 ) 

907 

908 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed. 

909 doWrite = pexConfig.Field( 

910 dtype=bool, 

911 doc="Persist postISRCCD?", 

912 default=True, 

913 ) 

914 

915 def validate(self): 

916 super().validate() 

917 if self.doFlat and self.doApplyGains: 

918 raise ValueError("You may not specify both doFlat and doApplyGains") 

919 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib: 

920 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib") 

921 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate: 

922 self.maskListToInterpolate.append(self.saturatedMaskName) 

923 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate: 

924 self.maskListToInterpolate.remove(self.saturatedMaskName) 

925 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate: 

926 self.maskListToInterpolate.append("UNMASKEDNAN") 

927 

928 

929class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

930 """Apply common instrument signature correction algorithms to a raw frame. 

931 

932 The process for correcting imaging data is very similar from 

933 camera to camera. This task provides a vanilla implementation of 

934 doing these corrections, including the ability to turn certain 

935 corrections off if they are not needed. The inputs to the primary 

936 method, `run()`, are a raw exposure to be corrected and the 

937 calibration data products. The raw input is a single chip sized 

938 mosaic of all amps including overscans and other non-science 

939 pixels. The method `runDataRef()` identifies and defines the 

940 calibration data products, and is intended for use by a 

941 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 

942 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 

943 subclassed for different camera, although the most camera specific 

944 methods have been split into subtasks that can be redirected 

945 appropriately. 

946 

947 The __init__ method sets up the subtasks for ISR processing, using 

948 the defaults from `lsst.ip.isr`. 

949 

950 Parameters 

951 ---------- 

952 args : `list` 

953 Positional arguments passed to the Task constructor. None used at this time. 

954 kwargs : `dict`, optional 

955 Keyword arguments passed on to the Task constructor. None used at this time. 

956 """ 

957 ConfigClass = IsrTaskConfig 

958 _DefaultName = "isr" 

959 

960 def __init__(self, **kwargs): 

961 super().__init__(**kwargs) 

962 self.makeSubtask("assembleCcd") 

963 self.makeSubtask("crosstalk") 

964 self.makeSubtask("strayLight") 

965 self.makeSubtask("fringe") 

966 self.makeSubtask("masking") 

967 self.makeSubtask("overscan") 

968 self.makeSubtask("vignette") 

969 

970 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

971 inputs = butlerQC.get(inputRefs) 

972 

973 try: 

974 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector'] 

975 except Exception as e: 

976 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." % 

977 (inputRefs, e)) 

978 

979 inputs['isGen3'] = True 

980 

981 detector = inputs['ccdExposure'].getDetector() 

982 

983 if self.config.doCrosstalk is True: 

984 # Crosstalk sources need to be defined by the pipeline 

985 # yaml if they exist. 

986 if 'crosstalk' in inputs and inputs['crosstalk'] is not None: 

987 if not isinstance(inputs['crosstalk'], CrosstalkCalib): 

988 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk']) 

989 else: 

990 coeffVector = (self.config.crosstalk.crosstalkValues 

991 if self.config.crosstalk.useConfigCoefficients else None) 

992 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector) 

993 inputs['crosstalk'] = crosstalkCalib 

994 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0: 

995 if 'crosstalkSources' not in inputs: 

996 self.log.warning("No crosstalkSources found for chip with interChip terms!") 

997 

998 if self.doLinearize(detector) is True: 

999 if 'linearizer' in inputs: 

1000 if isinstance(inputs['linearizer'], dict): 

1001 linearizer = linearize.Linearizer(detector=detector, log=self.log) 

1002 linearizer.fromYaml(inputs['linearizer']) 

1003 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.") 

1004 elif isinstance(inputs['linearizer'], numpy.ndarray): 

1005 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), 

1006 detector=detector, 

1007 log=self.log) 

1008 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.") 

1009 else: 

1010 linearizer = inputs['linearizer'] 

1011 linearizer.log = self.log 

1012 inputs['linearizer'] = linearizer 

1013 else: 

1014 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log) 

1015 self.log.warning("Constructing linearizer from cameraGeom information.") 

1016 

1017 if self.config.doDefect is True: 

1018 if "defects" in inputs and inputs['defects'] is not None: 

1019 # defects is loaded as a BaseCatalog with columns x0, y0, width, height. 

1020 # masking expects a list of defects defined by their bounding box 

1021 if not isinstance(inputs["defects"], Defects): 

1022 inputs["defects"] = Defects.fromTable(inputs["defects"]) 

1023 

1024 # Load the correct style of brighter-fatter kernel, and repack 

1025 # the information as a numpy array. 

1026 if self.config.doBrighterFatter: 

1027 brighterFatterKernel = inputs.pop('newBFKernel', None) 

1028 if brighterFatterKernel is None: 

1029 brighterFatterKernel = inputs.get('bfKernel', None) 

1030 

1031 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1032 # This is a ISR calib kernel 

1033 detName = detector.getName() 

1034 level = brighterFatterKernel.level 

1035 

1036 # This is expected to be a dictionary of amp-wise gains. 

1037 inputs['bfGains'] = brighterFatterKernel.gain 

1038 if self.config.brighterFatterLevel == 'DETECTOR': 

1039 if level == 'DETECTOR': 

1040 if detName in brighterFatterKernel.detKernels: 

1041 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1042 else: 

1043 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1044 elif level == 'AMP': 

1045 self.log.warning("Making DETECTOR level kernel from AMP based brighter " 

1046 "fatter kernels.") 

1047 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName) 

1048 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName] 

1049 elif self.config.brighterFatterLevel == 'AMP': 

1050 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1051 

1052 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']): 

1053 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId() 

1054 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'], 

1055 expId=expId, 

1056 assembler=self.assembleCcd 

1057 if self.config.doAssembleIsrExposures else None) 

1058 else: 

1059 inputs['fringes'] = pipeBase.Struct(fringes=None) 

1060 

1061 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']): 

1062 if 'strayLightData' not in inputs: 

1063 inputs['strayLightData'] = None 

1064 

1065 outputs = self.run(**inputs) 

1066 butlerQC.put(outputs, outputRefs) 

1067 

1068 def readIsrData(self, dataRef, rawExposure): 

1069 """Retrieve necessary frames for instrument signature removal. 

1070 

1071 Pre-fetching all required ISR data products limits the IO 

1072 required by the ISR. Any conflict between the calibration data 

1073 available and that needed for ISR is also detected prior to 

1074 doing processing, allowing it to fail quickly. 

1075 

1076 Parameters 

1077 ---------- 

1078 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1079 Butler reference of the detector data to be processed 

1080 rawExposure : `afw.image.Exposure` 

1081 The raw exposure that will later be corrected with the 

1082 retrieved calibration data; should not be modified in this 

1083 method. 

1084 

1085 Returns 

1086 ------- 

1087 result : `lsst.pipe.base.Struct` 

1088 Result struct with components (which may be `None`): 

1089 - ``bias``: bias calibration frame (`afw.image.Exposure`) 

1090 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 

1091 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 

1092 - ``dark``: dark calibration frame (`afw.image.Exposure`) 

1093 - ``flat``: flat calibration frame (`afw.image.Exposure`) 

1094 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 

1095 - ``defects``: list of defects (`lsst.ip.isr.Defects`) 

1096 - ``fringes``: `lsst.pipe.base.Struct` with components: 

1097 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1098 - ``seed``: random seed derived from the ccdExposureId for random 

1099 number generator (`uint32`). 

1100 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 

1101 A ``TransmissionCurve`` that represents the throughput of the optics, 

1102 to be evaluated in focal-plane coordinates. 

1103 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 

1104 A ``TransmissionCurve`` that represents the throughput of the filter 

1105 itself, to be evaluated in focal-plane coordinates. 

1106 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 

1107 A ``TransmissionCurve`` that represents the throughput of the sensor 

1108 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1109 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 

1110 A ``TransmissionCurve`` that represents the throughput of the 

1111 atmosphere, assumed to be spatially constant. 

1112 - ``strayLightData`` : `object` 

1113 An opaque object containing calibration information for 

1114 stray-light correction. If `None`, no correction will be 

1115 performed. 

1116 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 

1117 

1118 Raises 

1119 ------ 

1120 NotImplementedError : 

1121 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 

1122 """ 

1123 try: 

1124 dateObs = rawExposure.getInfo().getVisitInfo().getDate() 

1125 dateObs = dateObs.toPython().isoformat() 

1126 except RuntimeError: 

1127 self.log.warning("Unable to identify dateObs for rawExposure.") 

1128 dateObs = None 

1129 

1130 ccd = rawExposure.getDetector() 

1131 filterLabel = rawExposure.getFilterLabel() 

1132 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1133 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing. 

1134 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName) 

1135 if self.config.doBias else None) 

1136 # immediate=True required for functors and linearizers are functors; see ticket DM-6515 

1137 linearizer = (dataRef.get("linearizer", immediate=True) 

1138 if self.doLinearize(ccd) else None) 

1139 if linearizer is not None and not isinstance(linearizer, numpy.ndarray): 

1140 linearizer.log = self.log 

1141 if isinstance(linearizer, numpy.ndarray): 

1142 linearizer = linearize.Linearizer(table=linearizer, detector=ccd) 

1143 

1144 crosstalkCalib = None 

1145 if self.config.doCrosstalk: 

1146 try: 

1147 crosstalkCalib = dataRef.get("crosstalk", immediate=True) 

1148 except NoResults: 

1149 coeffVector = (self.config.crosstalk.crosstalkValues 

1150 if self.config.crosstalk.useConfigCoefficients else None) 

1151 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector) 

1152 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib) 

1153 if self.config.doCrosstalk else None) 

1154 

1155 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName) 

1156 if self.config.doDark else None) 

1157 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName, 

1158 dateObs=dateObs) 

1159 if self.config.doFlat else None) 

1160 

1161 brighterFatterKernel = None 

1162 brighterFatterGains = None 

1163 if self.config.doBrighterFatter is True: 

1164 try: 

1165 # Use the new-style cp_pipe version of the kernel if it exists 

1166 # If using a new-style kernel, always use the self-consistent 

1167 # gains, i.e. the ones inside the kernel object itself 

1168 brighterFatterKernel = dataRef.get("brighterFatterKernel") 

1169 brighterFatterGains = brighterFatterKernel.gain 

1170 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded") 

1171 except NoResults: 

1172 try: # Fall back to the old-style numpy-ndarray style kernel if necessary. 

1173 brighterFatterKernel = dataRef.get("bfKernel") 

1174 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded") 

1175 except NoResults: 

1176 brighterFatterKernel = None 

1177 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray): 

1178 # If the kernel is not an ndarray, it's the cp_pipe version 

1179 # so extract the kernel for this detector, or raise an error 

1180 if self.config.brighterFatterLevel == 'DETECTOR': 

1181 if brighterFatterKernel.detKernels: 

1182 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()] 

1183 else: 

1184 raise RuntimeError("Failed to extract kernel from new-style BF kernel.") 

1185 else: 

1186 # TODO DM-15631 for implementing this 

1187 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented") 

1188 

1189 defectList = (dataRef.get("defects") 

1190 if self.config.doDefect else None) 

1191 expId = rawExposure.getInfo().getVisitInfo().getExposureId() 

1192 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd 

1193 if self.config.doAssembleIsrExposures else None) 

1194 if self.config.doFringe and self.fringe.checkFilter(rawExposure) 

1195 else pipeBase.Struct(fringes=None)) 

1196 

1197 if self.config.doAttachTransmissionCurve: 

1198 opticsTransmission = (dataRef.get("transmission_optics") 

1199 if self.config.doUseOpticsTransmission else None) 

1200 filterTransmission = (dataRef.get("transmission_filter") 

1201 if self.config.doUseFilterTransmission else None) 

1202 sensorTransmission = (dataRef.get("transmission_sensor") 

1203 if self.config.doUseSensorTransmission else None) 

1204 atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

1205 if self.config.doUseAtmosphereTransmission else None) 

1206 else: 

1207 opticsTransmission = None 

1208 filterTransmission = None 

1209 sensorTransmission = None 

1210 atmosphereTransmission = None 

1211 

1212 if self.config.doStrayLight: 

1213 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure) 

1214 else: 

1215 strayLightData = None 

1216 

1217 illumMaskedImage = (self.getIsrExposure(dataRef, 

1218 self.config.illuminationCorrectionDataProductName).getMaskedImage() 

1219 if (self.config.doIlluminationCorrection 

1220 and physicalFilter in self.config.illumFilters) 

1221 else None) 

1222 

1223 # Struct should include only kwargs to run() 

1224 return pipeBase.Struct(bias=biasExposure, 

1225 linearizer=linearizer, 

1226 crosstalk=crosstalkCalib, 

1227 crosstalkSources=crosstalkSources, 

1228 dark=darkExposure, 

1229 flat=flatExposure, 

1230 bfKernel=brighterFatterKernel, 

1231 bfGains=brighterFatterGains, 

1232 defects=defectList, 

1233 fringes=fringeStruct, 

1234 opticsTransmission=opticsTransmission, 

1235 filterTransmission=filterTransmission, 

1236 sensorTransmission=sensorTransmission, 

1237 atmosphereTransmission=atmosphereTransmission, 

1238 strayLightData=strayLightData, 

1239 illumMaskedImage=illumMaskedImage 

1240 ) 

1241 

1242 @pipeBase.timeMethod 

1243 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

1244 crosstalk=None, crosstalkSources=None, 

1245 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

1246 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

1247 sensorTransmission=None, atmosphereTransmission=None, 

1248 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

1249 isGen3=False, 

1250 ): 

1251 """Perform instrument signature removal on an exposure. 

1252 

1253 Steps included in the ISR processing, in order performed, are: 

1254 - saturation and suspect pixel masking 

1255 - overscan subtraction 

1256 - CCD assembly of individual amplifiers 

1257 - bias subtraction 

1258 - variance image construction 

1259 - linearization of non-linear response 

1260 - crosstalk masking 

1261 - brighter-fatter correction 

1262 - dark subtraction 

1263 - fringe correction 

1264 - stray light subtraction 

1265 - flat correction 

1266 - masking of known defects and camera specific features 

1267 - vignette calculation 

1268 - appending transmission curve and distortion model 

1269 

1270 Parameters 

1271 ---------- 

1272 ccdExposure : `lsst.afw.image.Exposure` 

1273 The raw exposure that is to be run through ISR. The 

1274 exposure is modified by this method. 

1275 camera : `lsst.afw.cameraGeom.Camera`, optional 

1276 The camera geometry for this exposure. Required if ``isGen3`` is 

1277 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or 

1278 ``flat`` does not have an associated detector. 

1279 bias : `lsst.afw.image.Exposure`, optional 

1280 Bias calibration frame. 

1281 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

1282 Functor for linearization. 

1283 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

1284 Calibration for crosstalk. 

1285 crosstalkSources : `list`, optional 

1286 List of possible crosstalk sources. 

1287 dark : `lsst.afw.image.Exposure`, optional 

1288 Dark calibration frame. 

1289 flat : `lsst.afw.image.Exposure`, optional 

1290 Flat calibration frame. 

1291 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

1292 Photon transfer curve dataset, with, e.g., gains 

1293 and read noise. 

1294 bfKernel : `numpy.ndarray`, optional 

1295 Brighter-fatter kernel. 

1296 bfGains : `dict` of `float`, optional 

1297 Gains used to override the detector's nominal gains for the 

1298 brighter-fatter correction. A dict keyed by amplifier name for 

1299 the detector in question. 

1300 defects : `lsst.ip.isr.Defects`, optional 

1301 List of defects. 

1302 fringes : `lsst.pipe.base.Struct`, optional 

1303 Struct containing the fringe correction data, with 

1304 elements: 

1305 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

1306 - ``seed``: random seed derived from the ccdExposureId for random 

1307 number generator (`uint32`) 

1308 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

1309 A ``TransmissionCurve`` that represents the throughput of the optics, 

1310 to be evaluated in focal-plane coordinates. 

1311 filterTransmission : `lsst.afw.image.TransmissionCurve` 

1312 A ``TransmissionCurve`` that represents the throughput of the filter 

1313 itself, to be evaluated in focal-plane coordinates. 

1314 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

1315 A ``TransmissionCurve`` that represents the throughput of the sensor 

1316 itself, to be evaluated in post-assembly trimmed detector coordinates. 

1317 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

1318 A ``TransmissionCurve`` that represents the throughput of the 

1319 atmosphere, assumed to be spatially constant. 

1320 detectorNum : `int`, optional 

1321 The integer number for the detector to process. 

1322 isGen3 : bool, optional 

1323 Flag this call to run() as using the Gen3 butler environment. 

1324 strayLightData : `object`, optional 

1325 Opaque object containing calibration information for stray-light 

1326 correction. If `None`, no correction will be performed. 

1327 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

1328 Illumination correction image. 

1329 

1330 Returns 

1331 ------- 

1332 result : `lsst.pipe.base.Struct` 

1333 Result struct with component: 

1334 - ``exposure`` : `afw.image.Exposure` 

1335 The fully ISR corrected exposure. 

1336 - ``outputExposure`` : `afw.image.Exposure` 

1337 An alias for `exposure` 

1338 - ``ossThumb`` : `numpy.ndarray` 

1339 Thumbnail image of the exposure after overscan subtraction. 

1340 - ``flattenedThumb`` : `numpy.ndarray` 

1341 Thumbnail image of the exposure after flat-field correction. 

1342 

1343 Raises 

1344 ------ 

1345 RuntimeError 

1346 Raised if a configuration option is set to True, but the 

1347 required calibration data has not been specified. 

1348 

1349 Notes 

1350 ----- 

1351 The current processed exposure can be viewed by setting the 

1352 appropriate lsstDebug entries in the `debug.display` 

1353 dictionary. The names of these entries correspond to some of 

1354 the IsrTaskConfig Boolean options, with the value denoting the 

1355 frame to use. The exposure is shown inside the matching 

1356 option check and after the processing of that step has 

1357 finished. The steps with debug points are: 

1358 

1359 doAssembleCcd 

1360 doBias 

1361 doCrosstalk 

1362 doBrighterFatter 

1363 doDark 

1364 doFringe 

1365 doStrayLight 

1366 doFlat 

1367 

1368 In addition, setting the "postISRCCD" entry displays the 

1369 exposure after all ISR processing has finished. 

1370 

1371 """ 

1372 

1373 if isGen3 is True: 

1374 # Gen3 currently cannot automatically do configuration overrides. 

1375 # DM-15257 looks to discuss this issue. 

1376 # Configure input exposures; 

1377 if detectorNum is None: 

1378 raise RuntimeError("Must supply the detectorNum if running as Gen3.") 

1379 

1380 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum) 

1381 bias = self.ensureExposure(bias, camera, detectorNum) 

1382 dark = self.ensureExposure(dark, camera, detectorNum) 

1383 flat = self.ensureExposure(flat, camera, detectorNum) 

1384 else: 

1385 if isinstance(ccdExposure, ButlerDataRef): 

1386 return self.runDataRef(ccdExposure) 

1387 

1388 ccd = ccdExposure.getDetector() 

1389 filterLabel = ccdExposure.getFilterLabel() 

1390 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

1391 

1392 if not ccd: 

1393 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd." 

1394 ccd = [FakeAmp(ccdExposure, self.config)] 

1395 

1396 # Validate Input 

1397 if self.config.doBias and bias is None: 

1398 raise RuntimeError("Must supply a bias exposure if config.doBias=True.") 

1399 if self.doLinearize(ccd) and linearizer is None: 

1400 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.") 

1401 if self.config.doBrighterFatter and bfKernel is None: 

1402 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.") 

1403 if self.config.doDark and dark is None: 

1404 raise RuntimeError("Must supply a dark exposure if config.doDark=True.") 

1405 if self.config.doFlat and flat is None: 

1406 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.") 

1407 if self.config.doDefect and defects is None: 

1408 raise RuntimeError("Must supply defects if config.doDefect=True.") 

1409 if (self.config.doFringe and physicalFilter in self.fringe.config.filters 

1410 and fringes.fringes is None): 

1411 # The `fringes` object needs to be a pipeBase.Struct, as 

1412 # we use it as a `dict` for the parameters of 

1413 # `FringeTask.run()`. The `fringes.fringes` `list` may 

1414 # not be `None` if `doFringe=True`. Otherwise, raise. 

1415 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.") 

1416 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters 

1417 and illumMaskedImage is None): 

1418 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.") 

1419 

1420 # Begin ISR processing. 

1421 if self.config.doConvertIntToFloat: 

1422 self.log.info("Converting exposure to floating point values.") 

1423 ccdExposure = self.convertIntToFloat(ccdExposure) 

1424 

1425 if self.config.doBias and self.config.doBiasBeforeOverscan: 

1426 self.log.info("Applying bias correction.") 

1427 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1428 trimToFit=self.config.doTrimToMatchCalib) 

1429 self.debugView(ccdExposure, "doBias") 

1430 

1431 # Amplifier level processing. 

1432 overscans = [] 

1433 for amp in ccd: 

1434 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

1435 if ccdExposure.getBBox().contains(amp.getBBox()): 

1436 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values. 

1437 badAmp = self.maskAmplifier(ccdExposure, amp, defects) 

1438 

1439 if self.config.doOverscan and not badAmp: 

1440 # Overscan correction on amp-by-amp basis. 

1441 overscanResults = self.overscanCorrection(ccdExposure, amp) 

1442 self.log.debug("Corrected overscan for amplifier %s.", amp.getName()) 

1443 if overscanResults is not None and \ 

1444 self.config.qa is not None and self.config.qa.saveStats is True: 

1445 if isinstance(overscanResults.overscanFit, float): 

1446 qaMedian = overscanResults.overscanFit 

1447 qaStdev = float("NaN") 

1448 else: 

1449 qaStats = afwMath.makeStatistics(overscanResults.overscanFit, 

1450 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1451 qaMedian = qaStats.getValue(afwMath.MEDIAN) 

1452 qaStdev = qaStats.getValue(afwMath.STDEVCLIP) 

1453 

1454 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian) 

1455 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev) 

1456 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f", 

1457 amp.getName(), qaMedian, qaStdev) 

1458 

1459 # Residuals after overscan correction 

1460 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage, 

1461 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1462 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN) 

1463 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP) 

1464 

1465 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter) 

1466 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter) 

1467 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f", 

1468 amp.getName(), qaMedianAfter, qaStdevAfter) 

1469 

1470 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected") 

1471 else: 

1472 if badAmp: 

1473 self.log.warning("Amplifier %s is bad.", amp.getName()) 

1474 overscanResults = None 

1475 

1476 overscans.append(overscanResults if overscanResults is not None else None) 

1477 else: 

1478 self.log.info("Skipped OSCAN for %s.", amp.getName()) 

1479 

1480 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble: 

1481 self.log.info("Applying crosstalk correction.") 

1482 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1483 crosstalkSources=crosstalkSources, camera=camera) 

1484 self.debugView(ccdExposure, "doCrosstalk") 

1485 

1486 if self.config.doAssembleCcd: 

1487 self.log.info("Assembling CCD from amplifiers.") 

1488 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

1489 

1490 if self.config.expectWcs and not ccdExposure.getWcs(): 

1491 self.log.warning("No WCS found in input exposure.") 

1492 self.debugView(ccdExposure, "doAssembleCcd") 

1493 

1494 ossThumb = None 

1495 if self.config.qa.doThumbnailOss: 

1496 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1497 

1498 if self.config.doBias and not self.config.doBiasBeforeOverscan: 

1499 self.log.info("Applying bias correction.") 

1500 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(), 

1501 trimToFit=self.config.doTrimToMatchCalib) 

1502 self.debugView(ccdExposure, "doBias") 

1503 

1504 if self.config.doVariance: 

1505 for amp, overscanResults in zip(ccd, overscans): 

1506 if ccdExposure.getBBox().contains(amp.getBBox()): 

1507 self.log.debug("Constructing variance map for amplifer %s.", amp.getName()) 

1508 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1509 if overscanResults is not None: 

1510 self.updateVariance(ampExposure, amp, 

1511 overscanImage=overscanResults.overscanImage, 

1512 ptcDataset=ptc) 

1513 else: 

1514 self.updateVariance(ampExposure, amp, 

1515 overscanImage=None, 

1516 ptcDataset=ptc) 

1517 if self.config.qa is not None and self.config.qa.saveStats is True: 

1518 qaStats = afwMath.makeStatistics(ampExposure.getVariance(), 

1519 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1520 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN", 

1521 qaStats.getValue(afwMath.MEDIAN)) 

1522 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV", 

1523 qaStats.getValue(afwMath.STDEVCLIP)) 

1524 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.", 

1525 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1526 qaStats.getValue(afwMath.STDEVCLIP)) 

1527 if self.config.maskNegativeVariance: 

1528 self.maskNegativeVariance(ccdExposure) 

1529 

1530 if self.doLinearize(ccd): 

1531 self.log.info("Applying linearizer.") 

1532 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(), 

1533 detector=ccd, log=self.log) 

1534 

1535 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble: 

1536 self.log.info("Applying crosstalk correction.") 

1537 self.crosstalk.run(ccdExposure, crosstalk=crosstalk, 

1538 crosstalkSources=crosstalkSources, isTrimmed=True) 

1539 self.debugView(ccdExposure, "doCrosstalk") 

1540 

1541 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do 

1542 # anything else the camera needs. Saturated and suspect pixels have already been masked. 

1543 if self.config.doDefect: 

1544 self.log.info("Masking defects.") 

1545 self.maskDefect(ccdExposure, defects) 

1546 

1547 if self.config.numEdgeSuspect > 0: 

1548 self.log.info("Masking edges as SUSPECT.") 

1549 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect, 

1550 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

1551 

1552 if self.config.doNanMasking: 

1553 self.log.info("Masking non-finite (NAN, inf) value pixels.") 

1554 self.maskNan(ccdExposure) 

1555 

1556 if self.config.doWidenSaturationTrails: 

1557 self.log.info("Widening saturation trails.") 

1558 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) 

1559 

1560 if self.config.doCameraSpecificMasking: 

1561 self.log.info("Masking regions for camera specific reasons.") 

1562 self.masking.run(ccdExposure) 

1563 

1564 if self.config.doBrighterFatter: 

1565 # We need to apply flats and darks before we can interpolate, and we 

1566 # need to interpolate before we do B-F, but we do B-F without the 

1567 # flats and darks applied so we can work in units of electrons or holes. 

1568 # This context manager applies and then removes the darks and flats. 

1569 # 

1570 # We also do not want to interpolate values here, so operate on temporary 

1571 # images so we can apply only the BF-correction and roll back the 

1572 # interpolation. 

1573 interpExp = ccdExposure.clone() 

1574 with self.flatContext(interpExp, flat, dark): 

1575 isrFunctions.interpolateFromMask( 

1576 maskedImage=interpExp.getMaskedImage(), 

1577 fwhm=self.config.fwhm, 

1578 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1579 maskNameList=list(self.config.brighterFatterMaskListToInterpolate) 

1580 ) 

1581 bfExp = interpExp.clone() 

1582 

1583 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.", 

1584 type(bfKernel), type(bfGains)) 

1585 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel, 

1586 self.config.brighterFatterMaxIter, 

1587 self.config.brighterFatterThreshold, 

1588 self.config.brighterFatterApplyGain, 

1589 bfGains) 

1590 if bfResults[1] == self.config.brighterFatterMaxIter: 

1591 self.log.warning("Brighter-fatter correction did not converge, final difference %f.", 

1592 bfResults[0]) 

1593 else: 

1594 self.log.info("Finished brighter-fatter correction in %d iterations.", 

1595 bfResults[1]) 

1596 image = ccdExposure.getMaskedImage().getImage() 

1597 bfCorr = bfExp.getMaskedImage().getImage() 

1598 bfCorr -= interpExp.getMaskedImage().getImage() 

1599 image += bfCorr 

1600 

1601 # Applying the brighter-fatter correction applies a 

1602 # convolution to the science image. At the edges this 

1603 # convolution may not have sufficient valid pixels to 

1604 # produce a valid correction. Mark pixels within the size 

1605 # of the brighter-fatter kernel as EDGE to warn of this 

1606 # fact. 

1607 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.") 

1608 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2, 

1609 maskPlane="EDGE") 

1610 

1611 if self.config.brighterFatterMaskGrowSize > 0: 

1612 self.log.info("Growing masks to account for brighter-fatter kernel convolution.") 

1613 for maskPlane in self.config.brighterFatterMaskListToInterpolate: 

1614 isrFunctions.growMasks(ccdExposure.getMask(), 

1615 radius=self.config.brighterFatterMaskGrowSize, 

1616 maskNameList=maskPlane, 

1617 maskValue=maskPlane) 

1618 

1619 self.debugView(ccdExposure, "doBrighterFatter") 

1620 

1621 if self.config.doDark: 

1622 self.log.info("Applying dark correction.") 

1623 self.darkCorrection(ccdExposure, dark) 

1624 self.debugView(ccdExposure, "doDark") 

1625 

1626 if self.config.doFringe and not self.config.fringeAfterFlat: 

1627 self.log.info("Applying fringe correction before flat.") 

1628 self.fringe.run(ccdExposure, **fringes.getDict()) 

1629 self.debugView(ccdExposure, "doFringe") 

1630 

1631 if self.config.doStrayLight and self.strayLight.check(ccdExposure): 

1632 self.log.info("Checking strayLight correction.") 

1633 self.strayLight.run(ccdExposure, strayLightData) 

1634 self.debugView(ccdExposure, "doStrayLight") 

1635 

1636 if self.config.doFlat: 

1637 self.log.info("Applying flat correction.") 

1638 self.flatCorrection(ccdExposure, flat) 

1639 self.debugView(ccdExposure, "doFlat") 

1640 

1641 if self.config.doApplyGains: 

1642 self.log.info("Applying gain correction instead of flat.") 

1643 if self.config.usePtcGains: 

1644 self.log.info("Using gains from the Photon Transfer Curve.") 

1645 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains, 

1646 ptcGains=ptc.gain) 

1647 else: 

1648 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains) 

1649 

1650 if self.config.doFringe and self.config.fringeAfterFlat: 

1651 self.log.info("Applying fringe correction after flat.") 

1652 self.fringe.run(ccdExposure, **fringes.getDict()) 

1653 

1654 if self.config.doVignette: 

1655 self.log.info("Constructing Vignette polygon.") 

1656 self.vignettePolygon = self.vignette.run(ccdExposure) 

1657 

1658 if self.config.vignette.doWriteVignettePolygon: 

1659 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) 

1660 

1661 if self.config.doAttachTransmissionCurve: 

1662 self.log.info("Adding transmission curves.") 

1663 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

1664 filterTransmission=filterTransmission, 

1665 sensorTransmission=sensorTransmission, 

1666 atmosphereTransmission=atmosphereTransmission) 

1667 

1668 flattenedThumb = None 

1669 if self.config.qa.doThumbnailFlattened: 

1670 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa) 

1671 

1672 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters: 

1673 self.log.info("Performing illumination correction.") 

1674 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(), 

1675 illumMaskedImage, illumScale=self.config.illumScale, 

1676 trimToFit=self.config.doTrimToMatchCalib) 

1677 

1678 preInterpExp = None 

1679 if self.config.doSaveInterpPixels: 

1680 preInterpExp = ccdExposure.clone() 

1681 

1682 # Reset and interpolate bad pixels. 

1683 # 

1684 # Large contiguous bad regions (which should have the BAD mask 

1685 # bit set) should have their values set to the image median. 

1686 # This group should include defects and bad amplifiers. As the 

1687 # area covered by these defects are large, there's little 

1688 # reason to expect that interpolation would provide a more 

1689 # useful value. 

1690 # 

1691 # Smaller defects can be safely interpolated after the larger 

1692 # regions have had their pixel values reset. This ensures 

1693 # that the remaining defects adjacent to bad amplifiers (as an 

1694 # example) do not attempt to interpolate extreme values. 

1695 if self.config.doSetBadRegions: 

1696 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure) 

1697 if badPixelCount > 0: 

1698 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue) 

1699 

1700 if self.config.doInterpolate: 

1701 self.log.info("Interpolating masked pixels.") 

1702 isrFunctions.interpolateFromMask( 

1703 maskedImage=ccdExposure.getMaskedImage(), 

1704 fwhm=self.config.fwhm, 

1705 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

1706 maskNameList=list(self.config.maskListToInterpolate) 

1707 ) 

1708 

1709 self.roughZeroPoint(ccdExposure) 

1710 

1711 if self.config.doMeasureBackground: 

1712 self.log.info("Measuring background level.") 

1713 self.measureBackground(ccdExposure, self.config.qa) 

1714 

1715 if self.config.qa is not None and self.config.qa.saveStats is True: 

1716 for amp in ccd: 

1717 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

1718 qaStats = afwMath.makeStatistics(ampExposure.getImage(), 

1719 afwMath.MEDIAN | afwMath.STDEVCLIP) 

1720 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()), 

1721 qaStats.getValue(afwMath.MEDIAN)) 

1722 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()), 

1723 qaStats.getValue(afwMath.STDEVCLIP)) 

1724 self.log.debug(" Background stats for amplifer %s: %f +/- %f", 

1725 amp.getName(), qaStats.getValue(afwMath.MEDIAN), 

1726 qaStats.getValue(afwMath.STDEVCLIP)) 

1727 

1728 self.debugView(ccdExposure, "postISRCCD") 

1729 

1730 return pipeBase.Struct( 

1731 exposure=ccdExposure, 

1732 ossThumb=ossThumb, 

1733 flattenedThumb=flattenedThumb, 

1734 

1735 preInterpExposure=preInterpExp, 

1736 outputExposure=ccdExposure, 

1737 outputOssThumbnail=ossThumb, 

1738 outputFlattenedThumbnail=flattenedThumb, 

1739 ) 

1740 

1741 @pipeBase.timeMethod 

1742 def runDataRef(self, sensorRef): 

1743 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 

1744 

1745 This method contains the `CmdLineTask` interface to the ISR 

1746 processing. All IO is handled here, freeing the `run()` method 

1747 to manage only pixel-level calculations. The steps performed 

1748 are: 

1749 - Read in necessary detrending/isr/calibration data. 

1750 - Process raw exposure in `run()`. 

1751 - Persist the ISR-corrected exposure as "postISRCCD" if 

1752 config.doWrite=True. 

1753 

1754 Parameters 

1755 ---------- 

1756 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1757 DataRef of the detector data to be processed 

1758 

1759 Returns 

1760 ------- 

1761 result : `lsst.pipe.base.Struct` 

1762 Result struct with component: 

1763 - ``exposure`` : `afw.image.Exposure` 

1764 The fully ISR corrected exposure. 

1765 

1766 Raises 

1767 ------ 

1768 RuntimeError 

1769 Raised if a configuration option is set to True, but the 

1770 required calibration data does not exist. 

1771 

1772 """ 

1773 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId) 

1774 

1775 ccdExposure = sensorRef.get(self.config.datasetType) 

1776 

1777 camera = sensorRef.get("camera") 

1778 isrData = self.readIsrData(sensorRef, ccdExposure) 

1779 

1780 result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

1781 

1782 if self.config.doWrite: 

1783 sensorRef.put(result.exposure, "postISRCCD") 

1784 if result.preInterpExposure is not None: 

1785 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated") 

1786 if result.ossThumb is not None: 

1787 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb") 

1788 if result.flattenedThumb is not None: 

1789 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb") 

1790 

1791 return result 

1792 

1793 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True): 

1794 """Retrieve a calibration dataset for removing instrument signature. 

1795 

1796 Parameters 

1797 ---------- 

1798 

1799 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

1800 DataRef of the detector data to find calibration datasets 

1801 for. 

1802 datasetType : `str` 

1803 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 

1804 dateObs : `str`, optional 

1805 Date of the observation. Used to correct butler failures 

1806 when using fallback filters. 

1807 immediate : `Bool` 

1808 If True, disable butler proxies to enable error handling 

1809 within this routine. 

1810 

1811 Returns 

1812 ------- 

1813 exposure : `lsst.afw.image.Exposure` 

1814 Requested calibration frame. 

1815 

1816 Raises 

1817 ------ 

1818 RuntimeError 

1819 Raised if no matching calibration frame can be found. 

1820 """ 

1821 try: 

1822 exp = dataRef.get(datasetType, immediate=immediate) 

1823 except Exception as exc1: 

1824 if not self.config.fallbackFilterName: 

1825 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1)) 

1826 try: 

1827 if self.config.useFallbackDate and dateObs: 

1828 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, 

1829 dateObs=dateObs, immediate=immediate) 

1830 else: 

1831 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

1832 except Exception as exc2: 

1833 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." % 

1834 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

1835 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName) 

1836 

1837 if self.config.doAssembleIsrExposures: 

1838 exp = self.assembleCcd.assembleCcd(exp) 

1839 return exp 

1840 

1841 def ensureExposure(self, inputExp, camera, detectorNum): 

1842 """Ensure that the data returned by Butler is a fully constructed exposure. 

1843 

1844 ISR requires exposure-level image data for historical reasons, so if we did 

1845 not recieve that from Butler, construct it from what we have, modifying the 

1846 input in place. 

1847 

1848 Parameters 

1849 ---------- 

1850 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 

1851 `lsst.afw.image.ImageF` 

1852 The input data structure obtained from Butler. 

1853 camera : `lsst.afw.cameraGeom.camera` 

1854 The camera associated with the image. Used to find the appropriate 

1855 detector. 

1856 detectorNum : `int` 

1857 The detector this exposure should match. 

1858 

1859 Returns 

1860 ------- 

1861 inputExp : `lsst.afw.image.Exposure` 

1862 The re-constructed exposure, with appropriate detector parameters. 

1863 

1864 Raises 

1865 ------ 

1866 TypeError 

1867 Raised if the input data cannot be used to construct an exposure. 

1868 """ 

1869 if isinstance(inputExp, afwImage.DecoratedImageU): 

1870 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1871 elif isinstance(inputExp, afwImage.ImageF): 

1872 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp)) 

1873 elif isinstance(inputExp, afwImage.MaskedImageF): 

1874 inputExp = afwImage.makeExposure(inputExp) 

1875 elif isinstance(inputExp, afwImage.Exposure): 

1876 pass 

1877 elif inputExp is None: 

1878 # Assume this will be caught by the setup if it is a problem. 

1879 return inputExp 

1880 else: 

1881 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." % 

1882 (type(inputExp), )) 

1883 

1884 if inputExp.getDetector() is None: 

1885 inputExp.setDetector(camera[detectorNum]) 

1886 

1887 return inputExp 

1888 

1889 def convertIntToFloat(self, exposure): 

1890 """Convert exposure image from uint16 to float. 

1891 

1892 If the exposure does not need to be converted, the input is 

1893 immediately returned. For exposures that are converted to use 

1894 floating point pixels, the variance is set to unity and the 

1895 mask to zero. 

1896 

1897 Parameters 

1898 ---------- 

1899 exposure : `lsst.afw.image.Exposure` 

1900 The raw exposure to be converted. 

1901 

1902 Returns 

1903 ------- 

1904 newexposure : `lsst.afw.image.Exposure` 

1905 The input ``exposure``, converted to floating point pixels. 

1906 

1907 Raises 

1908 ------ 

1909 RuntimeError 

1910 Raised if the exposure type cannot be converted to float. 

1911 

1912 """ 

1913 if isinstance(exposure, afwImage.ExposureF): 

1914 # Nothing to be done 

1915 self.log.debug("Exposure already of type float.") 

1916 return exposure 

1917 if not hasattr(exposure, "convertF"): 

1918 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure)) 

1919 

1920 newexposure = exposure.convertF() 

1921 newexposure.variance[:] = 1 

1922 newexposure.mask[:] = 0x0 

1923 

1924 return newexposure 

1925 

1926 def maskAmplifier(self, ccdExposure, amp, defects): 

1927 """Identify bad amplifiers, saturated and suspect pixels. 

1928 

1929 Parameters 

1930 ---------- 

1931 ccdExposure : `lsst.afw.image.Exposure` 

1932 Input exposure to be masked. 

1933 amp : `lsst.afw.table.AmpInfoCatalog` 

1934 Catalog of parameters defining the amplifier on this 

1935 exposure to mask. 

1936 defects : `lsst.ip.isr.Defects` 

1937 List of defects. Used to determine if the entire 

1938 amplifier is bad. 

1939 

1940 Returns 

1941 ------- 

1942 badAmp : `Bool` 

1943 If this is true, the entire amplifier area is covered by 

1944 defects and unusable. 

1945 

1946 """ 

1947 maskedImage = ccdExposure.getMaskedImage() 

1948 

1949 badAmp = False 

1950 

1951 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct 

1952 # comparison with current defects definition. 

1953 if defects is not None: 

1954 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) 

1955 

1956 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct 

1957 # association with pixels in current ccdExposure). 

1958 if badAmp: 

1959 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(), 

1960 afwImage.PARENT) 

1961 maskView = dataView.getMask() 

1962 maskView |= maskView.getPlaneBitMask("BAD") 

1963 del maskView 

1964 return badAmp 

1965 

1966 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries. 

1967 # Saturation and suspect pixels can be masked now, though. 

1968 limits = dict() 

1969 if self.config.doSaturation and not badAmp: 

1970 limits.update({self.config.saturatedMaskName: amp.getSaturation()}) 

1971 if self.config.doSuspect and not badAmp: 

1972 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()}) 

1973 if math.isfinite(self.config.saturation): 

1974 limits.update({self.config.saturatedMaskName: self.config.saturation}) 

1975 

1976 for maskName, maskThreshold in limits.items(): 

1977 if not math.isnan(maskThreshold): 

1978 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

1979 isrFunctions.makeThresholdMask( 

1980 maskedImage=dataView, 

1981 threshold=maskThreshold, 

1982 growFootprints=0, 

1983 maskName=maskName 

1984 ) 

1985 

1986 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels. 

1987 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(), 

1988 afwImage.PARENT) 

1989 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName, 

1990 self.config.suspectMaskName]) 

1991 if numpy.all(maskView.getArray() & maskVal > 0): 

1992 badAmp = True 

1993 maskView |= maskView.getPlaneBitMask("BAD") 

1994 

1995 return badAmp 

1996 

1997 def overscanCorrection(self, ccdExposure, amp): 

1998 """Apply overscan correction in place. 

1999 

2000 This method does initial pixel rejection of the overscan 

2001 region. The overscan can also be optionally segmented to 

2002 allow for discontinuous overscan responses to be fit 

2003 separately. The actual overscan subtraction is performed by 

2004 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 

2005 which is called here after the amplifier is preprocessed. 

2006 

2007 Parameters 

2008 ---------- 

2009 ccdExposure : `lsst.afw.image.Exposure` 

2010 Exposure to have overscan correction performed. 

2011 amp : `lsst.afw.cameraGeom.Amplifer` 

2012 The amplifier to consider while correcting the overscan. 

2013 

2014 Returns 

2015 ------- 

2016 overscanResults : `lsst.pipe.base.Struct` 

2017 Result struct with components: 

2018 - ``imageFit`` : scalar or `lsst.afw.image.Image` 

2019 Value or fit subtracted from the amplifier image data. 

2020 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 

2021 Value or fit subtracted from the overscan image data. 

2022 - ``overscanImage`` : `lsst.afw.image.Image` 

2023 Image of the overscan region with the overscan 

2024 correction applied. This quantity is used to estimate 

2025 the amplifier read noise empirically. 

2026 

2027 Raises 

2028 ------ 

2029 RuntimeError 

2030 Raised if the ``amp`` does not contain raw pixel information. 

2031 

2032 See Also 

2033 -------- 

2034 lsst.ip.isr.isrFunctions.overscanCorrection 

2035 """ 

2036 if amp.getRawHorizontalOverscanBBox().isEmpty(): 

2037 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.") 

2038 return None 

2039 

2040 statControl = afwMath.StatisticsControl() 

2041 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2042 

2043 # Determine the bounding boxes 

2044 dataBBox = amp.getRawDataBBox() 

2045 oscanBBox = amp.getRawHorizontalOverscanBBox() 

2046 dx0 = 0 

2047 dx1 = 0 

2048 

2049 prescanBBox = amp.getRawPrescanBBox() 

2050 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right 

2051 dx0 += self.config.overscanNumLeadingColumnsToSkip 

2052 dx1 -= self.config.overscanNumTrailingColumnsToSkip 

2053 else: 

2054 dx0 += self.config.overscanNumTrailingColumnsToSkip 

2055 dx1 -= self.config.overscanNumLeadingColumnsToSkip 

2056 

2057 # Determine if we need to work on subregions of the amplifier and overscan. 

2058 imageBBoxes = [] 

2059 overscanBBoxes = [] 

2060 

2061 if ((self.config.overscanBiasJump 

2062 and self.config.overscanBiasJumpLocation) 

2063 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) 

2064 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in 

2065 self.config.overscanBiasJumpDevices)): 

2066 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR): 

2067 yLower = self.config.overscanBiasJumpLocation 

2068 yUpper = dataBBox.getHeight() - yLower 

2069 else: 

2070 yUpper = self.config.overscanBiasJumpLocation 

2071 yLower = dataBBox.getHeight() - yUpper 

2072 

2073 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2074 lsst.geom.Extent2I(dataBBox.getWidth(), yLower))) 

2075 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2076 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2077 yLower))) 

2078 

2079 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower), 

2080 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper))) 

2081 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower), 

2082 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2083 yUpper))) 

2084 else: 

2085 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(), 

2086 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight()))) 

2087 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0), 

2088 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1, 

2089 oscanBBox.getHeight()))) 

2090 

2091 # Perform overscan correction on subregions, ensuring saturated pixels are masked. 

2092 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes): 

2093 ampImage = ccdExposure.maskedImage[imageBBox] 

2094 overscanImage = ccdExposure.maskedImage[overscanBBox] 

2095 

2096 overscanArray = overscanImage.image.array 

2097 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray)) 

2098 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) 

2099 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

2100 

2101 statControl = afwMath.StatisticsControl() 

2102 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT")) 

2103 

2104 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp) 

2105 

2106 # Measure average overscan levels and record them in the metadata. 

2107 levelStat = afwMath.MEDIAN 

2108 sigmaStat = afwMath.STDEVCLIP 

2109 

2110 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma, 

2111 self.config.qa.flatness.nIter) 

2112 metadata = ccdExposure.getMetadata() 

2113 ampNum = amp.getName() 

2114 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"): 

2115 if isinstance(overscanResults.overscanFit, float): 

2116 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit) 

2117 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0) 

2118 else: 

2119 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl) 

2120 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat)) 

2121 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat)) 

2122 

2123 return overscanResults 

2124 

2125 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None): 

2126 """Set the variance plane using the gain and read noise 

2127 

2128 The read noise is calculated from the ``overscanImage`` if the 

2129 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

2130 the value from the amplifier data is used. 

2131 

2132 Parameters 

2133 ---------- 

2134 ampExposure : `lsst.afw.image.Exposure` 

2135 Exposure to process. 

2136 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

2137 Amplifier detector data. 

2138 overscanImage : `lsst.afw.image.MaskedImage`, optional. 

2139 Image of overscan, required only for empirical read noise. 

2140 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

2141 PTC dataset containing the gains and read noise. 

2142 

2143 

2144 Raises 

2145 ------ 

2146 RuntimeError 

2147 Raised if either ``usePtcGains`` of ``usePtcReadNoise`` 

2148 are ``True``, but ptcDataset is not provided. 

2149 

2150 Raised if ```doEmpiricalReadNoise`` is ``True`` but 

2151 ``overscanImage`` is ``None``. 

2152 

2153 See also 

2154 -------- 

2155 lsst.ip.isr.isrFunctions.updateVariance 

2156 """ 

2157 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

2158 if self.config.usePtcGains: 

2159 if ptcDataset is None: 

2160 raise RuntimeError("No ptcDataset provided to use PTC gains.") 

2161 else: 

2162 gain = ptcDataset.gain[amp.getName()] 

2163 self.log.info("Using gain from Photon Transfer Curve.") 

2164 else: 

2165 gain = amp.getGain() 

2166 

2167 if math.isnan(gain): 

2168 gain = 1.0 

2169 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.") 

2170 elif gain <= 0: 

2171 patchedGain = 1.0 

2172 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.", 

2173 amp.getName(), gain, patchedGain) 

2174 gain = patchedGain 

2175 

2176 if self.config.doEmpiricalReadNoise and overscanImage is None: 

2177 raise RuntimeError("Overscan is none for EmpiricalReadNoise.") 

2178 

2179 if self.config.doEmpiricalReadNoise and overscanImage is not None: 

2180 stats = afwMath.StatisticsControl() 

2181 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

2182 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

2183 self.log.info("Calculated empirical read noise for amp %s: %f.", 

2184 amp.getName(), readNoise) 

2185 elif self.config.usePtcReadNoise: 

2186 if ptcDataset is None: 

2187 raise RuntimeError("No ptcDataset provided to use PTC readnoise.") 

2188 else: 

2189 readNoise = ptcDataset.noise[amp.getName()] 

2190 self.log.info("Using read noise from Photon Transfer Curve.") 

2191 else: 

2192 readNoise = amp.getReadNoise() 

2193 

2194 isrFunctions.updateVariance( 

2195 maskedImage=ampExposure.getMaskedImage(), 

2196 gain=gain, 

2197 readNoise=readNoise, 

2198 ) 

2199 

2200 def maskNegativeVariance(self, exposure): 

2201 """Identify and mask pixels with negative variance values. 

2202 

2203 Parameters 

2204 ---------- 

2205 exposure : `lsst.afw.image.Exposure` 

2206 Exposure to process. 

2207 

2208 See Also 

2209 -------- 

2210 lsst.ip.isr.isrFunctions.updateVariance 

2211 """ 

2212 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName) 

2213 bad = numpy.where(exposure.getVariance().getArray() <= 0.0) 

2214 exposure.mask.array[bad] |= maskPlane 

2215 

2216 def darkCorrection(self, exposure, darkExposure, invert=False): 

2217 """Apply dark correction in place. 

2218 

2219 Parameters 

2220 ---------- 

2221 exposure : `lsst.afw.image.Exposure` 

2222 Exposure to process. 

2223 darkExposure : `lsst.afw.image.Exposure` 

2224 Dark exposure of the same size as ``exposure``. 

2225 invert : `Bool`, optional 

2226 If True, re-add the dark to an already corrected image. 

2227 

2228 Raises 

2229 ------ 

2230 RuntimeError 

2231 Raised if either ``exposure`` or ``darkExposure`` do not 

2232 have their dark time defined. 

2233 

2234 See Also 

2235 -------- 

2236 lsst.ip.isr.isrFunctions.darkCorrection 

2237 """ 

2238 expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

2239 if math.isnan(expScale): 

2240 raise RuntimeError("Exposure darktime is NAN.") 

2241 if darkExposure.getInfo().getVisitInfo() is not None \ 

2242 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()): 

2243 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

2244 else: 

2245 # DM-17444: darkExposure.getInfo.getVisitInfo() is None 

2246 # so getDarkTime() does not exist. 

2247 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.") 

2248 darkScale = 1.0 

2249 

2250 isrFunctions.darkCorrection( 

2251 maskedImage=exposure.getMaskedImage(), 

2252 darkMaskedImage=darkExposure.getMaskedImage(), 

2253 expScale=expScale, 

2254 darkScale=darkScale, 

2255 invert=invert, 

2256 trimToFit=self.config.doTrimToMatchCalib 

2257 ) 

2258 

2259 def doLinearize(self, detector): 

2260 """Check if linearization is needed for the detector cameraGeom. 

2261 

2262 Checks config.doLinearize and the linearity type of the first 

2263 amplifier. 

2264 

2265 Parameters 

2266 ---------- 

2267 detector : `lsst.afw.cameraGeom.Detector` 

2268 Detector to get linearity type from. 

2269 

2270 Returns 

2271 ------- 

2272 doLinearize : `Bool` 

2273 If True, linearization should be performed. 

2274 """ 

2275 return self.config.doLinearize and \ 

2276 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType 

2277 

2278 def flatCorrection(self, exposure, flatExposure, invert=False): 

2279 """Apply flat correction in place. 

2280 

2281 Parameters 

2282 ---------- 

2283 exposure : `lsst.afw.image.Exposure` 

2284 Exposure to process. 

2285 flatExposure : `lsst.afw.image.Exposure` 

2286 Flat exposure of the same size as ``exposure``. 

2287 invert : `Bool`, optional 

2288 If True, unflatten an already flattened image. 

2289 

2290 See Also 

2291 -------- 

2292 lsst.ip.isr.isrFunctions.flatCorrection 

2293 """ 

2294 isrFunctions.flatCorrection( 

2295 maskedImage=exposure.getMaskedImage(), 

2296 flatMaskedImage=flatExposure.getMaskedImage(), 

2297 scalingType=self.config.flatScalingType, 

2298 userScale=self.config.flatUserScale, 

2299 invert=invert, 

2300 trimToFit=self.config.doTrimToMatchCalib 

2301 ) 

2302 

2303 def saturationDetection(self, exposure, amp): 

2304 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 

2305 

2306 Parameters 

2307 ---------- 

2308 exposure : `lsst.afw.image.Exposure` 

2309 Exposure to process. Only the amplifier DataSec is processed. 

2310 amp : `lsst.afw.table.AmpInfoCatalog` 

2311 Amplifier detector data. 

2312 

2313 See Also 

2314 -------- 

2315 lsst.ip.isr.isrFunctions.makeThresholdMask 

2316 """ 

2317 if not math.isnan(amp.getSaturation()): 

2318 maskedImage = exposure.getMaskedImage() 

2319 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2320 isrFunctions.makeThresholdMask( 

2321 maskedImage=dataView, 

2322 threshold=amp.getSaturation(), 

2323 growFootprints=0, 

2324 maskName=self.config.saturatedMaskName, 

2325 ) 

2326 

2327 def saturationInterpolation(self, exposure): 

2328 """Interpolate over saturated pixels, in place. 

2329 

2330 This method should be called after `saturationDetection`, to 

2331 ensure that the saturated pixels have been identified in the 

2332 SAT mask. It should also be called after `assembleCcd`, since 

2333 saturated regions may cross amplifier boundaries. 

2334 

2335 Parameters 

2336 ---------- 

2337 exposure : `lsst.afw.image.Exposure` 

2338 Exposure to process. 

2339 

2340 See Also 

2341 -------- 

2342 lsst.ip.isr.isrTask.saturationDetection 

2343 lsst.ip.isr.isrFunctions.interpolateFromMask 

2344 """ 

2345 isrFunctions.interpolateFromMask( 

2346 maskedImage=exposure.getMaskedImage(), 

2347 fwhm=self.config.fwhm, 

2348 growSaturatedFootprints=self.config.growSaturationFootprintSize, 

2349 maskNameList=list(self.config.saturatedMaskName), 

2350 ) 

2351 

2352 def suspectDetection(self, exposure, amp): 

2353 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 

2354 

2355 Parameters 

2356 ---------- 

2357 exposure : `lsst.afw.image.Exposure` 

2358 Exposure to process. Only the amplifier DataSec is processed. 

2359 amp : `lsst.afw.table.AmpInfoCatalog` 

2360 Amplifier detector data. 

2361 

2362 See Also 

2363 -------- 

2364 lsst.ip.isr.isrFunctions.makeThresholdMask 

2365 

2366 Notes 

2367 ----- 

2368 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

2369 This is intended to indicate pixels that may be affected by unknown systematics; 

2370 for example if non-linearity corrections above a certain level are unstable 

2371 then that would be a useful value for suspectLevel. A value of `nan` indicates 

2372 that no such level exists and no pixels are to be masked as suspicious. 

2373 """ 

2374 suspectLevel = amp.getSuspectLevel() 

2375 if math.isnan(suspectLevel): 

2376 return 

2377 

2378 maskedImage = exposure.getMaskedImage() 

2379 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

2380 isrFunctions.makeThresholdMask( 

2381 maskedImage=dataView, 

2382 threshold=suspectLevel, 

2383 growFootprints=0, 

2384 maskName=self.config.suspectMaskName, 

2385 ) 

2386 

2387 def maskDefect(self, exposure, defectBaseList): 

2388 """Mask defects using mask plane "BAD", in place. 

2389 

2390 Parameters 

2391 ---------- 

2392 exposure : `lsst.afw.image.Exposure` 

2393 Exposure to process. 

2394 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2395 `lsst.afw.image.DefectBase`. 

2396 List of defects to mask. 

2397 

2398 Notes 

2399 ----- 

2400 Call this after CCD assembly, since defects may cross amplifier boundaries. 

2401 """ 

2402 maskedImage = exposure.getMaskedImage() 

2403 if not isinstance(defectBaseList, Defects): 

2404 # Promotes DefectBase to Defect 

2405 defectList = Defects(defectBaseList) 

2406 else: 

2407 defectList = defectBaseList 

2408 defectList.maskPixels(maskedImage, maskName="BAD") 

2409 

2410 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'): 

2411 """Mask edge pixels with applicable mask plane. 

2412 

2413 Parameters 

2414 ---------- 

2415 exposure : `lsst.afw.image.Exposure` 

2416 Exposure to process. 

2417 numEdgePixels : `int`, optional 

2418 Number of edge pixels to mask. 

2419 maskPlane : `str`, optional 

2420 Mask plane name to use. 

2421 level : `str`, optional 

2422 Level at which to mask edges. 

2423 """ 

2424 maskedImage = exposure.getMaskedImage() 

2425 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

2426 

2427 if numEdgePixels > 0: 

2428 if level == 'DETECTOR': 

2429 boxes = [maskedImage.getBBox()] 

2430 elif level == 'AMP': 

2431 boxes = [amp.getBBox() for amp in exposure.getDetector()] 

2432 

2433 for box in boxes: 

2434 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side 

2435 subImage = maskedImage[box] 

2436 box.grow(-numEdgePixels) 

2437 # Mask pixels outside box 

2438 SourceDetectionTask.setEdgeBits( 

2439 subImage, 

2440 box, 

2441 maskBitMask) 

2442 

2443 def maskAndInterpolateDefects(self, exposure, defectBaseList): 

2444 """Mask and interpolate defects using mask plane "BAD", in place. 

2445 

2446 Parameters 

2447 ---------- 

2448 exposure : `lsst.afw.image.Exposure` 

2449 Exposure to process. 

2450 defectBaseList : `lsst.ip.isr.Defects` or `list` of 

2451 `lsst.afw.image.DefectBase`. 

2452 List of defects to mask and interpolate. 

2453 

2454 See Also 

2455 -------- 

2456 lsst.ip.isr.isrTask.maskDefect 

2457 """ 

2458 self.maskDefect(exposure, defectBaseList) 

2459 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect, 

2460 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

2461 isrFunctions.interpolateFromMask( 

2462 maskedImage=exposure.getMaskedImage(), 

2463 fwhm=self.config.fwhm, 

2464 growSaturatedFootprints=0, 

2465 maskNameList=["BAD"], 

2466 ) 

2467 

2468 def maskNan(self, exposure): 

2469 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 

2470 

2471 Parameters 

2472 ---------- 

2473 exposure : `lsst.afw.image.Exposure` 

2474 Exposure to process. 

2475 

2476 Notes 

2477 ----- 

2478 We mask over all non-finite values (NaN, inf), including those 

2479 that are masked with other bits (because those may or may not be 

2480 interpolated over later, and we want to remove all NaN/infs). 

2481 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to 

2482 preserve the historical name. 

2483 """ 

2484 maskedImage = exposure.getMaskedImage() 

2485 

2486 # Find and mask NaNs 

2487 maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

2488 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

2489 numNans = maskNans(maskedImage, maskVal) 

2490 self.metadata.set("NUMNANS", numNans) 

2491 if numNans > 0: 

2492 self.log.warning("There were %d unmasked NaNs.", numNans) 

2493 

2494 def maskAndInterpolateNan(self, exposure): 

2495 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN", 

2496 in place. 

2497 

2498 Parameters 

2499 ---------- 

2500 exposure : `lsst.afw.image.Exposure` 

2501 Exposure to process. 

2502 

2503 See Also 

2504 -------- 

2505 lsst.ip.isr.isrTask.maskNan 

2506 """ 

2507 self.maskNan(exposure) 

2508 isrFunctions.interpolateFromMask( 

2509 maskedImage=exposure.getMaskedImage(), 

2510 fwhm=self.config.fwhm, 

2511 growSaturatedFootprints=0, 

2512 maskNameList=["UNMASKEDNAN"], 

2513 ) 

2514 

2515 def measureBackground(self, exposure, IsrQaConfig=None): 

2516 """Measure the image background in subgrids, for quality control purposes. 

2517 

2518 Parameters 

2519 ---------- 

2520 exposure : `lsst.afw.image.Exposure` 

2521 Exposure to process. 

2522 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 

2523 Configuration object containing parameters on which background 

2524 statistics and subgrids to use. 

2525 """ 

2526 if IsrQaConfig is not None: 

2527 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma, 

2528 IsrQaConfig.flatness.nIter) 

2529 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"]) 

2530 statsControl.setAndMask(maskVal) 

2531 maskedImage = exposure.getMaskedImage() 

2532 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl) 

2533 skyLevel = stats.getValue(afwMath.MEDIAN) 

2534 skySigma = stats.getValue(afwMath.STDEVCLIP) 

2535 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma) 

2536 metadata = exposure.getMetadata() 

2537 metadata.set('SKYLEVEL', skyLevel) 

2538 metadata.set('SKYSIGMA', skySigma) 

2539 

2540 # calcluating flatlevel over the subgrids 

2541 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN 

2542 meshXHalf = int(IsrQaConfig.flatness.meshX/2.) 

2543 meshYHalf = int(IsrQaConfig.flatness.meshY/2.) 

2544 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX) 

2545 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY) 

2546 skyLevels = numpy.zeros((nX, nY)) 

2547 

2548 for j in range(nY): 

2549 yc = meshYHalf + j * IsrQaConfig.flatness.meshY 

2550 for i in range(nX): 

2551 xc = meshXHalf + i * IsrQaConfig.flatness.meshX 

2552 

2553 xLLC = xc - meshXHalf 

2554 yLLC = yc - meshYHalf 

2555 xURC = xc + meshXHalf - 1 

2556 yURC = yc + meshYHalf - 1 

2557 

2558 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC)) 

2559 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL) 

2560 

2561 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue() 

2562 

2563 good = numpy.where(numpy.isfinite(skyLevels)) 

2564 skyMedian = numpy.median(skyLevels[good]) 

2565 flatness = (skyLevels[good] - skyMedian) / skyMedian 

2566 flatness_rms = numpy.std(flatness) 

2567 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan 

2568 

2569 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian) 

2570 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.", 

2571 nX, nY, flatness_pp, flatness_rms) 

2572 

2573 metadata.set('FLATNESS_PP', float(flatness_pp)) 

2574 metadata.set('FLATNESS_RMS', float(flatness_rms)) 

2575 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY)) 

2576 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX) 

2577 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY) 

2578 

2579 def roughZeroPoint(self, exposure): 

2580 """Set an approximate magnitude zero point for the exposure. 

2581 

2582 Parameters 

2583 ---------- 

2584 exposure : `lsst.afw.image.Exposure` 

2585 Exposure to process. 

2586 """ 

2587 filterLabel = exposure.getFilterLabel() 

2588 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log) 

2589 

2590 if physicalFilter in self.config.fluxMag0T1: 

2591 fluxMag0 = self.config.fluxMag0T1[physicalFilter] 

2592 else: 

2593 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter) 

2594 fluxMag0 = self.config.defaultFluxMag0T1 

2595 

2596 expTime = exposure.getInfo().getVisitInfo().getExposureTime() 

2597 if not expTime > 0: # handle NaN as well as <= 0 

2598 self.log.warning("Non-positive exposure time; skipping rough zero point.") 

2599 return 

2600 

2601 self.log.info("Setting rough magnitude zero point for filter %s: %f", 

2602 physicalFilter, 2.5*math.log10(fluxMag0*expTime)) 

2603 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0)) 

2604 

2605 def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

2606 """Set the valid polygon as the intersection of fpPolygon and the ccd corners. 

2607 

2608 Parameters 

2609 ---------- 

2610 ccdExposure : `lsst.afw.image.Exposure` 

2611 Exposure to process. 

2612 fpPolygon : `lsst.afw.geom.Polygon` 

2613 Polygon in focal plane coordinates. 

2614 """ 

2615 # Get ccd corners in focal plane coordinates 

2616 ccd = ccdExposure.getDetector() 

2617 fpCorners = ccd.getCorners(FOCAL_PLANE) 

2618 ccdPolygon = Polygon(fpCorners) 

2619 

2620 # Get intersection of ccd corners with fpPolygon 

2621 intersect = ccdPolygon.intersectionSingle(fpPolygon) 

2622 

2623 # Transform back to pixel positions and build new polygon 

2624 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

2625 validPolygon = Polygon(ccdPoints) 

2626 ccdExposure.getInfo().setValidPolygon(validPolygon) 

2627 

2628 @contextmanager 

2629 def flatContext(self, exp, flat, dark=None): 

2630 """Context manager that applies and removes flats and darks, 

2631 if the task is configured to apply them. 

2632 

2633 Parameters 

2634 ---------- 

2635 exp : `lsst.afw.image.Exposure` 

2636 Exposure to process. 

2637 flat : `lsst.afw.image.Exposure` 

2638 Flat exposure the same size as ``exp``. 

2639 dark : `lsst.afw.image.Exposure`, optional 

2640 Dark exposure the same size as ``exp``. 

2641 

2642 Yields 

2643 ------ 

2644 exp : `lsst.afw.image.Exposure` 

2645 The flat and dark corrected exposure. 

2646 """ 

2647 if self.config.doDark and dark is not None: 

2648 self.darkCorrection(exp, dark) 

2649 if self.config.doFlat: 

2650 self.flatCorrection(exp, flat) 

2651 try: 

2652 yield exp 

2653 finally: 

2654 if self.config.doFlat: 

2655 self.flatCorrection(exp, flat, invert=True) 

2656 if self.config.doDark and dark is not None: 

2657 self.darkCorrection(exp, dark, invert=True) 

2658 

2659 def debugView(self, exposure, stepname): 

2660 """Utility function to examine ISR exposure at different stages. 

2661 

2662 Parameters 

2663 ---------- 

2664 exposure : `lsst.afw.image.Exposure` 

2665 Exposure to view. 

2666 stepname : `str` 

2667 State of processing to view. 

2668 """ 

2669 frame = getDebugFrame(self._display, stepname) 

2670 if frame: 

2671 display = getDisplay(frame) 

2672 display.scale('asinh', 'zscale') 

2673 display.mtv(exposure) 

2674 prompt = "Press Enter to continue [c]... " 

2675 while True: 

2676 ans = input(prompt).lower() 

2677 if ans in ("", "c",): 

2678 break 

2679 

2680 

2681class FakeAmp(object): 

2682 """A Detector-like object that supports returning gain and saturation level 

2683 

2684 This is used when the input exposure does not have a detector. 

2685 

2686 Parameters 

2687 ---------- 

2688 exposure : `lsst.afw.image.Exposure` 

2689 Exposure to generate a fake amplifier for. 

2690 config : `lsst.ip.isr.isrTaskConfig` 

2691 Configuration to apply to the fake amplifier. 

2692 """ 

2693 

2694 def __init__(self, exposure, config): 

2695 self._bbox = exposure.getBBox(afwImage.LOCAL) 

2696 self._RawHorizontalOverscanBBox = lsst.geom.Box2I() 

2697 self._gain = config.gain 

2698 self._readNoise = config.readNoise 

2699 self._saturation = config.saturation 

2700 

2701 def getBBox(self): 

2702 return self._bbox 

2703 

2704 def getRawBBox(self): 

2705 return self._bbox 

2706 

2707 def getRawHorizontalOverscanBBox(self): 

2708 return self._RawHorizontalOverscanBBox 

2709 

2710 def getGain(self): 

2711 return self._gain 

2712 

2713 def getReadNoise(self): 

2714 return self._readNoise 

2715 

2716 def getSaturation(self): 

2717 return self._saturation 

2718 

2719 def getSuspectLevel(self): 

2720 return float("NaN") 

2721 

2722 

2723class RunIsrConfig(pexConfig.Config): 

2724 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

2725 

2726 

2727class RunIsrTask(pipeBase.CmdLineTask): 

2728 """Task to wrap the default IsrTask to allow it to be retargeted. 

2729 

2730 The standard IsrTask can be called directly from a command line 

2731 program, but doing so removes the ability of the task to be 

2732 retargeted. As most cameras override some set of the IsrTask 

2733 methods, this would remove those data-specific methods in the 

2734 output post-ISR images. This wrapping class fixes the issue, 

2735 allowing identical post-ISR images to be generated by both the 

2736 processCcd and isrTask code. 

2737 """ 

2738 ConfigClass = RunIsrConfig 

2739 _DefaultName = "runIsr" 

2740 

2741 def __init__(self, *args, **kwargs): 

2742 super().__init__(*args, **kwargs) 

2743 self.makeSubtask("isr") 

2744 

2745 def runDataRef(self, dataRef): 

2746 """ 

2747 Parameters 

2748 ---------- 

2749 dataRef : `lsst.daf.persistence.ButlerDataRef` 

2750 data reference of the detector data to be processed 

2751 

2752 Returns 

2753 ------- 

2754 result : `pipeBase.Struct` 

2755 Result struct with component: 

2756 

2757 - exposure : `lsst.afw.image.Exposure` 

2758 Post-ISR processed exposure. 

2759 """ 

2760 return self.isr.runDataRef(dataRef)