Coverage for python/lsst/ip/isr/isrTaskLSST.py: 19%
475 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 04:06 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 04:06 -0700
1__all__ = ["IsrTaskLSST", "IsrTaskLSSTConfig"]
3import numpy
4import math
6from . import isrFunctions
7from . import isrQa
8from . import linearize
9from .defects import Defects
11from contextlib import contextmanager
12from lsst.afw.cameraGeom import NullLinearityType
13import lsst.pex.config as pexConfig
14import lsst.afw.math as afwMath
15import lsst.pipe.base as pipeBase
16import lsst.afw.image as afwImage
17import lsst.pipe.base.connectionTypes as cT
18from lsst.meas.algorithms.detection import SourceDetectionTask
20from .overscan import SerialOverscanCorrectionTask, ParallelOverscanCorrectionTask
21from .overscanAmpConfig import OverscanCameraConfig
22from .assembleCcdTask import AssembleCcdTask
23from .deferredCharge import DeferredChargeTask
24from .crosstalk import CrosstalkTask
25from .masking import MaskingTask
26from .isrStatistics import IsrStatisticsTask
27from .isr import maskNans
30class IsrTaskLSSTConnections(pipeBase.PipelineTaskConnections,
31 dimensions={"instrument", "exposure", "detector"},
32 defaultTemplates={}):
33 ccdExposure = cT.Input(
34 name="raw",
35 doc="Input exposure to process.",
36 storageClass="Exposure",
37 dimensions=["instrument", "exposure", "detector"],
38 )
39 camera = cT.PrerequisiteInput(
40 name="camera",
41 storageClass="Camera",
42 doc="Input camera to construct complete exposures.",
43 dimensions=["instrument"],
44 isCalibration=True,
45 )
46 dnlLUT = cT.PrerequisiteInput(
47 name="dnlLUT",
48 doc="Look-up table for differential non-linearity.",
49 storageClass="IsrCalib",
50 dimensions=["instrument", "exposure", "detector"],
51 isCalibration=True,
52 # TODO DM 36636
53 )
54 bias = cT.PrerequisiteInput(
55 name="bias",
56 doc="Input bias calibration.",
57 storageClass="ExposureF",
58 dimensions=["instrument", "detector"],
59 isCalibration=True,
60 )
61 deferredChargeCalib = cT.PrerequisiteInput(
62 name="cpCtiCalib",
63 doc="Deferred charge/CTI correction dataset.",
64 storageClass="IsrCalib",
65 dimensions=["instrument", "detector"],
66 isCalibration=True,
67 )
68 linearizer = cT.PrerequisiteInput(
69 name='linearizer',
70 storageClass="Linearizer",
71 doc="Linearity correction calibration.",
72 dimensions=["instrument", "detector"],
73 isCalibration=True,
74 )
75 ptc = cT.PrerequisiteInput(
76 name="ptc",
77 doc="Input Photon Transfer Curve dataset",
78 storageClass="PhotonTransferCurveDataset",
79 dimensions=["instrument", "detector"],
80 isCalibration=True,
81 )
82 crosstalk = cT.PrerequisiteInput(
83 name="crosstalk",
84 doc="Input crosstalk object",
85 storageClass="CrosstalkCalib",
86 dimensions=["instrument", "detector"],
87 isCalibration=True,
88 )
89 defects = cT.PrerequisiteInput(
90 name='defects',
91 doc="Input defect tables.",
92 storageClass="Defects",
93 dimensions=["instrument", "detector"],
94 isCalibration=True,
95 )
96 bfKernel = cT.PrerequisiteInput(
97 name='brighterFatterKernel',
98 doc="Complete kernel + gain solutions.",
99 storageClass="BrighterFatterKernel",
100 dimensions=["instrument", "detector"],
101 isCalibration=True,
102 )
103 dark = cT.PrerequisiteInput(
104 name='dark',
105 doc="Input dark calibration.",
106 storageClass="ExposureF",
107 dimensions=["instrument", "detector"],
108 isCalibration=True,
109 )
110 outputExposure = cT.Output(
111 name='postISRCCD',
112 doc="Output ISR processed exposure.",
113 storageClass="Exposure",
114 dimensions=["instrument", "exposure", "detector"],
115 )
116 preInterpExposure = cT.Output(
117 name='preInterpISRCCD',
118 doc="Output ISR processed exposure, with pixels left uninterpolated.",
119 storageClass="ExposureF",
120 dimensions=["instrument", "exposure", "detector"],
121 )
122 outputBin1Exposure = cT.Output(
123 name="postIsrBin1",
124 doc="First binned image.",
125 storageClass="ExposureF",
126 dimensions=["instrument", "exposure", "detector"],
127 )
128 outputBin2Exposure = cT.Output(
129 name="postIsrBin2",
130 doc="Second binned image.",
131 storageClass="ExposureF",
132 dimensions=["instrument", "exposure", "detector"],
133 )
135 outputStatistics = cT.Output(
136 name="isrStatistics",
137 doc="Output of additional statistics table.",
138 storageClass="StructuredDataDict",
139 dimensions=["instrument", "exposure", "detector"],
140 )
142 def __init__(self, *, config=None):
143 super().__init__(config=config)
145 if config.doDiffNonLinearCorrection is not True:
146 del self.dnlLUT
147 if config.doBias is not True:
148 del self.bias
149 if config.doDeferredCharge is not True:
150 del self.deferredChargeCalib
151 if config.doLinearize is not True:
152 del self.linearizer
153 if not config.doCrosstalk and not config.overscanCamera.doAnyParallelOverscanCrosstalk:
154 del self.crosstalk
155 if config.doDefect is not True:
156 del self.defects
157 if config.doBrighterFatter is not True:
158 del self.bfKernel
159 if config.doDark is not True:
160 del self.dark
162 if config.doBinnedExposures is not True:
163 del self.outputBin1Exposure
164 del self.outputBin2Exposure
165 if config.doSaveInterpPixels is not True:
166 del self.preInterpExposure
168 if config.doCalculateStatistics is not True:
169 del self.outputStatistics
172class IsrTaskLSSTConfig(pipeBase.PipelineTaskConfig,
173 pipelineConnections=IsrTaskLSSTConnections):
174 """Configuration parameters for IsrTaskLSST.
176 Items are grouped in the order in which they are executed by the task.
177 """
178 expectWcs = pexConfig.Field(
179 dtype=bool,
180 default=True,
181 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
182 )
183 qa = pexConfig.ConfigField(
184 dtype=isrQa.IsrQaConfig,
185 doc="QA related configuration options.",
186 )
187 doHeaderProvenance = pexConfig.Field(
188 dtype=bool,
189 default=True,
190 doc="Write calibration identifiers into output exposure header.",
191 )
193 # Differential non-linearity correction.
194 doDiffNonLinearCorrection = pexConfig.Field(
195 dtype=bool,
196 doc="Do differential non-linearity correction?",
197 default=False,
198 )
200 overscanCamera = pexConfig.ConfigField(
201 dtype=OverscanCameraConfig,
202 doc="Per-detector and per-amplifier overscan configurations.",
203 )
205 # Amplifier to CCD assembly configuration.
206 doAssembleCcd = pexConfig.Field(
207 dtype=bool,
208 default=True,
209 doc="Assemble amp-level exposures into a ccd-level exposure?"
210 )
211 assembleCcd = pexConfig.ConfigurableField(
212 target=AssembleCcdTask,
213 doc="CCD assembly task.",
214 )
216 # Bias subtraction.
217 doBias = pexConfig.Field(
218 dtype=bool,
219 doc="Apply bias frame correction?",
220 default=True,
221 )
223 # Deferred charge correction.
224 doDeferredCharge = pexConfig.Field(
225 dtype=bool,
226 doc="Apply deferred charge correction?",
227 default=True,
228 )
229 deferredChargeCorrection = pexConfig.ConfigurableField(
230 target=DeferredChargeTask,
231 doc="Deferred charge correction task.",
232 )
234 # Linearization.
235 doLinearize = pexConfig.Field(
236 dtype=bool,
237 doc="Correct for nonlinearity of the detector's response?",
238 default=True,
239 )
241 # Gains.
242 doGainsCorrection = pexConfig.Field(
243 dtype=bool,
244 doc="Apply temperature correction to the gains?",
245 default=False,
246 )
247 doApplyGains = pexConfig.Field(
248 dtype=bool,
249 doc="Apply gains to the image?",
250 default=True,
251 )
253 # Variance construction.
254 doVariance = pexConfig.Field(
255 dtype=bool,
256 doc="Calculate variance?",
257 default=True
258 )
259 gain = pexConfig.Field(
260 dtype=float,
261 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN).",
262 default=float("NaN"),
263 )
264 maskNegativeVariance = pexConfig.Field(
265 dtype=bool,
266 doc="Mask pixels that claim a negative variance. This likely indicates a failure "
267 "in the measurement of the overscan at an edge due to the data falling off faster "
268 "than the overscan model can account for it.",
269 default=True,
270 )
271 negativeVarianceMaskName = pexConfig.Field(
272 dtype=str,
273 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
274 default="BAD",
275 )
276 saturatedMaskName = pexConfig.Field(
277 dtype=str,
278 doc="Name of mask plane to use in saturation detection and interpolation.",
279 default="SAT",
280 )
281 suspectMaskName = pexConfig.Field(
282 dtype=str,
283 doc="Name of mask plane to use for suspect pixels.",
284 default="SUSPECT",
285 )
287 # Crosstalk.
288 doCrosstalk = pexConfig.Field(
289 dtype=bool,
290 doc="Apply intra-CCD crosstalk correction?",
291 default=True,
292 )
293 crosstalk = pexConfig.ConfigurableField(
294 target=CrosstalkTask,
295 doc="Intra-CCD crosstalk correction.",
296 )
298 # Masking options.
299 doDefect = pexConfig.Field(
300 dtype=bool,
301 doc="Apply correction for CCD defects, e.g. hot pixels?",
302 default=True,
303 )
304 doNanMasking = pexConfig.Field(
305 dtype=bool,
306 doc="Mask non-finite (NAN, inf) pixels.",
307 default=True,
308 )
309 doWidenSaturationTrails = pexConfig.Field(
310 dtype=bool,
311 doc="Widen bleed trails based on their width.",
312 default=True,
313 )
314 masking = pexConfig.ConfigurableField(
315 target=MaskingTask,
316 doc="Masking task."
317 )
319 # Interpolation options.
320 doInterpolate = pexConfig.Field(
321 dtype=bool,
322 doc="Interpolate masked pixels?",
323 default=True,
324 )
325 maskListToInterpolate = pexConfig.ListField(
326 dtype=str,
327 doc="List of mask planes that should be interpolated.",
328 default=['SAT', 'BAD'],
329 )
330 doSaveInterpPixels = pexConfig.Field(
331 dtype=bool,
332 doc="Save a copy of the pre-interpolated pixel values?",
333 default=False,
334 )
336 # Initial masking options.
337 doSetBadRegions = pexConfig.Field(
338 dtype=bool,
339 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
340 default=True,
341 )
343 # Brighter-Fatter correction.
344 doBrighterFatter = pexConfig.Field(
345 dtype=bool,
346 doc="Apply the brighter-fatter correction?",
347 default=True,
348 )
349 brighterFatterLevel = pexConfig.ChoiceField(
350 dtype=str,
351 doc="The level at which to correct for brighter-fatter.",
352 allowed={
353 "AMP": "Every amplifier treated separately.",
354 "DETECTOR": "One kernel per detector.",
355 },
356 default="DETECTOR",
357 )
358 brighterFatterMaxIter = pexConfig.Field(
359 dtype=int,
360 doc="Maximum number of iterations for the brighter-fatter correction.",
361 default=10,
362 )
363 brighterFatterThreshold = pexConfig.Field(
364 dtype=float,
365 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
366 "absolute value of the difference between the current corrected image and the one "
367 "from the previous iteration summed over all the pixels.",
368 default=1000,
369 )
370 brighterFatterApplyGain = pexConfig.Field(
371 dtype=bool,
372 doc="Should the gain be applied when applying the brighter-fatter correction?",
373 default=True,
374 )
375 brighterFatterMaskListToInterpolate = pexConfig.ListField(
376 dtype=str,
377 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
378 "correction.",
379 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
380 )
381 brighterFatterMaskGrowSize = pexConfig.Field(
382 dtype=int,
383 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
384 "when brighter-fatter correction is applied.",
385 default=0,
386 )
387 brighterFatterFwhmForInterpolation = pexConfig.Field(
388 dtype=float,
389 doc="FWHM of PSF in arcseconds used for interpolation in brighter-fatter correction "
390 "(currently unused).",
391 default=1.0,
392 )
393 growSaturationFootprintSize = pexConfig.Field(
394 dtype=int,
395 doc="Number of pixels by which to grow the saturation footprints.",
396 default=1,
397 )
398 brighterFatterMaskListToInterpolate = pexConfig.ListField(
399 dtype=str,
400 doc="List of mask planes that should be interpolated over when applying the brighter-fatter."
401 "correction.",
402 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
403 )
405 # Dark subtraction.
406 doDark = pexConfig.Field(
407 dtype=bool,
408 doc="Apply dark frame correction.",
409 default=True,
410 )
412 # Flat correction.
413 doFlat = pexConfig.Field(
414 dtype=bool,
415 doc="Apply flat field correction.",
416 default=True,
417 )
418 flatScalingType = pexConfig.ChoiceField(
419 dtype=str,
420 doc="The method for scaling the flat on the fly.",
421 default='USER',
422 allowed={
423 "USER": "Scale by flatUserScale",
424 "MEAN": "Scale by the inverse of the mean",
425 "MEDIAN": "Scale by the inverse of the median",
426 },
427 )
428 flatUserScale = pexConfig.Field(
429 dtype=float,
430 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise.",
431 default=1.0,
432 )
434 # Calculate image quality statistics?
435 doStandardStatistics = pexConfig.Field(
436 dtype=bool,
437 doc="Should standard image quality statistics be calculated?",
438 default=True,
439 )
440 # Calculate additional statistics?
441 doCalculateStatistics = pexConfig.Field(
442 dtype=bool,
443 doc="Should additional ISR statistics be calculated?",
444 default=True,
445 )
446 isrStats = pexConfig.ConfigurableField(
447 target=IsrStatisticsTask,
448 doc="Task to calculate additional statistics.",
449 )
451 # Make binned images?
452 doBinnedExposures = pexConfig.Field(
453 dtype=bool,
454 doc="Should binned exposures be calculated?",
455 default=False,
456 )
457 binFactor1 = pexConfig.Field( 457 ↛ exitline 457 didn't jump to the function exit
458 dtype=int,
459 doc="Binning factor for first binned exposure. This is intended for a finely binned output.",
460 default=8,
461 check=lambda x: x > 1,
462 )
463 binFactor2 = pexConfig.Field( 463 ↛ exitline 463 didn't jump to the function exit
464 dtype=int,
465 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.",
466 default=64,
467 check=lambda x: x > 1,
468 )
470 def validate(self):
471 super().validate()
473 if self.doCalculateStatistics and self.isrStats.doCtiStatistics:
474 # DM-41912: Implement doApplyGains in LSST IsrTask
475 # if self.doApplyGains !=
476 # self.isrStats.doApplyGainsForCtiStatistics:
477 raise ValueError("doApplyGains must match isrStats.applyGainForCtiStatistics.")
479 def setDefaults(self):
480 super().setDefaults()
483class IsrTaskLSST(pipeBase.PipelineTask):
484 ConfigClass = IsrTaskLSSTConfig
485 _DefaultName = "isr"
487 def __init__(self, **kwargs):
488 super().__init__(**kwargs)
489 self.makeSubtask("assembleCcd")
490 self.makeSubtask("deferredChargeCorrection")
491 self.makeSubtask("crosstalk")
492 self.makeSubtask("masking")
493 self.makeSubtask("isrStats")
495 def runQuantum(self, butlerQC, inputRefs, outputRefs):
497 inputs = butlerQC.get(inputRefs)
498 self.validateInput(inputs)
499 super().runQuantum(butlerQC, inputRefs, outputRefs)
501 def validateInput(self, inputs):
502 """
503 This is a check that all the inputs required by the config
504 are available.
505 """
507 doCrosstalk = self.config.doCrosstalk or self.config.overscanCamera.doAnyParallelOverscanCrosstalk
509 inputMap = {'dnlLUT': self.config.doDiffNonLinearCorrection,
510 'bias': self.config.doBias,
511 'deferredChargeCalib': self.config.doDeferredCharge,
512 'linearizer': self.config.doLinearize,
513 'ptc': self.config.doApplyGains,
514 'crosstalk': doCrosstalk,
515 'defects': self.config.doDefect,
516 'bfKernel': self.config.doBrighterFatter,
517 'dark': self.config.doDark,
518 }
520 for calibrationFile, configValue in inputMap.items():
521 if configValue and inputs[calibrationFile] is None:
522 raise RuntimeError("Must supply ", calibrationFile)
524 def diffNonLinearCorrection(self, ccdExposure, dnlLUT, **kwargs):
525 # TODO DM 36636
526 # isrFunctions.diffNonLinearCorrection
527 pass
529 def maskFullDefectAmplifiers(self, ccdExposure, detector, defects):
530 """
531 Check for fully masked bad amplifiers and mask them.
533 Full defect masking happens later to allow for defects which
534 cross amplifier boundaries.
536 Parameters
537 ----------
538 ccdExposure : `lsst.afw.image.Exposure`
539 Input exposure to be masked.
540 detector : `lsst.afw.cameraGeom.Detector`
541 Detector object.
542 defects : `lsst.ip.isr.Defects`
543 List of defects. Used to determine if an entire
544 amplifier is bad.
546 Returns
547 -------
548 badAmpDict : `str`[`bool`]
549 Dictionary of amplifiers, keyed by name, value is True if
550 amplifier is fully masked.
551 """
552 badAmpDict = {}
554 maskedImage = ccdExposure.getMaskedImage()
556 for amp in detector:
557 ampName = amp.getName()
558 badAmpDict[ampName] = False
560 # Check if entire amp region is defined as a defect
561 # NB: need to use amp.getBBox() for correct comparison with current
562 # defects definition.
563 if defects is not None:
564 badAmpDict[ampName] = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
566 # In the case of a bad amp, we will set mask to "BAD"
567 # (here use amp.getRawBBox() for correct association with pixels in
568 # current ccdExposure).
569 if badAmpDict[ampName]:
570 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
571 afwImage.PARENT)
572 maskView = dataView.getMask()
573 maskView |= maskView.getPlaneBitMask("BAD")
574 del maskView
576 self.log.warning("Amplifier %s is bad (completely covered with defects)", ampName)
578 return badAmpDict
580 def maskSaturatedPixels(self, badAmpDict, ccdExposure, detector):
581 """
582 Mask SATURATED and SUSPECT pixels and check if any amplifiers
583 are fully masked.
585 Parameters
586 ----------
587 badAmpDict : `str` [`bool`]
588 Dictionary of amplifiers, keyed by name, value is True if
589 amplifier is fully masked.
590 ccdExposure : `lsst.afw.image.Exposure`
591 Input exposure to be masked.
592 detector : `lsst.afw.cameraGeom.Detector`
593 Detector object.
594 defects : `lsst.ip.isr.Defects`
595 List of defects. Used to determine if an entire
596 amplifier is bad.
598 Returns
599 -------
600 badAmpDict : `str`[`bool`]
601 Dictionary of amplifiers, keyed by name.
602 """
603 maskedImage = ccdExposure.getMaskedImage()
605 for amp in detector:
606 ampName = amp.getName()
608 if badAmpDict[ampName]:
609 # No need to check fully bad amplifiers.
610 continue
612 # Mask saturated and suspect pixels.
613 limits = {}
614 if self.config.doSaturation:
615 # Set to the default from the camera model.
616 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
617 # And update if it is set in the config.
618 if math.isfinite(self.config.saturation):
619 limits.update({self.config.saturatedMaskName: self.config.saturation})
620 if self.config.doSuspect:
621 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
623 for maskName, maskThreshold in limits.items():
624 if not math.isnan(maskThreshold):
625 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
626 isrFunctions.makeThresholdMask(
627 maskedImage=dataView,
628 threshold=maskThreshold,
629 growFootprints=0,
630 maskName=maskName
631 )
633 # Determine if we've fully masked this amplifier with SUSPECT and
634 # SAT pixels.
635 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
636 afwImage.PARENT)
637 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
638 self.config.suspectMaskName])
639 if numpy.all(maskView.getArray() & maskVal > 0):
640 self.log.warning("Amplifier %s is bad (completely SATURATED or SUSPECT)", ampName)
641 badAmpDict[ampName] = True
642 maskView |= maskView.getPlaneBitMask("BAD")
644 return badAmpDict
646 def overscanCorrection(self, mode, detectorConfig, detector, badAmpDict, ccdExposure):
647 """Apply serial overscan correction in place to all amps.
649 The actual overscan subtraction is performed by the
650 `lsst.ip.isr.overscan.OverscanTask`, which is called here.
652 Parameters
653 ----------
654 mode : `str`
655 Must be `SERIAL` or `PARALLEL`.
656 detectorConfig : `lsst.ip.isr.OverscanDetectorConfig`
657 Per-amplifier configurations.
658 detector : `lsst.afw.cameraGeom.Detector`
659 Detector object.
660 badAmpDict : `dict`
661 Dictionary of amp name to whether it is a bad amp.
662 ccdExposure : `lsst.afw.image.Exposure`
663 Exposure to have overscan correction performed.
665 Returns
666 -------
667 overscans : `list` [`lsst.pipe.base.Struct` or None]
668 Each result struct has components:
670 ``imageFit``
671 Value or fit subtracted from the amplifier image data.
672 (scalar or `lsst.afw.image.Image`)
673 ``overscanFit``
674 Value or fit subtracted from the overscan image data.
675 (scalar or `lsst.afw.image.Image`)
676 ``overscanImage``
677 Image of the overscan region with the overscan
678 correction applied. This quantity is used to estimate
679 the amplifier read noise empirically.
680 (`lsst.afw.image.Image`)
681 ``overscanMean``
682 Mean overscan fit value. (`float`)
683 ``overscanMedian``
684 Median overscan fit value. (`float`)
685 ``overscanSigma``
686 Clipped standard deviation of the overscan fit. (`float`)
687 ``residualMean``
688 Mean of the overscan after fit subtraction. (`float`)
689 ``residualMedian``
690 Median of the overscan after fit subtraction. (`float`)
691 ``residualSigma``
692 Clipped standard deviation of the overscan after fit
693 subtraction. (`float`)
695 See Also
696 --------
697 lsst.ip.isr.overscan.OverscanTask
698 """
699 if mode not in ["SERIAL", "PARALLEL"]:
700 raise ValueError("Mode must be SERIAL or PARALLEL")
702 # This returns a list in amp order, with None for uncorrected amps.
703 overscans = []
705 for i, amp in enumerate(detector):
706 ampName = amp.getName()
708 ampConfig = detectorConfig.getOverscanAmpConfig(ampName)
710 if mode == "SERIAL" and not ampConfig.doSerialOverscan:
711 self.log.debug(
712 "ISR_OSCAN: Amplifier %s/%s configured to skip serial overscan.",
713 detector.getName(),
714 ampName,
715 )
716 results = None
717 elif mode == "PARALLEL" and not ampConfig.doParallelOverscan:
718 self.log.debug(
719 "ISR_OSCAN: Amplifier %s configured to skip parallel overscan.",
720 detector.getName(),
721 ampName,
722 )
723 results = None
724 elif badAmpDict[ampName] or not ccdExposure.getBBox().contains(amp.getBBox()):
725 results = None
726 else:
727 # This check is to confirm that we are not trying to run
728 # overscan on an already trimmed image. Therefore, always
729 # checking just the horizontal overscan bounding box is
730 # sufficient.
731 if amp.getRawHorizontalOverscanBBox().isEmpty():
732 self.log.warning(
733 "ISR_OSCAN: No overscan region for amp %s. Not performing overscan correction.",
734 ampName,
735 )
736 results = None
737 else:
738 if mode == "SERIAL":
739 # We need to set up the subtask here with a custom
740 # configuration.
741 serialOverscan = SerialOverscanCorrectionTask(config=ampConfig.serialOverscanConfig)
742 results = serialOverscan.run(ccdExposure, amp)
743 else:
744 parallelOverscan = ParallelOverscanCorrectionTask(
745 config=ampConfig.parallelOverscanConfig,
746 )
747 results = parallelOverscan.run(ccdExposure, amp)
749 metadata = ccdExposure.getMetadata()
750 keyBase = "LSST ISR OVERSCAN"
751 metadata[f"{keyBase} {mode} MEAN {ampName}"] = results.overscanMean
752 metadata[f"{keyBase} {mode} MEDIAN {ampName}"] = results.overscanMedian
753 metadata[f"{keyBase} {mode} STDEV {ampName}"] = results.overscanSigma
755 metadata[f"{keyBase} RESIDUAL {mode} MEAN {ampName}"] = results.residualMean
756 metadata[f"{keyBase} RESIDUAL {mode} MEDIAN {ampName}"] = results.residualMedian
757 metadata[f"{keyBase} RESIDUAL {mode} STDEV {ampName}"] = results.residualSigma
759 overscans[i] = results
761 # Question: should this be finer grained?
762 ccdExposure.getMetadata().set("OVERSCAN", "Overscan corrected")
764 return overscans
766 def getLinearizer(self, detector):
767 # Here we assume linearizer as dict or LUT are not supported
768 # TODO DM 28741
770 # TODO construct isrcalib input
771 linearizer = linearize.Linearizer(detector=detector, log=self.log)
772 self.log.warning("Constructing linearizer from cameraGeom information.")
774 return linearizer
776 def gainsCorrection(self, **kwargs):
777 # TODO DM 36639
778 gains = []
779 readNoise = []
781 return gains, readNoise
783 def updateVariance(self, ampExposure, amp, ptcDataset=None):
784 """Set the variance plane using the gain and read noise.
786 Parameters
787 ----------
788 ampExposure : `lsst.afw.image.Exposure`
789 Exposure to process.
790 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp`
791 Amplifier detector data.
792 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
793 PTC dataset containing the gains and read noise.
795 Raises
796 ------
797 RuntimeError
798 Raised if ptcDataset is not provided.
800 See also
801 --------
802 lsst.ip.isr.isrFunctions.updateVariance
803 """
804 # Get gains from PTC
805 if ptcDataset is None:
806 raise RuntimeError("No ptcDataset provided to use PTC gains.")
807 else:
808 gain = ptcDataset.gain[amp.getName()]
809 self.log.debug("Getting gain from Photon Transfer Curve.")
811 if math.isnan(gain):
812 gain = 1.0
813 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
814 elif gain <= 0:
815 patchedGain = 1.0
816 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
817 amp.getName(), gain, patchedGain)
818 gain = patchedGain
820 # Get read noise from PTC
821 if ptcDataset is None:
822 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
823 else:
824 readNoise = ptcDataset.noise[amp.getName()]
825 self.log.debug("Getting read noise from Photon Transfer Curve.")
827 metadata = ampExposure.getMetadata()
828 metadata[f'LSST GAIN {amp.getName()}'] = gain
829 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise
831 isrFunctions.updateVariance(
832 maskedImage=ampExposure.getMaskedImage(),
833 gain=gain,
834 readNoise=readNoise,
835 )
837 def maskNegativeVariance(self, exposure):
838 """Identify and mask pixels with negative variance values.
840 Parameters
841 ----------
842 exposure : `lsst.afw.image.Exposure`
843 Exposure to process.
845 See Also
846 --------
847 lsst.ip.isr.isrFunctions.updateVariance
848 """
849 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
850 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
851 exposure.mask.array[bad] |= maskPlane
853 def variancePlane(self, ccdExposure, ccd, ptc):
854 for amp in ccd:
855 if ccdExposure.getBBox().contains(amp.getBBox()):
856 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
857 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
859 self.updateVariance(ampExposure, amp, ptcDataset=ptc)
861 if self.config.qa is not None and self.config.qa.saveStats is True:
862 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
863 afwMath.MEDIAN | afwMath.STDEVCLIP)
864 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
865 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
866 qaStats.getValue(afwMath.STDEVCLIP))
867 if self.config.maskNegativeVariance:
868 self.maskNegativeVariance(ccdExposure)
870 def maskDefect(self, exposure, defectBaseList):
871 """Mask defects using mask plane "BAD", in place.
873 Parameters
874 ----------
875 exposure : `lsst.afw.image.Exposure`
876 Exposure to process.
878 defectBaseList : defect-type
879 List of defects to mask. Can be of type `lsst.ip.isr.Defects`
880 or `list` of `lsst.afw.image.DefectBase`.
881 """
882 maskedImage = exposure.getMaskedImage()
883 if not isinstance(defectBaseList, Defects):
884 # Promotes DefectBase to Defect
885 defectList = Defects(defectBaseList)
886 else:
887 defectList = defectBaseList
888 defectList.maskPixels(maskedImage, maskName="BAD")
890 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
891 """Mask edge pixels with applicable mask plane.
893 Parameters
894 ----------
895 exposure : `lsst.afw.image.Exposure`
896 Exposure to process.
897 numEdgePixels : `int`, optional
898 Number of edge pixels to mask.
899 maskPlane : `str`, optional
900 Mask plane name to use.
901 level : `str`, optional
902 Level at which to mask edges.
903 """
904 maskedImage = exposure.getMaskedImage()
905 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
907 if numEdgePixels > 0:
908 if level == 'DETECTOR':
909 boxes = [maskedImage.getBBox()]
910 elif level == 'AMP':
911 boxes = [amp.getBBox() for amp in exposure.getDetector()]
913 for box in boxes:
914 # This makes a bbox numEdgeSuspect pixels smaller than the
915 # image on each side
916 subImage = maskedImage[box]
917 box.grow(-numEdgePixels)
918 # Mask pixels outside box
919 SourceDetectionTask.setEdgeBits(
920 subImage,
921 box,
922 maskBitMask)
924 def maskNan(self, exposure):
925 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
927 Parameters
928 ----------
929 exposure : `lsst.afw.image.Exposure`
930 Exposure to process.
932 Notes
933 -----
934 We mask over all non-finite values (NaN, inf), including those
935 that are masked with other bits (because those may or may not be
936 interpolated over later, and we want to remove all NaN/infs).
937 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
938 preserve the historical name.
939 """
940 maskedImage = exposure.getMaskedImage()
942 # Find and mask NaNs
943 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
944 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
945 numNans = maskNans(maskedImage, maskVal)
946 self.metadata["NUMNANS"] = numNans
947 if numNans > 0:
948 self.log.warning("There were %d unmasked NaNs.", numNans)
950 def countBadPixels(self, exposure):
951 """
952 Notes
953 -----
954 Reset and interpolate bad pixels.
956 Large contiguous bad regions (which should have the BAD mask
957 bit set) should have their values set to the image median.
958 This group should include defects and bad amplifiers. As the
959 area covered by these defects are large, there's little
960 reason to expect that interpolation would provide a more
961 useful value.
963 Smaller defects can be safely interpolated after the larger
964 regions have had their pixel values reset. This ensures
965 that the remaining defects adjacent to bad amplifiers (as an
966 example) do not attempt to interpolate extreme values.
967 """
968 badPixelCount, badPixelValue = isrFunctions.setBadRegions(exposure)
969 if badPixelCount > 0:
970 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
972 @contextmanager
973 def flatContext(self, exp, flat, dark=None):
974 """Context manager that applies and removes flats and darks,
975 if the task is configured to apply them.
977 Parameters
978 ----------
979 exp : `lsst.afw.image.Exposure`
980 Exposure to process.
981 flat : `lsst.afw.image.Exposure`
982 Flat exposure the same size as ``exp``.
983 dark : `lsst.afw.image.Exposure`, optional
984 Dark exposure the same size as ``exp``.
986 Yields
987 ------
988 exp : `lsst.afw.image.Exposure`
989 The flat and dark corrected exposure.
990 """
991 if self.config.doDark and dark is not None:
992 self.darkCorrection(exp, dark)
993 if self.config.doFlat:
994 self.flatCorrection(exp, flat)
995 try:
996 yield exp
997 finally:
998 if self.config.doFlat:
999 self.flatCorrection(exp, flat, invert=True)
1000 if self.config.doDark and dark is not None:
1001 self.darkCorrection(exp, dark, invert=True)
1003 def getBrighterFatterKernel(self, detector, bfKernel):
1004 detName = detector.getName()
1006 # This is expected to be a dictionary of amp-wise gains.
1007 bfGains = bfKernel.gain
1008 if bfKernel.level == 'DETECTOR':
1009 if detName in bfKernel.detKernels:
1010 bfKernelOut = bfKernel.detKernels[detName]
1011 return bfKernelOut, bfGains
1012 else:
1013 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1014 elif bfKernel.level == 'AMP':
1015 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1016 "fatter kernels.")
1017 bfKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1018 bfKernelOut = bfKernel.detKernels[detName]
1019 return bfKernelOut, bfGains
1021 def applyBrighterFatterCorrection(self, ccdExposure, flat, dark, bfKernel, bfGains):
1022 # We need to apply flats and darks before we can interpolate, and
1023 # we need to interpolate before we do B-F, but we do B-F without
1024 # the flats and darks applied so we can work in units of electrons
1025 # or holes. This context manager applies and then removes the darks
1026 # and flats.
1027 #
1028 # We also do not want to interpolate values here, so operate on
1029 # temporary images so we can apply only the BF-correction and roll
1030 # back the interpolation.
1031 # This won't be necessary once the gain normalization
1032 # is done appropriately.
1033 interpExp = ccdExposure.clone()
1034 with self.flatContext(interpExp, flat, dark):
1035 isrFunctions.interpolateFromMask(
1036 maskedImage=interpExp.getMaskedImage(),
1037 fwhm=self.config.brighterFatterFwhmForInterpolation,
1038 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1039 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1040 )
1041 bfExp = interpExp.clone()
1042 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1043 type(bfKernel), type(bfGains))
1044 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1045 self.config.brighterFatterMaxIter,
1046 self.config.brighterFatterThreshold,
1047 self.config.brighterFatterApplyGain,
1048 bfGains)
1049 if bfResults[1] == self.config.brighterFatterMaxIter:
1050 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1051 bfResults[0])
1052 else:
1053 self.log.info("Finished brighter-fatter correction in %d iterations.",
1054 bfResults[1])
1056 image = ccdExposure.getMaskedImage().getImage()
1057 bfCorr = bfExp.getMaskedImage().getImage()
1058 bfCorr -= interpExp.getMaskedImage().getImage()
1059 image += bfCorr
1061 # Applying the brighter-fatter correction applies a
1062 # convolution to the science image. At the edges this
1063 # convolution may not have sufficient valid pixels to
1064 # produce a valid correction. Mark pixels within the size
1065 # of the brighter-fatter kernel as EDGE to warn of this
1066 # fact.
1067 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1068 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1069 maskPlane="EDGE")
1071 if self.config.brighterFatterMaskGrowSize > 0:
1072 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1073 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1074 isrFunctions.growMasks(ccdExposure.getMask(),
1075 radius=self.config.brighterFatterMaskGrowSize,
1076 maskNameList=maskPlane,
1077 maskValue=maskPlane)
1079 return ccdExposure
1081 def darkCorrection(self, exposure, darkExposure, invert=False):
1082 """Apply dark correction in place.
1084 Parameters
1085 ----------
1086 exposure : `lsst.afw.image.Exposure`
1087 Exposure to process.
1088 darkExposure : `lsst.afw.image.Exposure`
1089 Dark exposure of the same size as ``exposure``.
1090 invert : `Bool`, optional
1091 If True, re-add the dark to an already corrected image.
1093 Raises
1094 ------
1095 RuntimeError
1096 Raised if either ``exposure`` or ``darkExposure`` do not
1097 have their dark time defined.
1099 See Also
1100 --------
1101 lsst.ip.isr.isrFunctions.darkCorrection
1102 """
1103 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1104 if math.isnan(expScale):
1105 raise RuntimeError("Exposure darktime is NAN.")
1106 if darkExposure.getInfo().getVisitInfo() is not None \
1107 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
1108 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1109 else:
1110 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1111 # so getDarkTime() does not exist.
1112 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1113 darkScale = 1.0
1115 isrFunctions.darkCorrection(
1116 maskedImage=exposure.getMaskedImage(),
1117 darkMaskedImage=darkExposure.getMaskedImage(),
1118 expScale=expScale,
1119 darkScale=darkScale,
1120 invert=invert,
1121 )
1123 @staticmethod
1124 def extractCalibDate(calib):
1125 """Extract common calibration metadata values that will be written to
1126 output header.
1128 Parameters
1129 ----------
1130 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib`
1131 Calibration to pull date information from.
1133 Returns
1134 -------
1135 dateString : `str`
1136 Calibration creation date string to add to header.
1137 """
1138 if hasattr(calib, "getMetadata"):
1139 if 'CALIB_CREATION_DATE' in calib.getMetadata():
1140 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"),
1141 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown")))
1142 else:
1143 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"),
1144 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown")))
1145 else:
1146 return "Unknown Unknown"
1148 def doLinearize(self, detector):
1149 """Check if linearization is needed for the detector cameraGeom.
1151 Checks config.doLinearize and the linearity type of the first
1152 amplifier.
1154 Parameters
1155 ----------
1156 detector : `lsst.afw.cameraGeom.Detector`
1157 Detector to get linearity type from.
1159 Returns
1160 -------
1161 doLinearize : `Bool`
1162 If True, linearization should be performed.
1163 """
1164 return self.config.doLinearize and \
1165 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
1167 def flatCorrection(self, exposure, flatExposure, invert=False):
1168 """Apply flat correction in place.
1170 Parameters
1171 ----------
1172 exposure : `lsst.afw.image.Exposure`
1173 Exposure to process.
1174 flatExposure : `lsst.afw.image.Exposure`
1175 Flat exposure of the same size as ``exposure``.
1176 invert : `Bool`, optional
1177 If True, unflatten an already flattened image.
1179 See Also
1180 --------
1181 lsst.ip.isr.isrFunctions.flatCorrection
1182 """
1183 isrFunctions.flatCorrection(
1184 maskedImage=exposure.getMaskedImage(),
1185 flatMaskedImage=flatExposure.getMaskedImage(),
1186 scalingType=self.config.flatScalingType,
1187 userScale=self.config.flatUserScale,
1188 invert=invert
1189 )
1191 def makeBinnedImages(self, exposure):
1192 """Make visualizeVisit style binned exposures.
1194 Parameters
1195 ----------
1196 exposure : `lsst.afw.image.Exposure`
1197 Exposure to bin.
1199 Returns
1200 -------
1201 bin1 : `lsst.afw.image.Exposure`
1202 Binned exposure using binFactor1.
1203 bin2 : `lsst.afw.image.Exposure`
1204 Binned exposure using binFactor2.
1205 """
1206 mi = exposure.getMaskedImage()
1208 bin1 = afwMath.binImage(mi, self.config.binFactor1)
1209 bin2 = afwMath.binImage(mi, self.config.binFactor2)
1211 return bin1, bin2
1213 def run(self, *, ccdExposure, dnlLUT=None, bias=None, deferredChargeCalib=None, linearizer=None,
1214 ptc=None, crosstalk=None, defects=None, bfKernel=None, bfGains=None, dark=None,
1215 flat=None, camera=None, **kwargs
1216 ):
1218 detector = ccdExposure.getDetector()
1220 overscanDetectorConfig = self.config.overscanCamera.getOverscanDetectorConfig(detector)
1222 gains = ptc.gain
1224 if self.config.doHeaderProvenance:
1225 # Inputs have been validated, so we can add their date
1226 # information to the output header.
1227 exposureMetadata = ccdExposure.getMetadata()
1228 exposureMetadata["LSST CALIB OVERSCAN HASH"] = overscanDetectorConfig.md5
1229 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc)
1230 if self.config.doDiffNonLinearCorrection:
1231 exposureMetadata["LSST CALIB DATE DNL"] = self.extractCalibDate(dnlLUT)
1232 if self.config.doBias:
1233 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias)
1234 if self.config.doDeferredCharge:
1235 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib)
1236 if self.doLinearize(detector):
1237 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer)
1238 if self.config.doCrosstalk or overscanDetectorConfig.doAnyParallelOverscanCrosstalk:
1239 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk)
1240 if self.config.doDefect:
1241 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects)
1242 if self.config.doBrighterFatter:
1243 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel)
1244 if self.config.doDark:
1245 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark)
1246 if self.config.doFlat:
1247 exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat)
1249 # First we mark which amplifiers are completely bad from defects.
1250 badAmpDict = self.maskFullDefectAmplifiers(ccdExposure, detector, defects)
1252 if self.config.doDiffNonLinearCorrection:
1253 self.diffNonLinearCorrection(ccdExposure, dnlLUT)
1255 if overscanDetectorConfig.doAnySerialOverscan:
1256 # Input units: ADU
1257 serialOverscans = self.overscanCorrection(
1258 "SERIAL",
1259 overscanDetectorConfig,
1260 detector,
1261 badAmpDict,
1262 ccdExposure,
1263 )
1264 else:
1265 serialOverscans = [None]*len(detector)
1267 if overscanDetectorConfig.doAnyParallelOverscanCrosstalk:
1268 # Input units: ADU
1269 # Make sure that the units here are consistent with later
1270 # application.
1271 self.crosstalk.run(
1272 ccdExposure,
1273 crosstalk=crosstalk,
1274 camera=camera,
1275 parallelOverscanRegion=True,
1276 detectorConfig=overscanDetectorConfig,
1277 )
1279 # After serial overscan correction, we can mask SATURATED and
1280 # SUSPECT pixels. This updates badAmpDict if any amplifier
1281 # is fully saturated after serial overscan correction.
1282 badAmpDict = self.maskSaturatedPixels(badAmpDict, ccdExposure, detector)
1284 if overscanDetectorConfig.doAnyParallelOverscan:
1285 # Input units: ADU
1286 # At the moment we do not use the parallelOverscans return value.
1287 _ = self.overscanCorrection(
1288 "PARALLEL",
1289 overscanDetectorConfig,
1290 detector,
1291 badAmpDict,
1292 ccdExposure,
1293 )
1295 if self.config.doAssembleCcd:
1296 # Input units: ADU
1297 self.log.info("Assembling CCD from amplifiers.")
1298 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1300 if self.config.expectWcs and not ccdExposure.getWcs():
1301 self.log.warning("No WCS found in input exposure.")
1303 if self.config.doLinearize:
1304 # Input units: ADU
1305 self.log.info("Applying linearizer.")
1306 linearizer = self.getLinearizer(detector=detector)
1307 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1308 detector=detector, log=self.log)
1310 if self.config.doCrosstalk:
1311 # Input units: ADU
1312 self.log.info("Applying crosstalk correction.")
1313 self.crosstalk.run(ccdExposure, crosstalk=crosstalk)
1315 if self.config.doBias:
1316 # Input units: ADU
1317 self.log.info("Applying bias correction.")
1318 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage())
1320 if self.config.doGainsCorrection:
1321 # TODO DM 36639
1322 self.log.info("Apply temperature dependence to the gains.")
1323 gains, readNoise = self.gainsCorrection(**kwargs)
1325 if self.config.doApplyGains:
1326 # Input units: ADU
1327 # Output units: electrons
1328 self.log.info("Apply PTC gains (temperature corrected or not) to the image.")
1329 isrFunctions.applyGains(ccdExposure, normalizeGains=False, ptcGains=gains)
1331 if self.config.doDeferredCharge:
1332 # Input units: electrons
1333 self.log.info("Applying deferred charge/CTI correction.")
1334 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib)
1336 if self.config.doVariance:
1337 # Input units: electrons
1338 self.variancePlane(ccdExposure, detector, ptc)
1340 # Masking block (defects, NAN pixels and trails).
1341 # Saturated and suspect pixels have already been masked.
1342 if self.config.doDefect:
1343 # Input units: electrons
1344 self.log.info("Applying defects masking.")
1345 self.maskDefect(ccdExposure, defects)
1347 if self.config.doNanMasking:
1348 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1349 self.maskNan(ccdExposure)
1351 if self.config.doWidenSaturationTrails:
1352 self.log.info("Widening saturation trails.")
1353 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1355 preInterpExp = None
1356 if self.config.doSaveInterpPixels:
1357 preInterpExp = ccdExposure.clone()
1359 if self.config.doSetBadRegions:
1360 self.log.info('Counting pixels in BAD regions.')
1361 self.countBadPixels(ccdExposure)
1363 if self.config.doInterpolate:
1364 self.log.info("Interpolating masked pixels.")
1365 isrFunctions.interpolateFromMask(
1366 maskedImage=ccdExposure.getMaskedImage(),
1367 fwhm=self.config.brighterFatterFwhmForInterpolation,
1368 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1369 maskNameList=list(self.config.maskListToInterpolate)
1370 )
1372 if self.config.doDark:
1373 # Input units: electrons
1374 self.log.info("Applying dark subtraction.")
1375 self.darkCorrection(ccdExposure, dark)
1377 if self.config.doBrighterFatter:
1378 # Input units: electrons
1379 self.log.info("Applying Bright-Fatter kernels.")
1380 bfKernelOut, bfGains = self.getBrighterFatterKernel(detector, bfKernel)
1381 ccdExposure = self.applyBrighterFatterCorrection(ccdExposure, flat, dark, bfKernelOut, bfGains)
1383 if self.config.doFlat:
1384 # Input units: electrons
1385 self.log.info("Applying flat correction.")
1386 # Placeholder while the LSST flat procedure is done.
1387 # The flat here would be a background flat.
1388 self.flatCorrection(ccdExposure, flat)
1390 # Calculate standard image quality statistics
1391 if self.config.doStandardStatistics:
1392 metadata = ccdExposure.getMetadata()
1393 for amp in detector:
1394 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1395 ampName = amp.getName()
1396 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels(
1397 ampExposure.getMaskedImage(),
1398 [self.config.saturatedMaskName]
1399 )
1400 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels(
1401 ampExposure.getMaskedImage(),
1402 ["BAD"]
1403 )
1404 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1405 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP)
1407 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN)
1408 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN)
1409 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP)
1411 k1 = f"LSST ISR FINAL MEDIAN {ampName}"
1412 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}"
1413 if overscanDetectorConfig.doAnySerialOverscan and k1 in metadata and k2 in metadata:
1414 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2]
1415 else:
1416 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan
1418 # calculate additional statistics.
1419 outputStatistics = None
1420 if self.config.doCalculateStatistics:
1421 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=serialOverscans,
1422 bias=bias, dark=dark, flat=flat, ptc=ptc,
1423 defects=defects).results
1425 # do image binning.
1426 outputBin1Exposure = None
1427 outputBin2Exposure = None
1428 if self.config.doBinnedExposures:
1429 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure)
1431 return pipeBase.Struct(
1432 exposure=ccdExposure,
1434 outputBin1Exposure=outputBin1Exposure,
1435 outputBin2Exposure=outputBin2Exposure,
1437 preInterpExposure=preInterpExp,
1438 outputExposure=ccdExposure,
1439 outputStatistics=outputStatistics,
1440 )