lsst.ip.isr ged8ae655b3+88b164628e
Loading...
Searching...
No Matches
isrTaskLSST.py
Go to the documentation of this file.
1__all__ = ["IsrTaskLSST", "IsrTaskLSSTConfig"]
2
3import numpy
4import math
5
6from . import isrFunctions
7from . import isrQa
8from . import linearize
9from .defects import Defects
10
11from contextlib import contextmanager
12from lsst.afw.cameraGeom import NullLinearityType
13import lsst.pex.config as pexConfig
14import lsst.afw.math as afwMath
15import lsst.pipe.base as pipeBase
16import lsst.pipe.base.connectionTypes as cT
17from lsst.meas.algorithms.detection import SourceDetectionTask
18
19from .overscan import OverscanCorrectionTask
20from .assembleCcdTask import AssembleCcdTask
21from .deferredCharge import DeferredChargeTask
22from .crosstalk import CrosstalkTask
23from .masking import MaskingTask
24from .isrStatistics import IsrStatisticsTask
25from .isr import maskNans
26
27
28class IsrTaskLSSTConnections(pipeBase.PipelineTaskConnections,
29 dimensions={"instrument", "exposure", "detector"},
30 defaultTemplates={}):
31 ccdExposure = cT.Input(
32 name="raw",
33 doc="Input exposure to process.",
34 storageClass="Exposure",
35 dimensions=["instrument", "exposure", "detector"],
36 )
37 camera = cT.PrerequisiteInput(
38 name="camera",
39 storageClass="Camera",
40 doc="Input camera to construct complete exposures.",
41 dimensions=["instrument"],
42 isCalibration=True,
43 )
44 dnlLUT = cT.PrerequisiteInput(
45 name="dnlLUT",
46 doc="Look-up table for differential non-linearity.",
47 storageClass="IsrCalib",
48 dimensions=["instrument", "exposure", "detector"],
49 isCalibration=True,
50 # TODO DM 36636
51 )
52 bias = cT.PrerequisiteInput(
53 name="bias",
54 doc="Input bias calibration.",
55 storageClass="ExposureF",
56 dimensions=["instrument", "detector"],
57 isCalibration=True,
58 )
59 deferredChargeCalib = cT.PrerequisiteInput(
60 name="cpCtiCalib",
61 doc="Deferred charge/CTI correction dataset.",
62 storageClass="IsrCalib",
63 dimensions=["instrument", "detector"],
64 isCalibration=True,
65 )
66 linearizer = cT.PrerequisiteInput(
67 name='linearizer',
68 storageClass="Linearizer",
69 doc="Linearity correction calibration.",
70 dimensions=["instrument", "detector"],
71 isCalibration=True,
72 )
73 ptc = cT.PrerequisiteInput(
74 name="ptc",
75 doc="Input Photon Transfer Curve dataset",
76 storageClass="PhotonTransferCurveDataset",
77 dimensions=["instrument", "detector"],
78 isCalibration=True,
79 )
80 crosstalk = cT.PrerequisiteInput(
81 name="crosstalk",
82 doc="Input crosstalk object",
83 storageClass="CrosstalkCalib",
84 dimensions=["instrument", "detector"],
85 isCalibration=True,
86 )
87 defects = cT.PrerequisiteInput(
88 name='defects',
89 doc="Input defect tables.",
90 storageClass="Defects",
91 dimensions=["instrument", "detector"],
92 isCalibration=True,
93 )
94 bfKernel = cT.PrerequisiteInput(
95 name='brighterFatterKernel',
96 doc="Complete kernel + gain solutions.",
97 storageClass="BrighterFatterKernel",
98 dimensions=["instrument", "detector"],
99 isCalibration=True,
100 )
101 dark = cT.PrerequisiteInput(
102 name='dark',
103 doc="Input dark calibration.",
104 storageClass="ExposureF",
105 dimensions=["instrument", "detector"],
106 isCalibration=True,
107 )
108 outputExposure = cT.Output(
109 name='postISRCCD',
110 doc="Output ISR processed exposure.",
111 storageClass="Exposure",
112 dimensions=["instrument", "exposure", "detector"],
113 )
114 preInterpExposure = cT.Output(
115 name='preInterpISRCCD',
116 doc="Output ISR processed exposure, with pixels left uninterpolated.",
117 storageClass="ExposureF",
118 dimensions=["instrument", "exposure", "detector"],
119 )
120 outputBin1Exposure = cT.Output(
121 name="postIsrBin1",
122 doc="First binned image.",
123 storageClass="ExposureF",
124 dimensions=["instrument", "exposure", "detector"],
125 )
126 outputBin2Exposure = cT.Output(
127 name="postIsrBin2",
128 doc="Second binned image.",
129 storageClass="ExposureF",
130 dimensions=["instrument", "exposure", "detector"],
131 )
132
133 outputStatistics = cT.Output(
134 name="isrStatistics",
135 doc="Output of additional statistics table.",
136 storageClass="StructuredDataDict",
137 dimensions=["instrument", "exposure", "detector"],
138 )
139
140 def __init__(self, *, config=None):
141 super().__init__(config=config)
142
143 if config.doDiffNonLinearCorrection is not True:
144 del self.dnlLUT
145 if config.doBias is not True:
146 del self.bias
147 if config.doDeferredCharge is not True:
148 del self.deferredChargeCalib
149 if config.doLinearize is not True:
150 del self.linearizer
151 if config.doCrosstalk is not True:
152 del self.crosstalk
153 if config.doDefect is not True:
154 del self.defects
155 if config.doBrighterFatter is not True:
156 del self.bfKernel
157 if config.doDark is not True:
158 del self.dark
159
160 if config.doBinnedExposures is not True:
161 del self.outputBin1Exposure
162 del self.outputBin2Exposure
163 if config.doSaveInterpPixels is not True:
164 del self.preInterpExposure
165
166 if config.doCalculateStatistics is not True:
167 del self.outputStatistics
168
169
170class IsrTaskLSSTConfig(pipeBase.PipelineTaskConfig,
171 pipelineConnections=IsrTaskLSSTConnections):
172 """Configuration parameters for IsrTaskLSST.
173
174 Items are grouped in the order in which they are executed by the task.
175 """
176 expectWcs = pexConfig.Field(
177 dtype=bool,
178 default=True,
179 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
180 )
181 qa = pexConfig.ConfigField(
182 dtype=isrQa.IsrQaConfig,
183 doc="QA related configuration options.",
184 )
185 doHeaderProvenance = pexConfig.Field(
186 dtype=bool,
187 default=True,
188 doc="Write calibration identifiers into output exposure header.",
189 )
190
191 # Differential non-linearity correction.
192 doDiffNonLinearCorrection = pexConfig.Field(
193 dtype=bool,
194 doc="Do differential non-linearity correction?",
195 default=True,
196 )
197
198 doOverscan = pexConfig.Field(
199 dtype=bool,
200 doc="Do overscan subtraction?",
201 default=True,
202 )
203 overscan = pexConfig.ConfigurableField(
204 target=OverscanCorrectionTask,
205 doc="Overscan subtraction task for image segments.",
206 )
207
208 # Amplifier to CCD assembly configuration.
209 doAssembleCcd = pexConfig.Field(
210 dtype=bool,
211 default=True,
212 doc="Assemble amp-level exposures into a ccd-level exposure?"
213 )
214 assembleCcd = pexConfig.ConfigurableField(
215 target=AssembleCcdTask,
216 doc="CCD assembly task.",
217 )
218
219 # Bias subtraction.
220 doBias = pexConfig.Field(
221 dtype=bool,
222 doc="Apply bias frame correction?",
223 default=True,
224 )
225
226 # Deferred charge correction.
227 doDeferredCharge = pexConfig.Field(
228 dtype=bool,
229 doc="Apply deferred charge correction?",
230 default=True,
231 )
232 deferredChargeCorrection = pexConfig.ConfigurableField(
233 target=DeferredChargeTask,
234 doc="Deferred charge correction task.",
235 )
236
237 # Linearization.
238 doLinearize = pexConfig.Field(
239 dtype=bool,
240 doc="Correct for nonlinearity of the detector's response?",
241 default=True,
242 )
243
244 # Normalize gain.
245 doGainNormalize = pexConfig.Field(
246 dtype=bool,
247 doc="Normalize by the gain.",
248 default=True,
249 )
250
251 # Variance construction.
252 doVariance = pexConfig.Field(
253 dtype=bool,
254 doc="Calculate variance?",
255 default=True
256 )
257 gain = pexConfig.Field(
258 dtype=float,
259 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN).",
260 default=float("NaN"),
261 )
262 maskNegativeVariance = pexConfig.Field(
263 dtype=bool,
264 doc="Mask pixels that claim a negative variance. This likely indicates a failure "
265 "in the measurement of the overscan at an edge due to the data falling off faster "
266 "than the overscan model can account for it.",
267 default=True,
268 )
269 negativeVarianceMaskName = pexConfig.Field(
270 dtype=str,
271 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
272 default="BAD",
273 )
274 saturatedMaskName = pexConfig.Field(
275 dtype=str,
276 doc="Name of mask plane to use in saturation detection and interpolation.",
277 default="SAT",
278 )
279 suspectMaskName = pexConfig.Field(
280 dtype=str,
281 doc="Name of mask plane to use for suspect pixels.",
282 default="SUSPECT",
283 )
284
285 # Crosstalk.
286 doCrosstalk = pexConfig.Field(
287 dtype=bool,
288 doc="Apply intra-CCD crosstalk correction?",
289 default=True,
290 )
291 crosstalk = pexConfig.ConfigurableField(
292 target=CrosstalkTask,
293 doc="Intra-CCD crosstalk correction.",
294 )
295
296 # Masking options.
297 doDefect = pexConfig.Field(
298 dtype=bool,
299 doc="Apply correction for CCD defects, e.g. hot pixels?",
300 default=True,
301 )
302 doNanMasking = pexConfig.Field(
303 dtype=bool,
304 doc="Mask non-finite (NAN, inf) pixels.",
305 default=True,
306 )
307 doWidenSaturationTrails = pexConfig.Field(
308 dtype=bool,
309 doc="Widen bleed trails based on their width.",
310 default=True,
311 )
312 masking = pexConfig.ConfigurableField(
313 target=MaskingTask,
314 doc="Masking task."
315 )
316
317 # Interpolation options.
318 doInterpolate = pexConfig.Field(
319 dtype=bool,
320 doc="Interpolate masked pixels?",
321 default=True,
322 )
323 maskListToInterpolate = pexConfig.ListField(
324 dtype=str,
325 doc="List of mask planes that should be interpolated.",
326 default=['SAT', 'BAD'],
327 )
328 doSaveInterpPixels = pexConfig.Field(
329 dtype=bool,
330 doc="Save a copy of the pre-interpolated pixel values?",
331 default=False,
332 )
333
334 # Initial masking options.
335 doSetBadRegions = pexConfig.Field(
336 dtype=bool,
337 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
338 default=True,
339 )
340
341 # Brighter-Fatter correction.
342 doBrighterFatter = pexConfig.Field(
343 dtype=bool,
344 doc="Apply the brighter-fatter correction?",
345 default=True,
346 )
347 brighterFatterLevel = pexConfig.ChoiceField(
348 dtype=str,
349 doc="The level at which to correct for brighter-fatter.",
350 allowed={
351 "AMP": "Every amplifier treated separately.",
352 "DETECTOR": "One kernel per detector.",
353 },
354 default="DETECTOR",
355 )
356 brighterFatterMaxIter = pexConfig.Field(
357 dtype=int,
358 doc="Maximum number of iterations for the brighter-fatter correction.",
359 default=10,
360 )
361 brighterFatterThreshold = pexConfig.Field(
362 dtype=float,
363 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
364 "absolute value of the difference between the current corrected image and the one "
365 "from the previous iteration summed over all the pixels.",
366 default=1000,
367 )
368 brighterFatterApplyGain = pexConfig.Field(
369 dtype=bool,
370 doc="Should the gain be applied when applying the brighter-fatter correction?",
371 default=True,
372 )
373 brighterFatterMaskListToInterpolate = pexConfig.ListField(
374 dtype=str,
375 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
376 "correction.",
377 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
378 )
379 brighterFatterMaskGrowSize = pexConfig.Field(
380 dtype=int,
381 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
382 "when brighter-fatter correction is applied.",
383 default=0,
384 )
385 brighterFatterFwhmForInterpolation = pexConfig.Field(
386 dtype=float,
387 doc="FWHM of PSF in arcseconds used for interpolation in brighter-fatter correction "
388 "(currently unused).",
389 default=1.0,
390 )
391 growSaturationFootprintSize = pexConfig.Field(
392 dtype=int,
393 doc="Number of pixels by which to grow the saturation footprints.",
394 default=1,
395 )
396 brighterFatterMaskListToInterpolate = pexConfig.ListField(
397 dtype=str,
398 doc="List of mask planes that should be interpolated over when applying the brighter-fatter."
399 "correction.",
400 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
401 )
402
403 # Dark subtraction.
404 doDark = pexConfig.Field(
405 dtype=bool,
406 doc="Apply dark frame correction.",
407 default=True,
408 )
409
410 # Calculate image quality statistics?
411 doStandardStatistics = pexConfig.Field(
412 dtype=bool,
413 doc="Should standard image quality statistics be calculated?",
414 default=True,
415 )
416 # Calculate additional statistics?
417 doCalculateStatistics = pexConfig.Field(
418 dtype=bool,
419 doc="Should additional ISR statistics be calculated?",
420 default=True,
421 )
422 isrStats = pexConfig.ConfigurableField(
423 target=IsrStatisticsTask,
424 doc="Task to calculate additional statistics.",
425 )
426
427 # Make binned images?
428 doBinnedExposures = pexConfig.Field(
429 dtype=bool,
430 doc="Should binned exposures be calculated?",
431 default=False,
432 )
433 binFactor1 = pexConfig.Field(
434 dtype=int,
435 doc="Binning factor for first binned exposure. This is intended for a finely binned output.",
436 default=8,
437 check=lambda x: x > 1,
438 )
439 binFactor2 = pexConfig.Field(
440 dtype=int,
441 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.",
442 default=64,
443 check=lambda x: x > 1,
444 )
445
446 def validate(self):
447 super().validate()
448
449 if self.doCalculateStatistics and self.isrStats.doCtiStatistics:
450 # DM-41912: Implement doApplyGains in LSST IsrTask
451 # if self.doApplyGains !=
452 # self.isrStats.doApplyGainsForCtiStatistics:
453 raise ValueError("doApplyGains must match isrStats.applyGainForCtiStatistics.")
454
455
456class IsrTaskLSST(pipeBase.PipelineTask):
457 ConfigClass = IsrTaskLSSTConfig
458 _DefaultName = "isr"
459
460 def __init__(self, **kwargs):
461 super().__init__(**kwargs)
462 self.makeSubtask("overscan")
463 self.makeSubtask("assembleCcd")
464 self.makeSubtask("deferredChargeCorrection")
465 self.makeSubtask("crosstalk")
466 self.makeSubtask("masking")
467 self.makeSubtask("isrStats")
468
469 def runQuantum(self, butlerQC, inputRefs, outputRefs):
470
471 inputs = butlerQC.get(inputRefs)
472 self.validateInput(inputs)
473 super().runQuantum(butlerQC, inputRefs, outputRefs)
474
475 def validateInput(self, inputs):
476 """
477 This is a check that all the inputs required by the config
478 are available.
479 """
480
481 inputMap = {'dnlLUT': self.config.doDiffNonLinearCorrection,
482 'bias': self.config.doBias,
483 'deferredChargeCalib': self.config.doDeferredCharge,
484 'linearizer': self.config.doLinearize,
485 'ptc': self.config.doGainNormalize,
486 'crosstalk': self.config.doCrosstalk,
487 'defects': self.config.doDefect,
488 'bfKernel': self.config.doBrighterFatter,
489 'dark': self.config.doDark,
490 }
491
492 for calibrationFile, configValue in inputMap.items():
493 if configValue and inputs[calibrationFile] is None:
494 raise RuntimeError("Must supply ", calibrationFile)
495
496 def diffNonLinearCorrection(self, ccdExposure, dnlLUT, **kwargs):
497 # TODO DM 36636
498 # isrFunctions.diffNonLinearCorrection
499 pass
500
501 def overscanCorrection(self, ccd, ccdExposure):
502 # TODO DM 36637 for per amp
503
504 overscans = []
505 for amp in ccd:
506
507 # Overscan correction on amp-by-amp basis.
508 if amp.getRawHorizontalOverscanBBox().isEmpty():
509 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
510 overscans.append(None)
511 else:
512
513 # Perform overscan correction on subregions.
514 overscanResults = self.overscan.run(ccdExposure, amp)
515
516 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
517 if len(overscans) == 0:
518 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
519
520 overscans.append(overscanResults if overscanResults is not None else None)
521
522 return overscans
523
524 def getLinearizer(self, detector):
525 # Here we assume linearizer as dict or LUT are not supported
526 # TODO DM 28741
527
528 # TODO construct isrcalib input
529 linearizer = linearize.Linearizer(detector=detector, log=self.log)
530 self.log.warning("Constructing linearizer from cameraGeom information.")
531
532 return linearizer
533
534 def gainNormalize(self, **kwargs):
535 # TODO DM 36639
536 gains = []
537 readNoise = []
538
539 return gains, readNoise
540
541 def updateVariance(self, ampExposure, amp, ptcDataset=None):
542 """Set the variance plane using the gain and read noise.
543
544 Parameters
545 ----------
546 ampExposure : `lsst.afw.image.Exposure`
547 Exposure to process.
548 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp`
549 Amplifier detector data.
550 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
551 PTC dataset containing the gains and read noise.
552
553 Raises
554 ------
555 RuntimeError
556 Raised if ptcDataset is not provided.
557
558 See also
559 --------
560 lsst.ip.isr.isrFunctions.updateVariance
561 """
562 # Get gains from PTC
563 if ptcDataset is None:
564 raise RuntimeError("No ptcDataset provided to use PTC gains.")
565 else:
566 gain = ptcDataset.gain[amp.getName()]
567 self.log.debug("Getting gain from Photon Transfer Curve.")
568
569 if math.isnan(gain):
570 gain = 1.0
571 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
572 elif gain <= 0:
573 patchedGain = 1.0
574 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
575 amp.getName(), gain, patchedGain)
576 gain = patchedGain
577
578 # Get read noise from PTC
579 if ptcDataset is None:
580 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
581 else:
582 readNoise = ptcDataset.noise[amp.getName()]
583 self.log.debug("Getting read noise from Photon Transfer Curve.")
584
585 metadata = ampExposure.getMetadata()
586 metadata[f'LSST GAIN {amp.getName()}'] = gain
587 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise
588
589 isrFunctions.updateVariance(
590 maskedImage=ampExposure.getMaskedImage(),
591 gain=gain,
592 readNoise=readNoise,
593 )
594
595 def maskNegativeVariance(self, exposure):
596 """Identify and mask pixels with negative variance values.
597
598 Parameters
599 ----------
600 exposure : `lsst.afw.image.Exposure`
601 Exposure to process.
602
603 See Also
604 --------
605 lsst.ip.isr.isrFunctions.updateVariance
606 """
607 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
608 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
609 exposure.mask.array[bad] |= maskPlane
610
611 # TODO check make stats is necessary or not
612 def variancePlane(self, ccdExposure, ccd, overscans, ptc):
613 for amp, overscanResults in zip(ccd, overscans):
614 if ccdExposure.getBBox().contains(amp.getBBox()):
615 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
616 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
617
618 self.updateVariance(ampExposure, amp, ptcDataset=ptc)
619
620 if self.config.qa is not None and self.config.qa.saveStats is True:
621 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
622 afwMath.MEDIAN | afwMath.STDEVCLIP)
623 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
624 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
625 qaStats.getValue(afwMath.STDEVCLIP))
626 if self.config.maskNegativeVariance:
627 self.maskNegativeVariance(ccdExposure)
628
629 def maskDefect(self, exposure, defectBaseList):
630 """Mask defects using mask plane "BAD", in place.
631
632 Parameters
633 ----------
634 exposure : `lsst.afw.image.Exposure`
635 Exposure to process.
636
637 defectBaseList : defect-type
638 List of defects to mask. Can be of type `lsst.ip.isr.Defects`
639 or `list` of `lsst.afw.image.DefectBase`.
640 """
641 maskedImage = exposure.getMaskedImage()
642 if not isinstance(defectBaseList, Defects):
643 # Promotes DefectBase to Defect
644 defectList = Defects(defectBaseList)
645 else:
646 defectList = defectBaseList
647 defectList.maskPixels(maskedImage, maskName="BAD")
648
649 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
650 """Mask edge pixels with applicable mask plane.
651
652 Parameters
653 ----------
654 exposure : `lsst.afw.image.Exposure`
655 Exposure to process.
656 numEdgePixels : `int`, optional
657 Number of edge pixels to mask.
658 maskPlane : `str`, optional
659 Mask plane name to use.
660 level : `str`, optional
661 Level at which to mask edges.
662 """
663 maskedImage = exposure.getMaskedImage()
664 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
665
666 if numEdgePixels > 0:
667 if level == 'DETECTOR':
668 boxes = [maskedImage.getBBox()]
669 elif level == 'AMP':
670 boxes = [amp.getBBox() for amp in exposure.getDetector()]
671
672 for box in boxes:
673 # This makes a bbox numEdgeSuspect pixels smaller than the
674 # image on each side
675 subImage = maskedImage[box]
676 box.grow(-numEdgePixels)
677 # Mask pixels outside box
678 SourceDetectionTask.setEdgeBits(
679 subImage,
680 box,
681 maskBitMask)
682
683 def maskNan(self, exposure):
684 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
685
686 Parameters
687 ----------
688 exposure : `lsst.afw.image.Exposure`
689 Exposure to process.
690
691 Notes
692 -----
693 We mask over all non-finite values (NaN, inf), including those
694 that are masked with other bits (because those may or may not be
695 interpolated over later, and we want to remove all NaN/infs).
696 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
697 preserve the historical name.
698 """
699 maskedImage = exposure.getMaskedImage()
700
701 # Find and mask NaNs
702 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
703 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
704 numNans = maskNans(maskedImage, maskVal)
705 self.metadata["NUMNANS"] = numNans
706 if numNans > 0:
707 self.log.warning("There were %d unmasked NaNs.", numNans)
708
709 def countBadPixels(self, exposure):
710 """
711 Notes
712 -----
713 Reset and interpolate bad pixels.
714
715 Large contiguous bad regions (which should have the BAD mask
716 bit set) should have their values set to the image median.
717 This group should include defects and bad amplifiers. As the
718 area covered by these defects are large, there's little
719 reason to expect that interpolation would provide a more
720 useful value.
721
722 Smaller defects can be safely interpolated after the larger
723 regions have had their pixel values reset. This ensures
724 that the remaining defects adjacent to bad amplifiers (as an
725 example) do not attempt to interpolate extreme values.
726 """
727 badPixelCount, badPixelValue = isrFunctions.setBadRegions(exposure)
728 if badPixelCount > 0:
729 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
730
731 @contextmanager
732 def flatContext(self, exp, flat, dark=None):
733 """Context manager that applies and removes flats and darks,
734 if the task is configured to apply them.
735
736 Parameters
737 ----------
738 exp : `lsst.afw.image.Exposure`
739 Exposure to process.
740 flat : `lsst.afw.image.Exposure`
741 Flat exposure the same size as ``exp``.
742 dark : `lsst.afw.image.Exposure`, optional
743 Dark exposure the same size as ``exp``.
744
745 Yields
746 ------
747 exp : `lsst.afw.image.Exposure`
748 The flat and dark corrected exposure.
749 """
750 if self.config.doDark and dark is not None:
751 self.darkCorrection(exp, dark)
752 if self.config.doFlat:
753 self.flatCorrection(exp, flat)
754 try:
755 yield exp
756 finally:
757 if self.config.doFlat:
758 self.flatCorrection(exp, flat, invert=True)
759 if self.config.doDark and dark is not None:
760 self.darkCorrection(exp, dark, invert=True)
761
762 def getBrighterFatterKernel(self, detector, bfKernel):
763 detName = detector.getName()
764
765 # This is expected to be a dictionary of amp-wise gains.
766 bfGains = bfKernel.gain
767 if bfKernel.level == 'DETECTOR':
768 if detName in bfKernel.detKernels:
769 bfKernelOut = bfKernel.detKernels[detName]
770 return bfKernelOut, bfGains
771 else:
772 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
773 elif bfKernel.level == 'AMP':
774 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
775 "fatter kernels.")
776 bfKernel.makeDetectorKernelFromAmpwiseKernels(detName)
777 bfKernelOut = bfKernel.detKernels[detName]
778 return bfKernelOut, bfGains
779
780 def applyBrighterFatterCorrection(self, ccdExposure, flat, dark, bfKernel, bfGains):
781 # We need to apply flats and darks before we can interpolate, and
782 # we need to interpolate before we do B-F, but we do B-F without
783 # the flats and darks applied so we can work in units of electrons
784 # or holes. This context manager applies and then removes the darks
785 # and flats.
786 #
787 # We also do not want to interpolate values here, so operate on
788 # temporary images so we can apply only the BF-correction and roll
789 # back the interpolation.
790 # This won't be necessary once the gain normalization
791 # is done appropriately.
792 interpExp = ccdExposure.clone()
793 with self.flatContext(interpExp, flat, dark):
794 isrFunctions.interpolateFromMask(
795 maskedImage=interpExp.getMaskedImage(),
796 fwhm=self.config.brighterFatterFwhmForInterpolation,
797 growSaturatedFootprints=self.config.growSaturationFootprintSize,
798 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
799 )
800 bfExp = interpExp.clone()
801 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
802 type(bfKernel), type(bfGains))
803 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
804 self.config.brighterFatterMaxIter,
805 self.config.brighterFatterThreshold,
806 self.config.brighterFatterApplyGain,
807 bfGains)
808 if bfResults[1] == self.config.brighterFatterMaxIter:
809 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
810 bfResults[0])
811 else:
812 self.log.info("Finished brighter-fatter correction in %d iterations.",
813 bfResults[1])
814
815 image = ccdExposure.getMaskedImage().getImage()
816 bfCorr = bfExp.getMaskedImage().getImage()
817 bfCorr -= interpExp.getMaskedImage().getImage()
818 image += bfCorr
819
820 # Applying the brighter-fatter correction applies a
821 # convolution to the science image. At the edges this
822 # convolution may not have sufficient valid pixels to
823 # produce a valid correction. Mark pixels within the size
824 # of the brighter-fatter kernel as EDGE to warn of this
825 # fact.
826 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
827 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
828 maskPlane="EDGE")
829
830 if self.config.brighterFatterMaskGrowSize > 0:
831 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
832 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
833 isrFunctions.growMasks(ccdExposure.getMask(),
834 radius=self.config.brighterFatterMaskGrowSize,
835 maskNameList=maskPlane,
836 maskValue=maskPlane)
837
838 return ccdExposure
839
840 def darkCorrection(self, exposure, darkExposure, invert=False):
841 """Apply dark correction in place.
842
843 Parameters
844 ----------
845 exposure : `lsst.afw.image.Exposure`
846 Exposure to process.
847 darkExposure : `lsst.afw.image.Exposure`
848 Dark exposure of the same size as ``exposure``.
849 invert : `Bool`, optional
850 If True, re-add the dark to an already corrected image.
851
852 Raises
853 ------
854 RuntimeError
855 Raised if either ``exposure`` or ``darkExposure`` do not
856 have their dark time defined.
857
858 See Also
859 --------
860 lsst.ip.isr.isrFunctions.darkCorrection
861 """
862 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
863 if math.isnan(expScale):
864 raise RuntimeError("Exposure darktime is NAN.")
865 if darkExposure.getInfo().getVisitInfo() is not None \
866 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
867 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
868 else:
869 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
870 # so getDarkTime() does not exist.
871 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
872 darkScale = 1.0
873
874 isrFunctions.darkCorrection(
875 maskedImage=exposure.getMaskedImage(),
876 darkMaskedImage=darkExposure.getMaskedImage(),
877 expScale=expScale,
878 darkScale=darkScale,
879 invert=invert,
880 )
881
882 @staticmethod
884 """Extract common calibration metadata values that will be written to
885 output header.
886
887 Parameters
888 ----------
889 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib`
890 Calibration to pull date information from.
891
892 Returns
893 -------
894 dateString : `str`
895 Calibration creation date string to add to header.
896 """
897 if hasattr(calib, "getMetadata"):
898 if 'CALIB_CREATION_DATE' in calib.getMetadata():
899 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"),
900 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown")))
901 else:
902 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"),
903 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown")))
904 else:
905 return "Unknown Unknown"
906
907 def doLinearize(self, detector):
908 """Check if linearization is needed for the detector cameraGeom.
909
910 Checks config.doLinearize and the linearity type of the first
911 amplifier.
912
913 Parameters
914 ----------
915 detector : `lsst.afw.cameraGeom.Detector`
916 Detector to get linearity type from.
917
918 Returns
919 -------
920 doLinearize : `Bool`
921 If True, linearization should be performed.
922 """
923 return self.config.doLinearize and \
924 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
925
926 def makeBinnedImages(self, exposure):
927 """Make visualizeVisit style binned exposures.
928
929 Parameters
930 ----------
931 exposure : `lsst.afw.image.Exposure`
932 Exposure to bin.
933
934 Returns
935 -------
936 bin1 : `lsst.afw.image.Exposure`
937 Binned exposure using binFactor1.
938 bin2 : `lsst.afw.image.Exposure`
939 Binned exposure using binFactor2.
940 """
941 mi = exposure.getMaskedImage()
942
943 bin1 = afwMath.binImage(mi, self.config.binFactor1)
944 bin2 = afwMath.binImage(mi, self.config.binFactor2)
945
946 return bin1, bin2
947
948 def run(self, *, ccdExposure, dnlLUT=None, bias=None, deferredChargeCalib=None, linearizer=None,
949 ptc=None, crosstalk=None, defects=None, bfKernel=None, bfGains=None, dark=None,
950 flat=None, **kwargs
951 ):
952
953 detector = ccdExposure.getDetector()
954
955 if self.config.doHeaderProvenance:
956 # Inputs have been validated, so we can add their date
957 # information to the output header.
958 exposureMetadata = ccdExposure.getMetadata()
959 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc)
960 if self.config.doDiffNonLinearCorrection:
961 exposureMetadata["LSST CALIB DATE DNL"] = self.extractCalibDate(dnlLUT)
962 if self.config.doBias:
963 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias)
964 if self.config.doDeferredCharge:
965 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib)
966 if self.doLinearize(detector):
967 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer)
968 if self.config.doCrosstalk:
969 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk)
970 if self.config.doDefect:
971 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects)
972 if self.config.doBrighterFatter:
973 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel)
974 if self.config.doDark:
975 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark)
976
977 if self.config.doDiffNonLinearCorrection:
978 self.diffNonLinearCorrection(ccdExposure, dnlLUT)
979
980 if self.config.doOverscan:
981 # Input units: ADU
982 overscans = self.overscanCorrection(detector, ccdExposure)
983
984 if self.config.doAssembleCcd:
985 # Input units: ADU
986 self.log.info("Assembling CCD from amplifiers.")
987 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
988
989 if self.config.expectWcs and not ccdExposure.getWcs():
990 self.log.warning("No WCS found in input exposure.")
991
992 if self.config.doBias:
993 # Input units: ADU
994 self.log.info("Applying bias correction.")
995 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage())
996
997 if self.config.doDeferredCharge:
998 # Input units: ADU
999 self.log.info("Applying deferred charge/CTI correction.")
1000 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib)
1001
1002 if self.config.doLinearize:
1003 # Input units: ADU
1004 self.log.info("Applying linearizer.")
1005 linearizer = self.getLinearizer(detector=detector)
1006 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1007 detector=detector, log=self.log)
1008
1009 if self.config.doGainNormalize:
1010 # Input units: ADU
1011 # Output units: electrons
1012 # TODO DM 36639
1013 gains, readNoise = self.gainNormalize(**kwargs)
1014
1015 if self.config.doVariance:
1016 # Input units: electrons
1017 self.variancePlane(ccdExposure, detector, overscans, ptc)
1018
1019 if self.config.doCrosstalk:
1020 # Input units: electrons
1021 self.log.info("Applying crosstalk correction.")
1022 self.crosstalk.run(ccdExposure, crosstalk=crosstalk)
1023
1024 # Masking block (defects, NAN pixels and trails).
1025 # Saturated and suspect pixels have already been masked.
1026 if self.config.doDefect:
1027 # Input units: electrons
1028 self.log.info("Applying defects masking.")
1029 self.maskDefect(ccdExposure, defects)
1030
1031 if self.config.doNanMasking:
1032 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1033 self.maskNan(ccdExposure)
1034
1035 if self.config.doWidenSaturationTrails:
1036 self.log.info("Widening saturation trails.")
1037 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1038
1039 preInterpExp = None
1040 if self.config.doSaveInterpPixels:
1041 preInterpExp = ccdExposure.clone()
1042
1043 if self.config.doSetBadRegions:
1044 self.log.info('Counting pixels in BAD regions.')
1045 self.countBadPixels(ccdExposure)
1046
1047 if self.config.doInterpolate:
1048 self.log.info("Interpolating masked pixels.")
1049 isrFunctions.interpolateFromMask(
1050 maskedImage=ccdExposure.getMaskedImage(),
1051 fwhm=self.config.brighterFatterFwhmForInterpolation,
1052 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1053 maskNameList=list(self.config.maskListToInterpolate)
1054 )
1055
1056 if self.config.doBrighterFatter:
1057 # Input units: electrons
1058 self.log.info("Applying Bright-Fatter kernels.")
1059 bfKernelOut, bfGains = self.getBrighterFatterKernel(detector, bfKernel)
1060 ccdExposure = self.applyBrighterFatterCorrection(ccdExposure, flat, dark, bfKernelOut, bfGains)
1061
1062 if self.config.doDark:
1063 # Input units: electrons
1064 self.log.info("Applying dark subtraction.")
1065 self.darkCorrection(ccdExposure, dark)
1066
1067 # Calculate standard image quality statistics
1068 if self.config.doStandardStatistics:
1069 metadata = ccdExposure.getMetadata()
1070 for amp in detector:
1071 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1072 ampName = amp.getName()
1073 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels(
1074 ampExposure.getMaskedImage(),
1075 [self.config.saturatedMaskName]
1076 )
1077 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels(
1078 ampExposure.getMaskedImage(),
1079 ["BAD"]
1080 )
1081 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1082 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP)
1083
1084 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN)
1085 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN)
1086 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP)
1087
1088 k1 = f"LSST ISR FINAL MEDIAN {ampName}"
1089 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}"
1090 if self.config.doOverscan and k1 in metadata and k2 in metadata:
1091 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2]
1092 else:
1093 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan
1094
1095 # calculate additional statistics.
1096 outputStatistics = None
1097 if self.config.doCalculateStatistics:
1098 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans,
1099 ptc=ptc).results
1100
1101 # do image binning.
1102 outputBin1Exposure = None
1103 outputBin2Exposure = None
1104 if self.config.doBinnedExposures:
1105 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure)
1106
1107 return pipeBase.Struct(
1108 exposure=ccdExposure,
1109
1110 outputBin1Exposure=outputBin1Exposure,
1111 outputBin2Exposure=outputBin2Exposure,
1112
1113 preInterpExposure=preInterpExp,
1114 outputExposure=ccdExposure,
1115 outputStatistics=outputStatistics,
1116 )
variancePlane(self, ccdExposure, ccd, overscans, ptc)
runQuantum(self, butlerQC, inputRefs, outputRefs)
flatContext(self, exp, flat, dark=None)
overscanCorrection(self, ccd, ccdExposure)
updateVariance(self, ampExposure, amp, ptcDataset=None)
run(self, *ccdExposure, dnlLUT=None, bias=None, deferredChargeCalib=None, linearizer=None, ptc=None, crosstalk=None, defects=None, bfKernel=None, bfGains=None, dark=None, flat=None, **kwargs)
getBrighterFatterKernel(self, detector, bfKernel)
darkCorrection(self, exposure, darkExposure, invert=False)
diffNonLinearCorrection(self, ccdExposure, dnlLUT, **kwargs)
maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
maskDefect(self, exposure, defectBaseList)
applyBrighterFatterCorrection(self, ccdExposure, flat, dark, bfKernel, bfGains)