lsst.ip.isr ga69e4da736+23d5bada5b
Loading...
Searching...
No Matches
isrTaskLSST.py
Go to the documentation of this file.
1__all__ = ["IsrTaskLSST", "IsrTaskLSSTConfig"]
2
3import numpy
4import math
5
6from . import isrFunctions
7from . import isrQa
8from . import linearize
9from .defects import Defects
10
11from contextlib import contextmanager
12from lsst.afw.cameraGeom import NullLinearityType
13import lsst.pex.config as pexConfig
14import lsst.afw.math as afwMath
15import lsst.pipe.base as pipeBase
16import lsst.pipe.base.connectionTypes as cT
17from lsst.meas.algorithms.detection import SourceDetectionTask
18
19from .overscan import OverscanCorrectionTask
20from .assembleCcdTask import AssembleCcdTask
21from .deferredCharge import DeferredChargeTask
22from .crosstalk import CrosstalkTask
23from .masking import MaskingTask
24from .isrStatistics import IsrStatisticsTask
25from .isr import maskNans
26
27
28class IsrTaskLSSTConnections(pipeBase.PipelineTaskConnections,
29 dimensions={"instrument", "exposure", "detector"},
30 defaultTemplates={}):
31 ccdExposure = cT.Input(
32 name="raw",
33 doc="Input exposure to process.",
34 storageClass="Exposure",
35 dimensions=["instrument", "exposure", "detector"],
36 )
37 camera = cT.PrerequisiteInput(
38 name="camera",
39 storageClass="Camera",
40 doc="Input camera to construct complete exposures.",
41 dimensions=["instrument"],
42 isCalibration=True,
43 )
44 dnlLUT = cT.PrerequisiteInput(
45 name="dnlLUT",
46 doc="Look-up table for differential non-linearity.",
47 storageClass="IsrCalib",
48 dimensions=["instrument", "exposure", "detector"],
49 isCalibration=True,
50 # TODO DM 36636
51 )
52 bias = cT.PrerequisiteInput(
53 name="bias",
54 doc="Input bias calibration.",
55 storageClass="ExposureF",
56 dimensions=["instrument", "detector"],
57 isCalibration=True,
58 )
59 deferredChargeCalib = cT.PrerequisiteInput(
60 name="cpCtiCalib",
61 doc="Deferred charge/CTI correction dataset.",
62 storageClass="IsrCalib",
63 dimensions=["instrument", "detector"],
64 isCalibration=True,
65 )
66 linearizer = cT.PrerequisiteInput(
67 name='linearizer',
68 storageClass="Linearizer",
69 doc="Linearity correction calibration.",
70 dimensions=["instrument", "detector"],
71 isCalibration=True,
72 )
73 ptc = cT.PrerequisiteInput(
74 name="ptc",
75 doc="Input Photon Transfer Curve dataset",
76 storageClass="PhotonTransferCurveDataset",
77 dimensions=["instrument", "detector"],
78 isCalibration=True,
79 )
80 crosstalk = cT.PrerequisiteInput(
81 name="crosstalk",
82 doc="Input crosstalk object",
83 storageClass="CrosstalkCalib",
84 dimensions=["instrument", "detector"],
85 isCalibration=True,
86 )
87 defects = cT.PrerequisiteInput(
88 name='defects',
89 doc="Input defect tables.",
90 storageClass="Defects",
91 dimensions=["instrument", "detector"],
92 isCalibration=True,
93 )
94 bfKernel = cT.PrerequisiteInput(
95 name='brighterFatterKernel',
96 doc="Complete kernel + gain solutions.",
97 storageClass="BrighterFatterKernel",
98 dimensions=["instrument", "detector"],
99 isCalibration=True,
100 )
101 dark = cT.PrerequisiteInput(
102 name='dark',
103 doc="Input dark calibration.",
104 storageClass="ExposureF",
105 dimensions=["instrument", "detector"],
106 isCalibration=True,
107 )
108 outputExposure = cT.Output(
109 name='postISRCCD',
110 doc="Output ISR processed exposure.",
111 storageClass="Exposure",
112 dimensions=["instrument", "exposure", "detector"],
113 )
114 preInterpExposure = cT.Output(
115 name='preInterpISRCCD',
116 doc="Output ISR processed exposure, with pixels left uninterpolated.",
117 storageClass="ExposureF",
118 dimensions=["instrument", "exposure", "detector"],
119 )
120 outputBin1Exposure = cT.Output(
121 name="postIsrBin1",
122 doc="First binned image.",
123 storageClass="ExposureF",
124 dimensions=["instrument", "exposure", "detector"],
125 )
126 outputBin2Exposure = cT.Output(
127 name="postIsrBin2",
128 doc="Second binned image.",
129 storageClass="ExposureF",
130 dimensions=["instrument", "exposure", "detector"],
131 )
132
133 outputStatistics = cT.Output(
134 name="isrStatistics",
135 doc="Output of additional statistics table.",
136 storageClass="StructuredDataDict",
137 dimensions=["instrument", "exposure", "detector"],
138 )
139
140 def __init__(self, *, config=None):
141 super().__init__(config=config)
142
143 if config.doDiffNonLinearCorrection is not True:
144 del self.dnlLUT
145 if config.doBias is not True:
146 del self.bias
147 if config.doDeferredCharge is not True:
148 del self.deferredChargeCalib
149 if config.doLinearize is not True:
150 del self.linearizer
151 if config.doCrosstalk is not True:
152 del self.crosstalk
153 if config.doDefect is not True:
154 del self.defects
155 if config.doBrighterFatter is not True:
156 del self.bfKernel
157 if config.doDark is not True:
158 del self.dark
159
160 if config.doBinnedExposures is not True:
161 del self.outputBin1Exposure
162 del self.outputBin2Exposure
163 if config.doSaveInterpPixels is not True:
164 del self.preInterpExposure
165
166 if config.doCalculateStatistics is not True:
167 del self.outputStatistics
168
169
170class IsrTaskLSSTConfig(pipeBase.PipelineTaskConfig,
171 pipelineConnections=IsrTaskLSSTConnections):
172 """Configuration parameters for IsrTaskLSST.
173
174 Items are grouped in the order in which they are executed by the task.
175 """
176 expectWcs = pexConfig.Field(
177 dtype=bool,
178 default=True,
179 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
180 )
181 qa = pexConfig.ConfigField(
182 dtype=isrQa.IsrQaConfig,
183 doc="QA related configuration options.",
184 )
185 doHeaderProvenance = pexConfig.Field(
186 dtype=bool,
187 default=True,
188 doc="Write calibration identifiers into output exposure header.",
189 )
190
191 # Differential non-linearity correction.
192 doDiffNonLinearCorrection = pexConfig.Field(
193 dtype=bool,
194 doc="Do differential non-linearity correction?",
195 default=True,
196 )
197
198 doOverscan = pexConfig.Field(
199 dtype=bool,
200 doc="Do overscan subtraction?",
201 default=True,
202 )
203 overscan = pexConfig.ConfigurableField(
204 target=OverscanCorrectionTask,
205 doc="Overscan subtraction task for image segments.",
206 )
207
208 # Amplifier to CCD assembly configuration.
209 doAssembleCcd = pexConfig.Field(
210 dtype=bool,
211 default=True,
212 doc="Assemble amp-level exposures into a ccd-level exposure?"
213 )
214 assembleCcd = pexConfig.ConfigurableField(
215 target=AssembleCcdTask,
216 doc="CCD assembly task.",
217 )
218
219 # Bias subtraction.
220 doBias = pexConfig.Field(
221 dtype=bool,
222 doc="Apply bias frame correction?",
223 default=True,
224 )
225
226 # Deferred charge correction.
227 doDeferredCharge = pexConfig.Field(
228 dtype=bool,
229 doc="Apply deferred charge correction?",
230 default=True,
231 )
232 deferredChargeCorrection = pexConfig.ConfigurableField(
233 target=DeferredChargeTask,
234 doc="Deferred charge correction task.",
235 )
236
237 # Linearization.
238 doLinearize = pexConfig.Field(
239 dtype=bool,
240 doc="Correct for nonlinearity of the detector's response?",
241 default=True,
242 )
243
244 # Normalize gain.
245 doGainNormalize = pexConfig.Field(
246 dtype=bool,
247 doc="Normalize by the gain.",
248 default=True,
249 )
250
251 # Variance construction.
252 doVariance = pexConfig.Field(
253 dtype=bool,
254 doc="Calculate variance?",
255 default=True
256 )
257 gain = pexConfig.Field(
258 dtype=float,
259 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN).",
260 default=float("NaN"),
261 )
262 maskNegativeVariance = pexConfig.Field(
263 dtype=bool,
264 doc="Mask pixels that claim a negative variance. This likely indicates a failure "
265 "in the measurement of the overscan at an edge due to the data falling off faster "
266 "than the overscan model can account for it.",
267 default=True,
268 )
269 negativeVarianceMaskName = pexConfig.Field(
270 dtype=str,
271 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
272 default="BAD",
273 )
274 saturatedMaskName = pexConfig.Field(
275 dtype=str,
276 doc="Name of mask plane to use in saturation detection and interpolation.",
277 default="SAT",
278 )
279 suspectMaskName = pexConfig.Field(
280 dtype=str,
281 doc="Name of mask plane to use for suspect pixels.",
282 default="SUSPECT",
283 )
284
285 # Crosstalk.
286 doCrosstalk = pexConfig.Field(
287 dtype=bool,
288 doc="Apply intra-CCD crosstalk correction?",
289 default=True,
290 )
291 crosstalk = pexConfig.ConfigurableField(
292 target=CrosstalkTask,
293 doc="Intra-CCD crosstalk correction.",
294 )
295
296 # Masking options.
297 doDefect = pexConfig.Field(
298 dtype=bool,
299 doc="Apply correction for CCD defects, e.g. hot pixels?",
300 default=True,
301 )
302 doNanMasking = pexConfig.Field(
303 dtype=bool,
304 doc="Mask non-finite (NAN, inf) pixels.",
305 default=True,
306 )
307 doWidenSaturationTrails = pexConfig.Field(
308 dtype=bool,
309 doc="Widen bleed trails based on their width.",
310 default=True,
311 )
312 masking = pexConfig.ConfigurableField(
313 target=MaskingTask,
314 doc="Masking task."
315 )
316
317 # Interpolation options.
318 doInterpolate = pexConfig.Field(
319 dtype=bool,
320 doc="Interpolate masked pixels?",
321 default=True,
322 )
323 maskListToInterpolate = pexConfig.ListField(
324 dtype=str,
325 doc="List of mask planes that should be interpolated.",
326 default=['SAT', 'BAD'],
327 )
328 doSaveInterpPixels = pexConfig.Field(
329 dtype=bool,
330 doc="Save a copy of the pre-interpolated pixel values?",
331 default=False,
332 )
333
334 # Initial masking options.
335 doSetBadRegions = pexConfig.Field(
336 dtype=bool,
337 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
338 default=True,
339 )
340
341 # Brighter-Fatter correction.
342 doBrighterFatter = pexConfig.Field(
343 dtype=bool,
344 doc="Apply the brighter-fatter correction?",
345 default=True,
346 )
347 brighterFatterLevel = pexConfig.ChoiceField(
348 dtype=str,
349 doc="The level at which to correct for brighter-fatter.",
350 allowed={
351 "AMP": "Every amplifier treated separately.",
352 "DETECTOR": "One kernel per detector.",
353 },
354 default="DETECTOR",
355 )
356 brighterFatterMaxIter = pexConfig.Field(
357 dtype=int,
358 doc="Maximum number of iterations for the brighter-fatter correction.",
359 default=10,
360 )
361 brighterFatterThreshold = pexConfig.Field(
362 dtype=float,
363 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
364 "absolute value of the difference between the current corrected image and the one "
365 "from the previous iteration summed over all the pixels.",
366 default=1000,
367 )
368 brighterFatterApplyGain = pexConfig.Field(
369 dtype=bool,
370 doc="Should the gain be applied when applying the brighter-fatter correction?",
371 default=True,
372 )
373 brighterFatterMaskListToInterpolate = pexConfig.ListField(
374 dtype=str,
375 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
376 "correction.",
377 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
378 )
379 brighterFatterMaskGrowSize = pexConfig.Field(
380 dtype=int,
381 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
382 "when brighter-fatter correction is applied.",
383 default=0,
384 )
385 brighterFatterFwhmForInterpolation = pexConfig.Field(
386 dtype=float,
387 doc="FWHM of PSF in arcseconds used for interpolation in brighter-fatter correction "
388 "(currently unused).",
389 default=1.0,
390 )
391 growSaturationFootprintSize = pexConfig.Field(
392 dtype=int,
393 doc="Number of pixels by which to grow the saturation footprints.",
394 default=1,
395 )
396 brighterFatterMaskListToInterpolate = pexConfig.ListField(
397 dtype=str,
398 doc="List of mask planes that should be interpolated over when applying the brighter-fatter."
399 "correction.",
400 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
401 )
402
403 # Dark subtraction.
404 doDark = pexConfig.Field(
405 dtype=bool,
406 doc="Apply dark frame correction.",
407 default=True,
408 )
409
410 # Calculate image quality statistics?
411 doStandardStatistics = pexConfig.Field(
412 dtype=bool,
413 doc="Should standard image quality statistics be calculated?",
414 default=True,
415 )
416 # Calculate additional statistics?
417 doCalculateStatistics = pexConfig.Field(
418 dtype=bool,
419 doc="Should additional ISR statistics be calculated?",
420 default=True,
421 )
422 isrStats = pexConfig.ConfigurableField(
423 target=IsrStatisticsTask,
424 doc="Task to calculate additional statistics.",
425 )
426
427 # Make binned images?
428 doBinnedExposures = pexConfig.Field(
429 dtype=bool,
430 doc="Should binned exposures be calculated?",
431 default=False,
432 )
433 binFactor1 = pexConfig.Field(
434 dtype=int,
435 doc="Binning factor for first binned exposure. This is intended for a finely binned output.",
436 default=8,
437 check=lambda x: x > 1,
438 )
439 binFactor2 = pexConfig.Field(
440 dtype=int,
441 doc="Binning factor for second binned exposure. This is intended for a coarsely binned output.",
442 default=64,
443 check=lambda x: x > 1,
444 )
445
446
447class IsrTaskLSST(pipeBase.PipelineTask):
448 ConfigClass = IsrTaskLSSTConfig
449 _DefaultName = "isr"
450
451 def __init__(self, **kwargs):
452 super().__init__(**kwargs)
453 self.makeSubtask("overscan")
454 self.makeSubtask("assembleCcd")
455 self.makeSubtask("deferredChargeCorrection")
456 self.makeSubtask("crosstalk")
457 self.makeSubtask("masking")
458 self.makeSubtask("isrStats")
459
460 def runQuantum(self, butlerQC, inputRefs, outputRefs):
461
462 inputs = butlerQC.get(inputRefs)
463 self.validateInput(inputs)
464 super().runQuantum(butlerQC, inputRefs, outputRefs)
465
466 def validateInput(self, inputs):
467 """
468 This is a check that all the inputs required by the config
469 are available.
470 """
471
472 inputMap = {'dnlLUT': self.config.doDiffNonLinearCorrection,
473 'bias': self.config.doBias,
474 'deferredChargeCalib': self.config.doDeferredCharge,
475 'linearizer': self.config.doLinearize,
476 'ptc': self.config.doGainNormalize,
477 'crosstalk': self.config.doCrosstalk,
478 'defects': self.config.doDefect,
479 'bfKernel': self.config.doBrighterFatter,
480 'dark': self.config.doDark,
481 }
482
483 for calibrationFile, configValue in inputMap.items():
484 if configValue and inputs[calibrationFile] is None:
485 raise RuntimeError("Must supply ", calibrationFile)
486
487 def diffNonLinearCorrection(self, ccdExposure, dnlLUT, **kwargs):
488 # TODO DM 36636
489 # isrFunctions.diffNonLinearCorrection
490 pass
491
492 def overscanCorrection(self, ccd, ccdExposure):
493 # TODO DM 36637 for per amp
494
495 overscans = []
496 for amp in ccd:
497
498 # Overscan correction on amp-by-amp basis.
499 if amp.getRawHorizontalOverscanBBox().isEmpty():
500 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
501 overscans.append(None)
502 else:
503
504 # Perform overscan correction on subregions.
505 overscanResults = self.overscan.run(ccdExposure, amp)
506
507 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
508 if len(overscans) == 0:
509 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
510
511 overscans.append(overscanResults if overscanResults is not None else None)
512
513 return overscans
514
515 def getLinearizer(self, detector):
516 # Here we assume linearizer as dict or LUT are not supported
517 # TODO DM 28741
518
519 # TODO construct isrcalib input
520 linearizer = linearize.Linearizer(detector=detector, log=self.log)
521 self.log.warning("Constructing linearizer from cameraGeom information.")
522
523 return linearizer
524
525 def gainNormalize(self, **kwargs):
526 # TODO DM 36639
527 gains = []
528 readNoise = []
529
530 return gains, readNoise
531
532 def updateVariance(self, ampExposure, amp, ptcDataset=None):
533 """Set the variance plane using the gain and read noise.
534
535 Parameters
536 ----------
537 ampExposure : `lsst.afw.image.Exposure`
538 Exposure to process.
539 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp`
540 Amplifier detector data.
541 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
542 PTC dataset containing the gains and read noise.
543
544 Raises
545 ------
546 RuntimeError
547 Raised if ptcDataset is not provided.
548
549 See also
550 --------
551 lsst.ip.isr.isrFunctions.updateVariance
552 """
553 # Get gains from PTC
554 if ptcDataset is None:
555 raise RuntimeError("No ptcDataset provided to use PTC gains.")
556 else:
557 gain = ptcDataset.gain[amp.getName()]
558 self.log.debug("Getting gain from Photon Transfer Curve.")
559
560 if math.isnan(gain):
561 gain = 1.0
562 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
563 elif gain <= 0:
564 patchedGain = 1.0
565 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
566 amp.getName(), gain, patchedGain)
567 gain = patchedGain
568
569 # Get read noise from PTC
570 if ptcDataset is None:
571 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
572 else:
573 readNoise = ptcDataset.noise[amp.getName()]
574 self.log.debug("Getting read noise from Photon Transfer Curve.")
575
576 metadata = ampExposure.getMetadata()
577 metadata[f'LSST GAIN {amp.getName()}'] = gain
578 metadata[f'LSST READNOISE {amp.getName()}'] = readNoise
579
580 isrFunctions.updateVariance(
581 maskedImage=ampExposure.getMaskedImage(),
582 gain=gain,
583 readNoise=readNoise,
584 )
585
586 def maskNegativeVariance(self, exposure):
587 """Identify and mask pixels with negative variance values.
588
589 Parameters
590 ----------
591 exposure : `lsst.afw.image.Exposure`
592 Exposure to process.
593
594 See Also
595 --------
596 lsst.ip.isr.isrFunctions.updateVariance
597 """
598 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
599 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
600 exposure.mask.array[bad] |= maskPlane
601
602 # TODO check make stats is necessary or not
603 def variancePlane(self, ccdExposure, ccd, overscans, ptc):
604 for amp, overscanResults in zip(ccd, overscans):
605 if ccdExposure.getBBox().contains(amp.getBBox()):
606 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
607 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
608
609 self.updateVariance(ampExposure, amp, ptcDataset=ptc)
610
611 if self.config.qa is not None and self.config.qa.saveStats is True:
612 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
613 afwMath.MEDIAN | afwMath.STDEVCLIP)
614 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
615 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
616 qaStats.getValue(afwMath.STDEVCLIP))
617 if self.config.maskNegativeVariance:
618 self.maskNegativeVariance(ccdExposure)
619
620 def maskDefect(self, exposure, defectBaseList):
621 """Mask defects using mask plane "BAD", in place.
622
623 Parameters
624 ----------
625 exposure : `lsst.afw.image.Exposure`
626 Exposure to process.
627
628 defectBaseList : defect-type
629 List of defects to mask. Can be of type `lsst.ip.isr.Defects`
630 or `list` of `lsst.afw.image.DefectBase`.
631 """
632 maskedImage = exposure.getMaskedImage()
633 if not isinstance(defectBaseList, Defects):
634 # Promotes DefectBase to Defect
635 defectList = Defects(defectBaseList)
636 else:
637 defectList = defectBaseList
638 defectList.maskPixels(maskedImage, maskName="BAD")
639
640 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
641 """Mask edge pixels with applicable mask plane.
642
643 Parameters
644 ----------
645 exposure : `lsst.afw.image.Exposure`
646 Exposure to process.
647 numEdgePixels : `int`, optional
648 Number of edge pixels to mask.
649 maskPlane : `str`, optional
650 Mask plane name to use.
651 level : `str`, optional
652 Level at which to mask edges.
653 """
654 maskedImage = exposure.getMaskedImage()
655 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
656
657 if numEdgePixels > 0:
658 if level == 'DETECTOR':
659 boxes = [maskedImage.getBBox()]
660 elif level == 'AMP':
661 boxes = [amp.getBBox() for amp in exposure.getDetector()]
662
663 for box in boxes:
664 # This makes a bbox numEdgeSuspect pixels smaller than the
665 # image on each side
666 subImage = maskedImage[box]
667 box.grow(-numEdgePixels)
668 # Mask pixels outside box
669 SourceDetectionTask.setEdgeBits(
670 subImage,
671 box,
672 maskBitMask)
673
674 def maskNan(self, exposure):
675 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
676
677 Parameters
678 ----------
679 exposure : `lsst.afw.image.Exposure`
680 Exposure to process.
681
682 Notes
683 -----
684 We mask over all non-finite values (NaN, inf), including those
685 that are masked with other bits (because those may or may not be
686 interpolated over later, and we want to remove all NaN/infs).
687 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
688 preserve the historical name.
689 """
690 maskedImage = exposure.getMaskedImage()
691
692 # Find and mask NaNs
693 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
694 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
695 numNans = maskNans(maskedImage, maskVal)
696 self.metadata["NUMNANS"] = numNans
697 if numNans > 0:
698 self.log.warning("There were %d unmasked NaNs.", numNans)
699
700 def countBadPixels(self, exposure):
701 """
702 Notes
703 -----
704 Reset and interpolate bad pixels.
705
706 Large contiguous bad regions (which should have the BAD mask
707 bit set) should have their values set to the image median.
708 This group should include defects and bad amplifiers. As the
709 area covered by these defects are large, there's little
710 reason to expect that interpolation would provide a more
711 useful value.
712
713 Smaller defects can be safely interpolated after the larger
714 regions have had their pixel values reset. This ensures
715 that the remaining defects adjacent to bad amplifiers (as an
716 example) do not attempt to interpolate extreme values.
717 """
718 badPixelCount, badPixelValue = isrFunctions.setBadRegions(exposure)
719 if badPixelCount > 0:
720 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
721
722 @contextmanager
723 def flatContext(self, exp, flat, dark=None):
724 """Context manager that applies and removes flats and darks,
725 if the task is configured to apply them.
726
727 Parameters
728 ----------
729 exp : `lsst.afw.image.Exposure`
730 Exposure to process.
731 flat : `lsst.afw.image.Exposure`
732 Flat exposure the same size as ``exp``.
733 dark : `lsst.afw.image.Exposure`, optional
734 Dark exposure the same size as ``exp``.
735
736 Yields
737 ------
738 exp : `lsst.afw.image.Exposure`
739 The flat and dark corrected exposure.
740 """
741 if self.config.doDark and dark is not None:
742 self.darkCorrection(exp, dark)
743 if self.config.doFlat:
744 self.flatCorrection(exp, flat)
745 try:
746 yield exp
747 finally:
748 if self.config.doFlat:
749 self.flatCorrection(exp, flat, invert=True)
750 if self.config.doDark and dark is not None:
751 self.darkCorrection(exp, dark, invert=True)
752
753 def getBrighterFatterKernel(self, detector, bfKernel):
754 detName = detector.getName()
755
756 # This is expected to be a dictionary of amp-wise gains.
757 bfGains = bfKernel.gain
758 if bfKernel.level == 'DETECTOR':
759 if detName in bfKernel.detKernels:
760 bfKernelOut = bfKernel.detKernels[detName]
761 return bfKernelOut, bfGains
762 else:
763 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
764 elif bfKernel.level == 'AMP':
765 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
766 "fatter kernels.")
767 bfKernel.makeDetectorKernelFromAmpwiseKernels(detName)
768 bfKernelOut = bfKernel.detKernels[detName]
769 return bfKernelOut, bfGains
770
771 def applyBrighterFatterCorrection(self, ccdExposure, flat, dark, bfKernel, bfGains):
772 # We need to apply flats and darks before we can interpolate, and
773 # we need to interpolate before we do B-F, but we do B-F without
774 # the flats and darks applied so we can work in units of electrons
775 # or holes. This context manager applies and then removes the darks
776 # and flats.
777 #
778 # We also do not want to interpolate values here, so operate on
779 # temporary images so we can apply only the BF-correction and roll
780 # back the interpolation.
781 # This won't be necessary once the gain normalization
782 # is done appropriately.
783 interpExp = ccdExposure.clone()
784 with self.flatContext(interpExp, flat, dark):
785 isrFunctions.interpolateFromMask(
786 maskedImage=interpExp.getMaskedImage(),
787 fwhm=self.config.brighterFatterFwhmForInterpolation,
788 growSaturatedFootprints=self.config.growSaturationFootprintSize,
789 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
790 )
791 bfExp = interpExp.clone()
792 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
793 type(bfKernel), type(bfGains))
794 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
795 self.config.brighterFatterMaxIter,
796 self.config.brighterFatterThreshold,
797 self.config.brighterFatterApplyGain,
798 bfGains)
799 if bfResults[1] == self.config.brighterFatterMaxIter:
800 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
801 bfResults[0])
802 else:
803 self.log.info("Finished brighter-fatter correction in %d iterations.",
804 bfResults[1])
805
806 image = ccdExposure.getMaskedImage().getImage()
807 bfCorr = bfExp.getMaskedImage().getImage()
808 bfCorr -= interpExp.getMaskedImage().getImage()
809 image += bfCorr
810
811 # Applying the brighter-fatter correction applies a
812 # convolution to the science image. At the edges this
813 # convolution may not have sufficient valid pixels to
814 # produce a valid correction. Mark pixels within the size
815 # of the brighter-fatter kernel as EDGE to warn of this
816 # fact.
817 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
818 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
819 maskPlane="EDGE")
820
821 if self.config.brighterFatterMaskGrowSize > 0:
822 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
823 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
824 isrFunctions.growMasks(ccdExposure.getMask(),
825 radius=self.config.brighterFatterMaskGrowSize,
826 maskNameList=maskPlane,
827 maskValue=maskPlane)
828
829 return ccdExposure
830
831 def darkCorrection(self, exposure, darkExposure, invert=False):
832 """Apply dark correction in place.
833
834 Parameters
835 ----------
836 exposure : `lsst.afw.image.Exposure`
837 Exposure to process.
838 darkExposure : `lsst.afw.image.Exposure`
839 Dark exposure of the same size as ``exposure``.
840 invert : `Bool`, optional
841 If True, re-add the dark to an already corrected image.
842
843 Raises
844 ------
845 RuntimeError
846 Raised if either ``exposure`` or ``darkExposure`` do not
847 have their dark time defined.
848
849 See Also
850 --------
851 lsst.ip.isr.isrFunctions.darkCorrection
852 """
853 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
854 if math.isnan(expScale):
855 raise RuntimeError("Exposure darktime is NAN.")
856 if darkExposure.getInfo().getVisitInfo() is not None \
857 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
858 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
859 else:
860 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
861 # so getDarkTime() does not exist.
862 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
863 darkScale = 1.0
864
865 isrFunctions.darkCorrection(
866 maskedImage=exposure.getMaskedImage(),
867 darkMaskedImage=darkExposure.getMaskedImage(),
868 expScale=expScale,
869 darkScale=darkScale,
870 invert=invert,
871 )
872
873 @staticmethod
875 """Extract common calibration metadata values that will be written to
876 output header.
877
878 Parameters
879 ----------
880 calib : `lsst.afw.image.Exposure` or `lsst.ip.isr.IsrCalib`
881 Calibration to pull date information from.
882
883 Returns
884 -------
885 dateString : `str`
886 Calibration creation date string to add to header.
887 """
888 if hasattr(calib, "getMetadata"):
889 if 'CALIB_CREATION_DATE' in calib.getMetadata():
890 return " ".join((calib.getMetadata().get("CALIB_CREATION_DATE", "Unknown"),
891 calib.getMetadata().get("CALIB_CREATION_TIME", "Unknown")))
892 else:
893 return " ".join((calib.getMetadata().get("CALIB_CREATE_DATE", "Unknown"),
894 calib.getMetadata().get("CALIB_CREATE_TIME", "Unknown")))
895 else:
896 return "Unknown Unknown"
897
898 def doLinearize(self, detector):
899 """Check if linearization is needed for the detector cameraGeom.
900
901 Checks config.doLinearize and the linearity type of the first
902 amplifier.
903
904 Parameters
905 ----------
906 detector : `lsst.afw.cameraGeom.Detector`
907 Detector to get linearity type from.
908
909 Returns
910 -------
911 doLinearize : `Bool`
912 If True, linearization should be performed.
913 """
914 return self.config.doLinearize and \
915 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
916
917 def makeBinnedImages(self, exposure):
918 """Make visualizeVisit style binned exposures.
919
920 Parameters
921 ----------
922 exposure : `lsst.afw.image.Exposure`
923 Exposure to bin.
924
925 Returns
926 -------
927 bin1 : `lsst.afw.image.Exposure`
928 Binned exposure using binFactor1.
929 bin2 : `lsst.afw.image.Exposure`
930 Binned exposure using binFactor2.
931 """
932 mi = exposure.getMaskedImage()
933
934 bin1 = afwMath.binImage(mi, self.config.binFactor1)
935 bin2 = afwMath.binImage(mi, self.config.binFactor2)
936
937 return bin1, bin2
938
939 def run(self, *, ccdExposure, dnlLUT=None, bias=None, deferredChargeCalib=None, linearizer=None,
940 ptc=None, crosstalk=None, defects=None, bfKernel=None, bfGains=None, dark=None,
941 flat=None, **kwargs
942 ):
943
944 detector = ccdExposure.getDetector()
945
946 if self.config.doHeaderProvenance:
947 # Inputs have been validated, so we can add their date
948 # information to the output header.
949 exposureMetadata = ccdExposure.getMetadata()
950 exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc)
951 if self.config.doDiffNonLinearCorrection:
952 exposureMetadata["LSST CALIB DATE DNL"] = self.extractCalibDate(dnlLUT)
953 if self.config.doBias:
954 exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias)
955 if self.config.doDeferredCharge:
956 exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib)
957 if self.doLinearize(detector):
958 exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer)
959 if self.config.doCrosstalk:
960 exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk)
961 if self.config.doDefect:
962 exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects)
963 if self.config.doBrighterFatter:
964 exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel)
965 if self.config.doDark:
966 exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark)
967
968 if self.config.doDiffNonLinearCorrection:
969 self.diffNonLinearCorrection(ccdExposure, dnlLUT)
970
971 if self.config.doOverscan:
972 # Input units: ADU
973 overscans = self.overscanCorrection(detector, ccdExposure)
974
975 if self.config.doAssembleCcd:
976 # Input units: ADU
977 self.log.info("Assembling CCD from amplifiers.")
978 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
979
980 if self.config.expectWcs and not ccdExposure.getWcs():
981 self.log.warning("No WCS found in input exposure.")
982
983 if self.config.doBias:
984 # Input units: ADU
985 self.log.info("Applying bias correction.")
986 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage())
987
988 if self.config.doDeferredCharge:
989 # Input units: ADU
990 self.log.info("Applying deferred charge/CTI correction.")
991 self.deferredChargeCorrection.run(ccdExposure, deferredChargeCalib)
992
993 if self.config.doLinearize:
994 # Input units: ADU
995 self.log.info("Applying linearizer.")
996 linearizer = self.getLinearizer(detector=detector)
997 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
998 detector=detector, log=self.log)
999
1000 if self.config.doGainNormalize:
1001 # Input units: ADU
1002 # Output units: electrons
1003 # TODO DM 36639
1004 gains, readNoise = self.gainNormalize(**kwargs)
1005
1006 if self.config.doVariance:
1007 # Input units: electrons
1008 self.variancePlane(ccdExposure, detector, overscans, ptc)
1009
1010 if self.config.doCrosstalk:
1011 # Input units: electrons
1012 self.log.info("Applying crosstalk correction.")
1013 self.crosstalk.run(ccdExposure, crosstalk=crosstalk)
1014
1015 # Masking block (defects, NAN pixels and trails).
1016 # Saturated and suspect pixels have already been masked.
1017 if self.config.doDefect:
1018 # Input units: electrons
1019 self.log.info("Applying defects masking.")
1020 self.maskDefect(ccdExposure, defects)
1021
1022 if self.config.doNanMasking:
1023 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1024 self.maskNan(ccdExposure)
1025
1026 if self.config.doWidenSaturationTrails:
1027 self.log.info("Widening saturation trails.")
1028 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1029
1030 preInterpExp = None
1031 if self.config.doSaveInterpPixels:
1032 preInterpExp = ccdExposure.clone()
1033
1034 if self.config.doSetBadRegions:
1035 self.log.info('Counting pixels in BAD regions.')
1036 self.countBadPixels(ccdExposure)
1037
1038 if self.config.doInterpolate:
1039 self.log.info("Interpolating masked pixels.")
1040 isrFunctions.interpolateFromMask(
1041 maskedImage=ccdExposure.getMaskedImage(),
1042 fwhm=self.config.brighterFatterFwhmForInterpolation,
1043 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1044 maskNameList=list(self.config.maskListToInterpolate)
1045 )
1046
1047 if self.config.doBrighterFatter:
1048 # Input units: electrons
1049 self.log.info("Applying Bright-Fatter kernels.")
1050 bfKernelOut, bfGains = self.getBrighterFatterKernel(detector, bfKernel)
1051 ccdExposure = self.applyBrighterFatterCorrection(ccdExposure, flat, dark, bfKernelOut, bfGains)
1052
1053 if self.config.doDark:
1054 # Input units: electrons
1055 self.log.info("Applying dark subtraction.")
1056 self.darkCorrection(ccdExposure, dark)
1057
1058 # Calculate standard image quality statistics
1059 if self.config.doStandardStatistics:
1060 metadata = ccdExposure.getMetadata()
1061 for amp in detector:
1062 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1063 ampName = amp.getName()
1064 metadata[f"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels(
1065 ampExposure.getMaskedImage(),
1066 [self.config.saturatedMaskName]
1067 )
1068 metadata[f"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels(
1069 ampExposure.getMaskedImage(),
1070 ["BAD"]
1071 )
1072 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1073 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP)
1074
1075 metadata[f"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN)
1076 metadata[f"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN)
1077 metadata[f"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP)
1078
1079 k1 = f"LSST ISR FINAL MEDIAN {ampName}"
1080 k2 = f"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}"
1081 if self.config.doOverscan and k1 in metadata and k2 in metadata:
1082 metadata[f"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2]
1083 else:
1084 metadata[f"LSST ISR LEVEL {ampName}"] = numpy.nan
1085
1086 # calculate additional statistics.
1087 outputStatistics = None
1088 if self.config.doCalculateStatistics:
1089 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans,
1090 ptc=ptc).results
1091
1092 # do image binning.
1093 outputBin1Exposure = None
1094 outputBin2Exposure = None
1095 if self.config.doBinnedExposures:
1096 outputBin1Exposure, outputBin2Exposure = self.makeBinnedImages(ccdExposure)
1097
1098 return pipeBase.Struct(
1099 exposure=ccdExposure,
1100
1101 outputBin1Exposure=outputBin1Exposure,
1102 outputBin2Exposure=outputBin2Exposure,
1103
1104 preInterpExposure=preInterpExp,
1105 outputExposure=ccdExposure,
1106 outputStatistics=outputStatistics,
1107 )
variancePlane(self, ccdExposure, ccd, overscans, ptc)
runQuantum(self, butlerQC, inputRefs, outputRefs)
flatContext(self, exp, flat, dark=None)
overscanCorrection(self, ccd, ccdExposure)
updateVariance(self, ampExposure, amp, ptcDataset=None)
run(self, *ccdExposure, dnlLUT=None, bias=None, deferredChargeCalib=None, linearizer=None, ptc=None, crosstalk=None, defects=None, bfKernel=None, bfGains=None, dark=None, flat=None, **kwargs)
getBrighterFatterKernel(self, detector, bfKernel)
darkCorrection(self, exposure, darkExposure, invert=False)
diffNonLinearCorrection(self, ccdExposure, dnlLUT, **kwargs)
maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
maskDefect(self, exposure, defectBaseList)
applyBrighterFatterCorrection(self, ccdExposure, flat, dark, bfKernel, bfGains)