28 import lsst.pex.config
as pexConfig
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .straylight
import StrayLightTask
54 from .vignette
import VignetteTask
57 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
61 dimensions={
"instrument",
"visit",
"detector"},
63 ccdExposure = cT.PrerequisiteInput(
65 doc=
"Input exposure to process.",
66 storageClass=
"Exposure",
67 dimensions=[
"instrument",
"visit",
"detector"],
69 camera = cT.PrerequisiteInput(
71 storageClass=
"Camera",
72 doc=
"Input camera to construct complete exposures.",
73 dimensions=[
"instrument",
"calibration_label"],
75 bias = cT.PrerequisiteInput(
77 doc=
"Input bias calibration.",
78 storageClass=
"ImageF",
79 dimensions=[
"instrument",
"calibration_label",
"detector"],
81 dark = cT.PrerequisiteInput(
83 doc=
"Input dark calibration.",
84 storageClass=
"ImageF",
85 dimensions=[
"instrument",
"calibration_label",
"detector"],
87 flat = cT.PrerequisiteInput(
89 doc=
"Input flat calibration.",
90 storageClass=
"MaskedImageF",
91 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
93 fringes = cT.PrerequisiteInput(
95 doc=
"Input fringe calibration.",
96 storageClass=
"ExposureF",
97 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
99 strayLightData = cT.PrerequisiteInput(
101 doc=
"Input stray light calibration.",
102 storageClass=
"StrayLightData",
103 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
105 bfKernel = cT.PrerequisiteInput(
107 doc=
"Input brighter-fatter kernel.",
108 storageClass=
"NumpyArray",
109 dimensions=[
"instrument",
"calibration_label"],
111 newBFKernel = cT.PrerequisiteInput(
112 name=
'brighterFatterKernel',
113 doc=
"Newer complete kernel + gain solutions.",
114 storageClass=
"BrighterFatterKernel",
115 dimensions=[
"instrument",
"calibration_label",
"detector"],
117 defects = cT.PrerequisiteInput(
119 doc=
"Input defect tables.",
120 storageClass=
"DefectsList",
121 dimensions=[
"instrument",
"calibration_label",
"detector"],
123 opticsTransmission = cT.PrerequisiteInput(
124 name=
"transmission_optics",
125 storageClass=
"TransmissionCurve",
126 doc=
"Transmission curve due to the optics.",
127 dimensions=[
"instrument",
"calibration_label"],
129 filterTransmission = cT.PrerequisiteInput(
130 name=
"transmission_filter",
131 storageClass=
"TransmissionCurve",
132 doc=
"Transmission curve due to the filter.",
133 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
135 sensorTransmission = cT.PrerequisiteInput(
136 name=
"transmission_sensor",
137 storageClass=
"TransmissionCurve",
138 doc=
"Transmission curve due to the sensor.",
139 dimensions=[
"instrument",
"calibration_label",
"detector"],
141 atmosphereTransmission = cT.PrerequisiteInput(
142 name=
"transmission_atmosphere",
143 storageClass=
"TransmissionCurve",
144 doc=
"Transmission curve due to the atmosphere.",
145 dimensions=[
"instrument"],
147 illumMaskedImage = cT.PrerequisiteInput(
149 doc=
"Input illumination correction.",
150 storageClass=
"MaskedImageF",
151 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
154 outputExposure = cT.Output(
156 doc=
"Output ISR processed exposure.",
157 storageClass=
"ExposureF",
158 dimensions=[
"instrument",
"visit",
"detector"],
160 preInterpExposure = cT.Output(
161 name=
'preInterpISRCCD',
162 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
163 storageClass=
"ExposureF",
164 dimensions=[
"instrument",
"visit",
"detector"],
166 outputOssThumbnail = cT.Output(
168 doc=
"Output Overscan-subtracted thumbnail image.",
169 storageClass=
"Thumbnail",
170 dimensions=[
"instrument",
"visit",
"detector"],
172 outputFlattenedThumbnail = cT.Output(
173 name=
"FlattenedThumb",
174 doc=
"Output flat-corrected thumbnail image.",
175 storageClass=
"Thumbnail",
176 dimensions=[
"instrument",
"visit",
"detector"],
182 if config.doBias
is not True:
183 self.prerequisiteInputs.discard(
"bias")
184 if config.doLinearize
is not True:
185 self.prerequisiteInputs.discard(
"linearizer")
186 if config.doCrosstalk
is not True:
187 self.prerequisiteInputs.discard(
"crosstalkSources")
188 if config.doBrighterFatter
is not True:
189 self.prerequisiteInputs.discard(
"bfKernel")
190 self.prerequisiteInputs.discard(
"newBFKernel")
191 if config.doDefect
is not True:
192 self.prerequisiteInputs.discard(
"defects")
193 if config.doDark
is not True:
194 self.prerequisiteInputs.discard(
"dark")
195 if config.doFlat
is not True:
196 self.prerequisiteInputs.discard(
"flat")
197 if config.doAttachTransmissionCurve
is not True:
198 self.prerequisiteInputs.discard(
"opticsTransmission")
199 self.prerequisiteInputs.discard(
"filterTransmission")
200 self.prerequisiteInputs.discard(
"sensorTransmission")
201 self.prerequisiteInputs.discard(
"atmosphereTransmission")
202 if config.doUseOpticsTransmission
is not True:
203 self.prerequisiteInputs.discard(
"opticsTransmission")
204 if config.doUseFilterTransmission
is not True:
205 self.prerequisiteInputs.discard(
"filterTransmission")
206 if config.doUseSensorTransmission
is not True:
207 self.prerequisiteInputs.discard(
"sensorTransmission")
208 if config.doUseAtmosphereTransmission
is not True:
209 self.prerequisiteInputs.discard(
"atmosphereTransmission")
210 if config.doIlluminationCorrection
is not True:
211 self.prerequisiteInputs.discard(
"illumMaskedImage")
213 if config.doWrite
is not True:
214 self.outputs.discard(
"outputExposure")
215 self.outputs.discard(
"preInterpExposure")
216 self.outputs.discard(
"outputFlattenedThumbnail")
217 self.outputs.discard(
"outputOssThumbnail")
218 if config.doSaveInterpPixels
is not True:
219 self.outputs.discard(
"preInterpExposure")
220 if config.qa.doThumbnailOss
is not True:
221 self.outputs.discard(
"outputOssThumbnail")
222 if config.qa.doThumbnailFlattened
is not True:
223 self.outputs.discard(
"outputFlattenedThumbnail")
227 pipelineConnections=IsrTaskConnections):
228 """Configuration parameters for IsrTask. 230 Items are grouped in the order in which they are executed by the task. 232 datasetType = pexConfig.Field(
234 doc=
"Dataset type for input data; users will typically leave this alone, " 235 "but camera-specific ISR tasks will override it",
239 fallbackFilterName = pexConfig.Field(
241 doc=
"Fallback default filter name for calibrations.",
244 useFallbackDate = pexConfig.Field(
246 doc=
"Pass observation date when using fallback filter.",
249 expectWcs = pexConfig.Field(
252 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 254 fwhm = pexConfig.Field(
256 doc=
"FWHM of PSF in arcseconds.",
259 qa = pexConfig.ConfigField(
261 doc=
"QA related configuration options.",
265 doConvertIntToFloat = pexConfig.Field(
267 doc=
"Convert integer raw images to floating point values?",
272 doSaturation = pexConfig.Field(
274 doc=
"Mask saturated pixels? NB: this is totally independent of the" 275 " interpolation option - this is ONLY setting the bits in the mask." 276 " To have them interpolated make sure doSaturationInterpolation=True",
279 saturatedMaskName = pexConfig.Field(
281 doc=
"Name of mask plane to use in saturation detection and interpolation",
284 saturation = pexConfig.Field(
286 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
287 default=float(
"NaN"),
289 growSaturationFootprintSize = pexConfig.Field(
291 doc=
"Number of pixels by which to grow the saturation footprints",
296 doSuspect = pexConfig.Field(
298 doc=
"Mask suspect pixels?",
301 suspectMaskName = pexConfig.Field(
303 doc=
"Name of mask plane to use for suspect pixels",
306 numEdgeSuspect = pexConfig.Field(
308 doc=
"Number of edge pixels to be flagged as untrustworthy.",
313 doSetBadRegions = pexConfig.Field(
315 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
318 badStatistic = pexConfig.ChoiceField(
320 doc=
"How to estimate the average value for BAD regions.",
323 "MEANCLIP":
"Correct using the (clipped) mean of good data",
324 "MEDIAN":
"Correct using the median of the good data",
329 doOverscan = pexConfig.Field(
331 doc=
"Do overscan subtraction?",
334 overscanFitType = pexConfig.ChoiceField(
336 doc=
"The method for fitting the overscan bias level.",
339 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
340 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
341 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
342 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
343 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
344 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
345 "MEAN":
"Correct using the mean of the overscan region",
346 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
347 "MEDIAN":
"Correct using the median of the overscan region",
350 overscanOrder = pexConfig.Field(
352 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
353 "or number of spline knots if overscan fit type is a spline."),
356 overscanNumSigmaClip = pexConfig.Field(
358 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
361 overscanIsInt = pexConfig.Field(
363 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
366 overscanNumLeadingColumnsToSkip = pexConfig.Field(
368 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
371 overscanNumTrailingColumnsToSkip = pexConfig.Field(
373 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
376 overscanMaxDev = pexConfig.Field(
378 doc=
"Maximum deviation from the median for overscan",
379 default=1000.0, check=
lambda x: x > 0
381 overscanBiasJump = pexConfig.Field(
383 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
386 overscanBiasJumpKeyword = pexConfig.Field(
388 doc=
"Header keyword containing information about devices.",
389 default=
"NO_SUCH_KEY",
391 overscanBiasJumpDevices = pexConfig.ListField(
393 doc=
"List of devices that need piecewise overscan correction.",
396 overscanBiasJumpLocation = pexConfig.Field(
398 doc=
"Location of bias jump along y-axis.",
403 doAssembleCcd = pexConfig.Field(
406 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 408 assembleCcd = pexConfig.ConfigurableField(
409 target=AssembleCcdTask,
410 doc=
"CCD assembly task",
414 doAssembleIsrExposures = pexConfig.Field(
417 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 419 doTrimToMatchCalib = pexConfig.Field(
422 doc=
"Trim raw data to match calibration bounding boxes?" 426 doBias = pexConfig.Field(
428 doc=
"Apply bias frame correction?",
431 biasDataProductName = pexConfig.Field(
433 doc=
"Name of the bias data product",
438 doVariance = pexConfig.Field(
440 doc=
"Calculate variance?",
443 gain = pexConfig.Field(
445 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
446 default=float(
"NaN"),
448 readNoise = pexConfig.Field(
450 doc=
"The read noise to use if no Detector is present in the Exposure",
453 doEmpiricalReadNoise = pexConfig.Field(
456 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 460 doLinearize = pexConfig.Field(
462 doc=
"Correct for nonlinearity of the detector's response?",
467 doCrosstalk = pexConfig.Field(
469 doc=
"Apply intra-CCD crosstalk correction?",
472 doCrosstalkBeforeAssemble = pexConfig.Field(
474 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
477 crosstalk = pexConfig.ConfigurableField(
478 target=CrosstalkTask,
479 doc=
"Intra-CCD crosstalk correction",
483 doDefect = pexConfig.Field(
485 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
488 doNanMasking = pexConfig.Field(
490 doc=
"Mask NAN pixels?",
493 doWidenSaturationTrails = pexConfig.Field(
495 doc=
"Widen bleed trails based on their width?",
500 doBrighterFatter = pexConfig.Field(
503 doc=
"Apply the brighter fatter correction" 505 brighterFatterLevel = pexConfig.ChoiceField(
508 doc=
"The level at which to correct for brighter-fatter.",
510 "AMP":
"Every amplifier treated separately.",
511 "DETECTOR":
"One kernel per detector",
514 brighterFatterMaxIter = pexConfig.Field(
517 doc=
"Maximum number of iterations for the brighter fatter correction" 519 brighterFatterThreshold = pexConfig.Field(
522 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 523 " absolute value of the difference between the current corrected image and the one" 524 " from the previous iteration summed over all the pixels." 526 brighterFatterApplyGain = pexConfig.Field(
529 doc=
"Should the gain be applied when applying the brighter fatter correction?" 531 brighterFatterMaskGrowSize = pexConfig.Field(
534 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate " 535 " when brighter-fatter correction is applied." 539 doDark = pexConfig.Field(
541 doc=
"Apply dark frame correction?",
544 darkDataProductName = pexConfig.Field(
546 doc=
"Name of the dark data product",
551 doStrayLight = pexConfig.Field(
553 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
556 strayLight = pexConfig.ConfigurableField(
557 target=StrayLightTask,
558 doc=
"y-band stray light correction" 562 doFlat = pexConfig.Field(
564 doc=
"Apply flat field correction?",
567 flatDataProductName = pexConfig.Field(
569 doc=
"Name of the flat data product",
572 flatScalingType = pexConfig.ChoiceField(
574 doc=
"The method for scaling the flat on the fly.",
577 "USER":
"Scale by flatUserScale",
578 "MEAN":
"Scale by the inverse of the mean",
579 "MEDIAN":
"Scale by the inverse of the median",
582 flatUserScale = pexConfig.Field(
584 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
587 doTweakFlat = pexConfig.Field(
589 doc=
"Tweak flats to match observed amplifier ratios?",
594 doApplyGains = pexConfig.Field(
596 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
599 normalizeGains = pexConfig.Field(
601 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
606 doFringe = pexConfig.Field(
608 doc=
"Apply fringe correction?",
611 fringe = pexConfig.ConfigurableField(
613 doc=
"Fringe subtraction task",
615 fringeAfterFlat = pexConfig.Field(
617 doc=
"Do fringe subtraction after flat-fielding?",
622 doAddDistortionModel = pexConfig.Field(
624 doc=
"Apply a distortion model based on camera geometry to the WCS?",
626 deprecated=(
"Camera geometry is incorporated when reading the raw files." 627 " This option no longer is used, and will be removed after v19.")
631 doMeasureBackground = pexConfig.Field(
633 doc=
"Measure the background level on the reduced image?",
638 doCameraSpecificMasking = pexConfig.Field(
640 doc=
"Mask camera-specific bad regions?",
643 masking = pexConfig.ConfigurableField(
650 doInterpolate = pexConfig.Field(
652 doc=
"Interpolate masked pixels?",
655 doSaturationInterpolation = pexConfig.Field(
657 doc=
"Perform interpolation over pixels masked as saturated?" 658 " NB: This is independent of doSaturation; if that is False this plane" 659 " will likely be blank, resulting in a no-op here.",
662 doNanInterpolation = pexConfig.Field(
664 doc=
"Perform interpolation over pixels masked as NaN?" 665 " NB: This is independent of doNanMasking; if that is False this plane" 666 " will likely be blank, resulting in a no-op here.",
669 doNanInterpAfterFlat = pexConfig.Field(
671 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 672 "also have to interpolate them before flat-fielding."),
675 maskListToInterpolate = pexConfig.ListField(
677 doc=
"List of mask planes that should be interpolated.",
678 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
680 doSaveInterpPixels = pexConfig.Field(
682 doc=
"Save a copy of the pre-interpolated pixel values?",
687 fluxMag0T1 = pexConfig.DictField(
690 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
691 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
694 defaultFluxMag0T1 = pexConfig.Field(
696 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
697 default=pow(10.0, 0.4*28.0)
701 doVignette = pexConfig.Field(
703 doc=
"Apply vignetting parameters?",
706 vignette = pexConfig.ConfigurableField(
708 doc=
"Vignetting task.",
712 doAttachTransmissionCurve = pexConfig.Field(
715 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 717 doUseOpticsTransmission = pexConfig.Field(
720 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 722 doUseFilterTransmission = pexConfig.Field(
725 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 727 doUseSensorTransmission = pexConfig.Field(
730 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 732 doUseAtmosphereTransmission = pexConfig.Field(
735 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 739 doIlluminationCorrection = pexConfig.Field(
742 doc=
"Perform illumination correction?" 744 illuminationCorrectionDataProductName = pexConfig.Field(
746 doc=
"Name of the illumination correction data product.",
749 illumScale = pexConfig.Field(
751 doc=
"Scale factor for the illumination correction.",
754 illumFilters = pexConfig.ListField(
757 doc=
"Only perform illumination correction for these filters." 761 doWrite = pexConfig.Field(
763 doc=
"Persist postISRCCD?",
770 raise ValueError(
"You may not specify both doFlat and doApplyGains")
772 self.config.maskListToInterpolate.append(
"SAT")
774 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
777 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
778 """Apply common instrument signature correction algorithms to a raw frame. 780 The process for correcting imaging data is very similar from 781 camera to camera. This task provides a vanilla implementation of 782 doing these corrections, including the ability to turn certain 783 corrections off if they are not needed. The inputs to the primary 784 method, `run()`, are a raw exposure to be corrected and the 785 calibration data products. The raw input is a single chip sized 786 mosaic of all amps including overscans and other non-science 787 pixels. The method `runDataRef()` identifies and defines the 788 calibration data products, and is intended for use by a 789 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 790 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 791 subclassed for different camera, although the most camera specific 792 methods have been split into subtasks that can be redirected 795 The __init__ method sets up the subtasks for ISR processing, using 796 the defaults from `lsst.ip.isr`. 801 Positional arguments passed to the Task constructor. None used at this time. 802 kwargs : `dict`, optional 803 Keyword arguments passed on to the Task constructor. None used at this time. 805 ConfigClass = IsrTaskConfig
810 self.makeSubtask(
"assembleCcd")
811 self.makeSubtask(
"crosstalk")
812 self.makeSubtask(
"strayLight")
813 self.makeSubtask(
"fringe")
814 self.makeSubtask(
"masking")
815 self.makeSubtask(
"vignette")
818 inputs = butlerQC.get(inputRefs)
821 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
822 except Exception
as e:
823 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
826 inputs[
'isGen3'] =
True 828 detector = inputs[
'ccdExposure'].getDetector()
830 if 'linearizer' not in inputs:
831 linearityName = detector.getAmplifiers()[0].getLinearityType()
832 inputs[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
834 if self.config.doDefect
is True:
835 if "defects" in inputs
and inputs[
'defects']
is not None:
838 if not isinstance(inputs[
"defects"], Defects):
839 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
843 if self.config.doBrighterFatter:
844 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
845 if brighterFatterKernel
is None:
846 brighterFatterKernel = inputs.get(
'bfKernel',
None)
848 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
849 detId = detector.getId()
850 inputs[
'bfGains'] = brighterFatterKernel.gain
853 if self.config.brighterFatterLevel ==
'DETECTOR':
854 if brighterFatterKernel.detectorKernel:
855 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
856 elif brighterFatterKernel.detectorKernelFromAmpKernels:
857 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
859 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
862 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
870 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
871 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
872 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
874 assembler=self.assembleCcd
875 if self.config.doAssembleIsrExposures
else None)
877 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
879 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
880 if 'strayLightData' not in inputs:
881 inputs[
'strayLightData'] =
None 883 outputs = self.
run(**inputs)
884 butlerQC.put(outputs, outputRefs)
887 """!Retrieve necessary frames for instrument signature removal. 889 Pre-fetching all required ISR data products limits the IO 890 required by the ISR. Any conflict between the calibration data 891 available and that needed for ISR is also detected prior to 892 doing processing, allowing it to fail quickly. 896 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 897 Butler reference of the detector data to be processed 898 rawExposure : `afw.image.Exposure` 899 The raw exposure that will later be corrected with the 900 retrieved calibration data; should not be modified in this 905 result : `lsst.pipe.base.Struct` 906 Result struct with components (which may be `None`): 907 - ``bias``: bias calibration frame (`afw.image.Exposure`) 908 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 909 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 910 - ``dark``: dark calibration frame (`afw.image.Exposure`) 911 - ``flat``: flat calibration frame (`afw.image.Exposure`) 912 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 913 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 914 - ``fringes``: `lsst.pipe.base.Struct` with components: 915 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 916 - ``seed``: random seed derived from the ccdExposureId for random 917 number generator (`uint32`). 918 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 919 A ``TransmissionCurve`` that represents the throughput of the optics, 920 to be evaluated in focal-plane coordinates. 921 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 922 A ``TransmissionCurve`` that represents the throughput of the filter 923 itself, to be evaluated in focal-plane coordinates. 924 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 925 A ``TransmissionCurve`` that represents the throughput of the sensor 926 itself, to be evaluated in post-assembly trimmed detector coordinates. 927 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 928 A ``TransmissionCurve`` that represents the throughput of the 929 atmosphere, assumed to be spatially constant. 930 - ``strayLightData`` : `object` 931 An opaque object containing calibration information for 932 stray-light correction. If `None`, no correction will be 934 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 938 NotImplementedError : 939 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 942 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
943 dateObs = dateObs.toPython().isoformat()
945 self.log.warn(
"Unable to identify dateObs for rawExposure.")
948 ccd = rawExposure.getDetector()
949 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
950 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
951 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
952 if self.config.doBias
else None)
954 linearizer = (dataRef.get(
"linearizer", immediate=
True)
956 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
957 if self.config.doCrosstalk
else None)
958 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
959 if self.config.doDark
else None)
960 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
962 if self.config.doFlat
else None)
964 brighterFatterKernel =
None 965 brighterFatterGains =
None 966 if self.config.doBrighterFatter
is True:
971 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
972 brighterFatterGains = brighterFatterKernel.gain
973 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
976 brighterFatterKernel = dataRef.get(
"bfKernel")
977 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
979 brighterFatterKernel =
None 980 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
983 if self.config.brighterFatterLevel ==
'DETECTOR':
984 if brighterFatterKernel.detectorKernel:
985 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
986 elif brighterFatterKernel.detectorKernelFromAmpKernels:
987 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
989 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
992 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
994 defectList = (dataRef.get(
"defects")
995 if self.config.doDefect
else None)
996 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
997 if self.config.doAssembleIsrExposures
else None)
998 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
999 else pipeBase.Struct(fringes=
None))
1001 if self.config.doAttachTransmissionCurve:
1002 opticsTransmission = (dataRef.get(
"transmission_optics")
1003 if self.config.doUseOpticsTransmission
else None)
1004 filterTransmission = (dataRef.get(
"transmission_filter")
1005 if self.config.doUseFilterTransmission
else None)
1006 sensorTransmission = (dataRef.get(
"transmission_sensor")
1007 if self.config.doUseSensorTransmission
else None)
1008 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1009 if self.config.doUseAtmosphereTransmission
else None)
1011 opticsTransmission =
None 1012 filterTransmission =
None 1013 sensorTransmission =
None 1014 atmosphereTransmission =
None 1016 if self.config.doStrayLight:
1017 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1019 strayLightData =
None 1022 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1023 if (self.config.doIlluminationCorrection
and 1024 filterName
in self.config.illumFilters)
1028 return pipeBase.Struct(bias=biasExposure,
1029 linearizer=linearizer,
1030 crosstalkSources=crosstalkSources,
1033 bfKernel=brighterFatterKernel,
1034 bfGains=brighterFatterGains,
1036 fringes=fringeStruct,
1037 opticsTransmission=opticsTransmission,
1038 filterTransmission=filterTransmission,
1039 sensorTransmission=sensorTransmission,
1040 atmosphereTransmission=atmosphereTransmission,
1041 strayLightData=strayLightData,
1042 illumMaskedImage=illumMaskedImage
1045 @pipeBase.timeMethod
1046 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1047 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1048 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1049 sensorTransmission=
None, atmosphereTransmission=
None,
1050 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1053 """!Perform instrument signature removal on an exposure. 1055 Steps included in the ISR processing, in order performed, are: 1056 - saturation and suspect pixel masking 1057 - overscan subtraction 1058 - CCD assembly of individual amplifiers 1060 - variance image construction 1061 - linearization of non-linear response 1063 - brighter-fatter correction 1066 - stray light subtraction 1068 - masking of known defects and camera specific features 1069 - vignette calculation 1070 - appending transmission curve and distortion model 1074 ccdExposure : `lsst.afw.image.Exposure` 1075 The raw exposure that is to be run through ISR. The 1076 exposure is modified by this method. 1077 camera : `lsst.afw.cameraGeom.Camera`, optional 1078 The camera geometry for this exposure. Used to select the 1079 distortion model appropriate for this data. 1080 bias : `lsst.afw.image.Exposure`, optional 1081 Bias calibration frame. 1082 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 1083 Functor for linearization. 1084 crosstalkSources : `list`, optional 1085 List of possible crosstalk sources. 1086 dark : `lsst.afw.image.Exposure`, optional 1087 Dark calibration frame. 1088 flat : `lsst.afw.image.Exposure`, optional 1089 Flat calibration frame. 1090 bfKernel : `numpy.ndarray`, optional 1091 Brighter-fatter kernel. 1092 bfGains : `dict` of `float`, optional 1093 Gains used to override the detector's nominal gains for the 1094 brighter-fatter correction. A dict keyed by amplifier name for 1095 the detector in question. 1096 defects : `lsst.meas.algorithms.Defects`, optional 1098 fringes : `lsst.pipe.base.Struct`, optional 1099 Struct containing the fringe correction data, with 1101 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 1102 - ``seed``: random seed derived from the ccdExposureId for random 1103 number generator (`uint32`) 1104 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 1105 A ``TransmissionCurve`` that represents the throughput of the optics, 1106 to be evaluated in focal-plane coordinates. 1107 filterTransmission : `lsst.afw.image.TransmissionCurve` 1108 A ``TransmissionCurve`` that represents the throughput of the filter 1109 itself, to be evaluated in focal-plane coordinates. 1110 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1111 A ``TransmissionCurve`` that represents the throughput of the sensor 1112 itself, to be evaluated in post-assembly trimmed detector coordinates. 1113 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1114 A ``TransmissionCurve`` that represents the throughput of the 1115 atmosphere, assumed to be spatially constant. 1116 detectorNum : `int`, optional 1117 The integer number for the detector to process. 1118 isGen3 : bool, optional 1119 Flag this call to run() as using the Gen3 butler environment. 1120 strayLightData : `object`, optional 1121 Opaque object containing calibration information for stray-light 1122 correction. If `None`, no correction will be performed. 1123 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 1124 Illumination correction image. 1128 result : `lsst.pipe.base.Struct` 1129 Result struct with component: 1130 - ``exposure`` : `afw.image.Exposure` 1131 The fully ISR corrected exposure. 1132 - ``outputExposure`` : `afw.image.Exposure` 1133 An alias for `exposure` 1134 - ``ossThumb`` : `numpy.ndarray` 1135 Thumbnail image of the exposure after overscan subtraction. 1136 - ``flattenedThumb`` : `numpy.ndarray` 1137 Thumbnail image of the exposure after flat-field correction. 1142 Raised if a configuration option is set to True, but the 1143 required calibration data has not been specified. 1147 The current processed exposure can be viewed by setting the 1148 appropriate lsstDebug entries in the `debug.display` 1149 dictionary. The names of these entries correspond to some of 1150 the IsrTaskConfig Boolean options, with the value denoting the 1151 frame to use. The exposure is shown inside the matching 1152 option check and after the processing of that step has 1153 finished. The steps with debug points are: 1164 In addition, setting the "postISRCCD" entry displays the 1165 exposure after all ISR processing has finished. 1173 if detectorNum
is None:
1174 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1176 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1181 if isinstance(ccdExposure, ButlerDataRef):
1184 ccd = ccdExposure.getDetector()
1185 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1188 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd." 1189 ccd = [
FakeAmp(ccdExposure, self.config)]
1192 if self.config.doBias
and bias
is None:
1193 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1195 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1196 if self.config.doBrighterFatter
and bfKernel
is None:
1197 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1198 if self.config.doDark
and dark
is None:
1199 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1200 if self.config.doFlat
and flat
is None:
1201 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1202 if self.config.doDefect
and defects
is None:
1203 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1204 if (self.config.doFringe
and filterName
in self.fringe.config.filters
and 1205 fringes.fringes
is None):
1210 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1211 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
and 1212 illumMaskedImage
is None):
1213 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1216 if self.config.doConvertIntToFloat:
1217 self.log.info(
"Converting exposure to floating point values.")
1224 if ccdExposure.getBBox().contains(amp.getBBox()):
1228 if self.config.doOverscan
and not badAmp:
1231 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1232 if overscanResults
is not None and \
1233 self.config.qa
is not None and self.config.qa.saveStats
is True:
1234 if isinstance(overscanResults.overscanFit, float):
1235 qaMedian = overscanResults.overscanFit
1236 qaStdev = float(
"NaN")
1238 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1239 afwMath.MEDIAN | afwMath.STDEVCLIP)
1240 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1241 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1243 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1244 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1245 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1246 amp.getName(), qaMedian, qaStdev)
1247 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1250 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1251 overscanResults =
None 1253 overscans.append(overscanResults
if overscanResults
is not None else None)
1255 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1257 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1258 self.log.info(
"Applying crosstalk correction.")
1259 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1260 self.
debugView(ccdExposure,
"doCrosstalk")
1262 if self.config.doAssembleCcd:
1263 self.log.info(
"Assembling CCD from amplifiers.")
1264 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1266 if self.config.expectWcs
and not ccdExposure.getWcs():
1267 self.log.warn(
"No WCS found in input exposure.")
1268 self.
debugView(ccdExposure,
"doAssembleCcd")
1271 if self.config.qa.doThumbnailOss:
1272 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1274 if self.config.doBias:
1275 self.log.info(
"Applying bias correction.")
1276 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1277 trimToFit=self.config.doTrimToMatchCalib)
1280 if self.config.doVariance:
1281 for amp, overscanResults
in zip(ccd, overscans):
1282 if ccdExposure.getBBox().contains(amp.getBBox()):
1283 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1284 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1285 if overscanResults
is not None:
1287 overscanImage=overscanResults.overscanImage)
1291 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1292 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1293 afwMath.MEDIAN | afwMath.STDEVCLIP)
1294 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1295 qaStats.getValue(afwMath.MEDIAN))
1296 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1297 qaStats.getValue(afwMath.STDEVCLIP))
1298 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1299 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1300 qaStats.getValue(afwMath.STDEVCLIP))
1303 self.log.info(
"Applying linearizer.")
1304 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1306 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1307 self.log.info(
"Applying crosstalk correction.")
1308 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=
True)
1309 self.
debugView(ccdExposure,
"doCrosstalk")
1313 if self.config.doDefect:
1314 self.log.info(
"Masking defects.")
1317 if self.config.numEdgeSuspect > 0:
1318 self.log.info(
"Masking edges as SUSPECT.")
1319 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1320 maskPlane=
"SUSPECT")
1322 if self.config.doNanMasking:
1323 self.log.info(
"Masking NAN value pixels.")
1326 if self.config.doWidenSaturationTrails:
1327 self.log.info(
"Widening saturation trails.")
1328 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1330 if self.config.doCameraSpecificMasking:
1331 self.log.info(
"Masking regions for camera specific reasons.")
1332 self.masking.
run(ccdExposure)
1334 if self.config.doBrighterFatter:
1343 interpExp = ccdExposure.clone()
1345 isrFunctions.interpolateFromMask(
1346 maskedImage=interpExp.getMaskedImage(),
1347 fwhm=self.config.fwhm,
1348 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1349 maskNameList=self.config.maskListToInterpolate
1351 bfExp = interpExp.clone()
1353 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1354 type(bfKernel), type(bfGains))
1355 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1356 self.config.brighterFatterMaxIter,
1357 self.config.brighterFatterThreshold,
1358 self.config.brighterFatterApplyGain,
1360 if bfResults[1] == self.config.brighterFatterMaxIter:
1361 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1364 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1366 image = ccdExposure.getMaskedImage().getImage()
1367 bfCorr = bfExp.getMaskedImage().getImage()
1368 bfCorr -= interpExp.getMaskedImage().getImage()
1377 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1378 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1381 if self.config.brighterFatterMaskGrowSize > 0:
1382 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1383 for maskPlane
in self.config.maskListToInterpolate:
1384 isrFunctions.growMasks(ccdExposure.getMask(),
1385 radius=self.config.brighterFatterMaskGrowSize,
1386 maskNameList=maskPlane,
1387 maskValue=maskPlane)
1389 self.
debugView(ccdExposure,
"doBrighterFatter")
1391 if self.config.doDark:
1392 self.log.info(
"Applying dark correction.")
1396 if self.config.doFringe
and not self.config.fringeAfterFlat:
1397 self.log.info(
"Applying fringe correction before flat.")
1398 self.fringe.
run(ccdExposure, **fringes.getDict())
1401 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1402 self.log.info(
"Checking strayLight correction.")
1403 self.strayLight.
run(ccdExposure, strayLightData)
1404 self.
debugView(ccdExposure,
"doStrayLight")
1406 if self.config.doFlat:
1407 self.log.info(
"Applying flat correction.")
1411 if self.config.doApplyGains:
1412 self.log.info(
"Applying gain correction instead of flat.")
1413 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1415 if self.config.doFringe
and self.config.fringeAfterFlat:
1416 self.log.info(
"Applying fringe correction after flat.")
1417 self.fringe.
run(ccdExposure, **fringes.getDict())
1419 if self.config.doVignette:
1420 self.log.info(
"Constructing Vignette polygon.")
1423 if self.config.vignette.doWriteVignettePolygon:
1426 if self.config.doAttachTransmissionCurve:
1427 self.log.info(
"Adding transmission curves.")
1428 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1429 filterTransmission=filterTransmission,
1430 sensorTransmission=sensorTransmission,
1431 atmosphereTransmission=atmosphereTransmission)
1433 flattenedThumb =
None 1434 if self.config.qa.doThumbnailFlattened:
1435 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1437 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1438 self.log.info(
"Performing illumination correction.")
1439 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1440 illumMaskedImage, illumScale=self.config.illumScale,
1441 trimToFit=self.config.doTrimToMatchCalib)
1444 if self.config.doSaveInterpPixels:
1445 preInterpExp = ccdExposure.clone()
1460 if self.config.doSetBadRegions:
1461 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1462 if badPixelCount > 0:
1463 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1465 if self.config.doInterpolate:
1466 self.log.info(
"Interpolating masked pixels.")
1467 isrFunctions.interpolateFromMask(
1468 maskedImage=ccdExposure.getMaskedImage(),
1469 fwhm=self.config.fwhm,
1470 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1471 maskNameList=list(self.config.maskListToInterpolate)
1476 if self.config.doMeasureBackground:
1477 self.log.info(
"Measuring background level.")
1480 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1482 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1483 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1484 afwMath.MEDIAN | afwMath.STDEVCLIP)
1485 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1486 qaStats.getValue(afwMath.MEDIAN))
1487 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1488 qaStats.getValue(afwMath.STDEVCLIP))
1489 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1490 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1491 qaStats.getValue(afwMath.STDEVCLIP))
1493 self.
debugView(ccdExposure,
"postISRCCD")
1495 return pipeBase.Struct(
1496 exposure=ccdExposure,
1498 flattenedThumb=flattenedThumb,
1500 preInterpolatedExposure=preInterpExp,
1501 outputExposure=ccdExposure,
1502 outputOssThumbnail=ossThumb,
1503 outputFlattenedThumbnail=flattenedThumb,
1506 @pipeBase.timeMethod
1508 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1510 This method contains the `CmdLineTask` interface to the ISR 1511 processing. All IO is handled here, freeing the `run()` method 1512 to manage only pixel-level calculations. The steps performed 1514 - Read in necessary detrending/isr/calibration data. 1515 - Process raw exposure in `run()`. 1516 - Persist the ISR-corrected exposure as "postISRCCD" if 1517 config.doWrite=True. 1521 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1522 DataRef of the detector data to be processed 1526 result : `lsst.pipe.base.Struct` 1527 Result struct with component: 1528 - ``exposure`` : `afw.image.Exposure` 1529 The fully ISR corrected exposure. 1534 Raised if a configuration option is set to True, but the 1535 required calibration data does not exist. 1538 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1540 ccdExposure = sensorRef.get(self.config.datasetType)
1542 camera = sensorRef.get(
"camera")
1543 isrData = self.
readIsrData(sensorRef, ccdExposure)
1545 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1547 if self.config.doWrite:
1548 sensorRef.put(result.exposure,
"postISRCCD")
1549 if result.preInterpolatedExposure
is not None:
1550 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1551 if result.ossThumb
is not None:
1552 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1553 if result.flattenedThumb
is not None:
1554 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1559 """!Retrieve a calibration dataset for removing instrument signature. 1564 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1565 DataRef of the detector data to find calibration datasets 1568 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1569 dateObs : `str`, optional 1570 Date of the observation. Used to correct butler failures 1571 when using fallback filters. 1573 If True, disable butler proxies to enable error handling 1574 within this routine. 1578 exposure : `lsst.afw.image.Exposure` 1579 Requested calibration frame. 1584 Raised if no matching calibration frame can be found. 1587 exp = dataRef.get(datasetType, immediate=immediate)
1588 except Exception
as exc1:
1589 if not self.config.fallbackFilterName:
1590 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1592 if self.config.useFallbackDate
and dateObs:
1593 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1594 dateObs=dateObs, immediate=immediate)
1596 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1597 except Exception
as exc2:
1598 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1599 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1600 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1602 if self.config.doAssembleIsrExposures:
1603 exp = self.assembleCcd.assembleCcd(exp)
1607 """Ensure that the data returned by Butler is a fully constructed exposure. 1609 ISR requires exposure-level image data for historical reasons, so if we did 1610 not recieve that from Butler, construct it from what we have, modifying the 1615 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1616 `lsst.afw.image.ImageF` 1617 The input data structure obtained from Butler. 1618 camera : `lsst.afw.cameraGeom.camera` 1619 The camera associated with the image. Used to find the appropriate 1622 The detector this exposure should match. 1626 inputExp : `lsst.afw.image.Exposure` 1627 The re-constructed exposure, with appropriate detector parameters. 1632 Raised if the input data cannot be used to construct an exposure. 1634 if isinstance(inputExp, afwImage.DecoratedImageU):
1635 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1636 elif isinstance(inputExp, afwImage.ImageF):
1637 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1638 elif isinstance(inputExp, afwImage.MaskedImageF):
1639 inputExp = afwImage.makeExposure(inputExp)
1640 elif isinstance(inputExp, afwImage.Exposure):
1642 elif inputExp
is None:
1646 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1649 if inputExp.getDetector()
is None:
1650 inputExp.setDetector(camera[detectorNum])
1655 """Convert exposure image from uint16 to float. 1657 If the exposure does not need to be converted, the input is 1658 immediately returned. For exposures that are converted to use 1659 floating point pixels, the variance is set to unity and the 1664 exposure : `lsst.afw.image.Exposure` 1665 The raw exposure to be converted. 1669 newexposure : `lsst.afw.image.Exposure` 1670 The input ``exposure``, converted to floating point pixels. 1675 Raised if the exposure type cannot be converted to float. 1678 if isinstance(exposure, afwImage.ExposureF):
1680 self.log.debug(
"Exposure already of type float.")
1682 if not hasattr(exposure,
"convertF"):
1683 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1685 newexposure = exposure.convertF()
1686 newexposure.variance[:] = 1
1687 newexposure.mask[:] = 0x0
1692 """Identify bad amplifiers, saturated and suspect pixels. 1696 ccdExposure : `lsst.afw.image.Exposure` 1697 Input exposure to be masked. 1698 amp : `lsst.afw.table.AmpInfoCatalog` 1699 Catalog of parameters defining the amplifier on this 1701 defects : `lsst.meas.algorithms.Defects` 1702 List of defects. Used to determine if the entire 1708 If this is true, the entire amplifier area is covered by 1709 defects and unusable. 1712 maskedImage = ccdExposure.getMaskedImage()
1718 if defects
is not None:
1719 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1724 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1726 maskView = dataView.getMask()
1727 maskView |= maskView.getPlaneBitMask(
"BAD")
1734 if self.config.doSaturation
and not badAmp:
1735 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1736 if self.config.doSuspect
and not badAmp:
1737 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1738 if math.isfinite(self.config.saturation):
1739 limits.update({self.config.saturatedMaskName: self.config.saturation})
1741 for maskName, maskThreshold
in limits.items():
1742 if not math.isnan(maskThreshold):
1743 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1744 isrFunctions.makeThresholdMask(
1745 maskedImage=dataView,
1746 threshold=maskThreshold,
1752 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1754 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1755 self.config.suspectMaskName])
1756 if numpy.all(maskView.getArray() & maskVal > 0):
1758 maskView |= maskView.getPlaneBitMask(
"BAD")
1763 """Apply overscan correction in place. 1765 This method does initial pixel rejection of the overscan 1766 region. The overscan can also be optionally segmented to 1767 allow for discontinuous overscan responses to be fit 1768 separately. The actual overscan subtraction is performed by 1769 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1770 which is called here after the amplifier is preprocessed. 1774 ccdExposure : `lsst.afw.image.Exposure` 1775 Exposure to have overscan correction performed. 1776 amp : `lsst.afw.table.AmpInfoCatalog` 1777 The amplifier to consider while correcting the overscan. 1781 overscanResults : `lsst.pipe.base.Struct` 1782 Result struct with components: 1783 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1784 Value or fit subtracted from the amplifier image data. 1785 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1786 Value or fit subtracted from the overscan image data. 1787 - ``overscanImage`` : `lsst.afw.image.Image` 1788 Image of the overscan region with the overscan 1789 correction applied. This quantity is used to estimate 1790 the amplifier read noise empirically. 1795 Raised if the ``amp`` does not contain raw pixel information. 1799 lsst.ip.isr.isrFunctions.overscanCorrection 1801 if not amp.getHasRawInfo():
1802 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1804 if amp.getRawHorizontalOverscanBBox().isEmpty():
1805 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1808 statControl = afwMath.StatisticsControl()
1809 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1812 dataBBox = amp.getRawDataBBox()
1813 oscanBBox = amp.getRawHorizontalOverscanBBox()
1817 prescanBBox = amp.getRawPrescanBBox()
1818 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1819 dx0 += self.config.overscanNumLeadingColumnsToSkip
1820 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1822 dx0 += self.config.overscanNumTrailingColumnsToSkip
1823 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1829 if ((self.config.overscanBiasJump
and 1830 self.config.overscanBiasJumpLocation)
and 1831 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1832 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1833 self.config.overscanBiasJumpDevices)):
1834 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1835 yLower = self.config.overscanBiasJumpLocation
1836 yUpper = dataBBox.getHeight() - yLower
1838 yUpper = self.config.overscanBiasJumpLocation
1839 yLower = dataBBox.getHeight() - yUpper
1858 oscanBBox.getHeight())))
1861 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1862 ampImage = ccdExposure.maskedImage[imageBBox]
1863 overscanImage = ccdExposure.maskedImage[overscanBBox]
1865 overscanArray = overscanImage.image.array
1866 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1867 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1868 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1870 statControl = afwMath.StatisticsControl()
1871 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1873 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1874 overscanImage=overscanImage,
1875 fitType=self.config.overscanFitType,
1876 order=self.config.overscanOrder,
1877 collapseRej=self.config.overscanNumSigmaClip,
1878 statControl=statControl,
1879 overscanIsInt=self.config.overscanIsInt
1883 levelStat = afwMath.MEDIAN
1884 sigmaStat = afwMath.STDEVCLIP
1886 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1887 self.config.qa.flatness.nIter)
1888 metadata = ccdExposure.getMetadata()
1889 ampNum = amp.getName()
1890 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1891 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1892 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1894 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1895 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1896 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1898 return overscanResults
1901 """Set the variance plane using the amplifier gain and read noise 1903 The read noise is calculated from the ``overscanImage`` if the 1904 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1905 the value from the amplifier data is used. 1909 ampExposure : `lsst.afw.image.Exposure` 1910 Exposure to process. 1911 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1912 Amplifier detector data. 1913 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1914 Image of overscan, required only for empirical read noise. 1918 lsst.ip.isr.isrFunctions.updateVariance 1920 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1921 gain = amp.getGain()
1923 if math.isnan(gain):
1925 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1928 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
1929 amp.getName(), gain, patchedGain)
1932 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1933 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
1935 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1936 stats = afwMath.StatisticsControl()
1937 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1938 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1939 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
1940 amp.getName(), readNoise)
1942 readNoise = amp.getReadNoise()
1944 isrFunctions.updateVariance(
1945 maskedImage=ampExposure.getMaskedImage(),
1947 readNoise=readNoise,
1951 """!Apply dark correction in place. 1955 exposure : `lsst.afw.image.Exposure` 1956 Exposure to process. 1957 darkExposure : `lsst.afw.image.Exposure` 1958 Dark exposure of the same size as ``exposure``. 1959 invert : `Bool`, optional 1960 If True, re-add the dark to an already corrected image. 1965 Raised if either ``exposure`` or ``darkExposure`` do not 1966 have their dark time defined. 1970 lsst.ip.isr.isrFunctions.darkCorrection 1972 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1973 if math.isnan(expScale):
1974 raise RuntimeError(
"Exposure darktime is NAN.")
1975 if darkExposure.getInfo().getVisitInfo()
is not None:
1976 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1980 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1983 if math.isnan(darkScale):
1984 raise RuntimeError(
"Dark calib darktime is NAN.")
1985 isrFunctions.darkCorrection(
1986 maskedImage=exposure.getMaskedImage(),
1987 darkMaskedImage=darkExposure.getMaskedImage(),
1989 darkScale=darkScale,
1991 trimToFit=self.config.doTrimToMatchCalib
1995 """!Check if linearization is needed for the detector cameraGeom. 1997 Checks config.doLinearize and the linearity type of the first 2002 detector : `lsst.afw.cameraGeom.Detector` 2003 Detector to get linearity type from. 2007 doLinearize : `Bool` 2008 If True, linearization should be performed. 2010 return self.config.doLinearize
and \
2011 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2014 """!Apply flat correction in place. 2018 exposure : `lsst.afw.image.Exposure` 2019 Exposure to process. 2020 flatExposure : `lsst.afw.image.Exposure` 2021 Flat exposure of the same size as ``exposure``. 2022 invert : `Bool`, optional 2023 If True, unflatten an already flattened image. 2027 lsst.ip.isr.isrFunctions.flatCorrection 2029 isrFunctions.flatCorrection(
2030 maskedImage=exposure.getMaskedImage(),
2031 flatMaskedImage=flatExposure.getMaskedImage(),
2032 scalingType=self.config.flatScalingType,
2033 userScale=self.config.flatUserScale,
2035 trimToFit=self.config.doTrimToMatchCalib
2039 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 2043 exposure : `lsst.afw.image.Exposure` 2044 Exposure to process. Only the amplifier DataSec is processed. 2045 amp : `lsst.afw.table.AmpInfoCatalog` 2046 Amplifier detector data. 2050 lsst.ip.isr.isrFunctions.makeThresholdMask 2052 if not math.isnan(amp.getSaturation()):
2053 maskedImage = exposure.getMaskedImage()
2054 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2055 isrFunctions.makeThresholdMask(
2056 maskedImage=dataView,
2057 threshold=amp.getSaturation(),
2059 maskName=self.config.saturatedMaskName,
2063 """!Interpolate over saturated pixels, in place. 2065 This method should be called after `saturationDetection`, to 2066 ensure that the saturated pixels have been identified in the 2067 SAT mask. It should also be called after `assembleCcd`, since 2068 saturated regions may cross amplifier boundaries. 2072 exposure : `lsst.afw.image.Exposure` 2073 Exposure to process. 2077 lsst.ip.isr.isrTask.saturationDetection 2078 lsst.ip.isr.isrFunctions.interpolateFromMask 2080 isrFunctions.interpolateFromMask(
2081 maskedImage=exposure.getMaskedImage(),
2082 fwhm=self.config.fwhm,
2083 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2084 maskNameList=list(self.config.saturatedMaskName),
2088 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 2092 exposure : `lsst.afw.image.Exposure` 2093 Exposure to process. Only the amplifier DataSec is processed. 2094 amp : `lsst.afw.table.AmpInfoCatalog` 2095 Amplifier detector data. 2099 lsst.ip.isr.isrFunctions.makeThresholdMask 2103 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 2104 This is intended to indicate pixels that may be affected by unknown systematics; 2105 for example if non-linearity corrections above a certain level are unstable 2106 then that would be a useful value for suspectLevel. A value of `nan` indicates 2107 that no such level exists and no pixels are to be masked as suspicious. 2109 suspectLevel = amp.getSuspectLevel()
2110 if math.isnan(suspectLevel):
2113 maskedImage = exposure.getMaskedImage()
2114 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2115 isrFunctions.makeThresholdMask(
2116 maskedImage=dataView,
2117 threshold=suspectLevel,
2119 maskName=self.config.suspectMaskName,
2123 """!Mask defects using mask plane "BAD", in place. 2127 exposure : `lsst.afw.image.Exposure` 2128 Exposure to process. 2129 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 2130 `lsst.afw.image.DefectBase`. 2131 List of defects to mask. 2135 Call this after CCD assembly, since defects may cross amplifier boundaries. 2137 maskedImage = exposure.getMaskedImage()
2138 if not isinstance(defectBaseList, Defects):
2140 defectList = Defects(defectBaseList)
2142 defectList = defectBaseList
2143 defectList.maskPixels(maskedImage, maskName=
"BAD")
2145 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2146 """!Mask edge pixels with applicable mask plane. 2150 exposure : `lsst.afw.image.Exposure` 2151 Exposure to process. 2152 numEdgePixels : `int`, optional 2153 Number of edge pixels to mask. 2154 maskPlane : `str`, optional 2155 Mask plane name to use. 2157 maskedImage = exposure.getMaskedImage()
2158 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2160 if numEdgePixels > 0:
2161 goodBBox = maskedImage.getBBox()
2163 goodBBox.grow(-numEdgePixels)
2165 SourceDetectionTask.setEdgeBits(
2172 """Mask and interpolate defects using mask plane "BAD", in place. 2176 exposure : `lsst.afw.image.Exposure` 2177 Exposure to process. 2178 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 2179 `lsst.afw.image.DefectBase`. 2180 List of defects to mask and interpolate. 2184 lsst.ip.isr.isrTask.maskDefect() 2187 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2188 maskPlane=
"SUSPECT")
2189 isrFunctions.interpolateFromMask(
2190 maskedImage=exposure.getMaskedImage(),
2191 fwhm=self.config.fwhm,
2192 growSaturatedFootprints=0,
2193 maskNameList=[
"BAD"],
2197 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 2201 exposure : `lsst.afw.image.Exposure` 2202 Exposure to process. 2206 We mask over all NaNs, including those that are masked with 2207 other bits (because those may or may not be interpolated over 2208 later, and we want to remove all NaNs). Despite this 2209 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 2210 the historical name. 2212 maskedImage = exposure.getMaskedImage()
2215 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2216 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2217 numNans =
maskNans(maskedImage, maskVal)
2218 self.metadata.set(
"NUMNANS", numNans)
2220 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2223 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 2227 exposure : `lsst.afw.image.Exposure` 2228 Exposure to process. 2232 lsst.ip.isr.isrTask.maskNan() 2235 isrFunctions.interpolateFromMask(
2236 maskedImage=exposure.getMaskedImage(),
2237 fwhm=self.config.fwhm,
2238 growSaturatedFootprints=0,
2239 maskNameList=[
"UNMASKEDNAN"],
2243 """Measure the image background in subgrids, for quality control purposes. 2247 exposure : `lsst.afw.image.Exposure` 2248 Exposure to process. 2249 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 2250 Configuration object containing parameters on which background 2251 statistics and subgrids to use. 2253 if IsrQaConfig
is not None:
2254 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2255 IsrQaConfig.flatness.nIter)
2256 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2257 statsControl.setAndMask(maskVal)
2258 maskedImage = exposure.getMaskedImage()
2259 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2260 skyLevel = stats.getValue(afwMath.MEDIAN)
2261 skySigma = stats.getValue(afwMath.STDEVCLIP)
2262 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2263 metadata = exposure.getMetadata()
2264 metadata.set(
'SKYLEVEL', skyLevel)
2265 metadata.set(
'SKYSIGMA', skySigma)
2268 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2269 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2270 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2271 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2272 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2273 skyLevels = numpy.zeros((nX, nY))
2276 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2278 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2280 xLLC = xc - meshXHalf
2281 yLLC = yc - meshYHalf
2282 xURC = xc + meshXHalf - 1
2283 yURC = yc + meshYHalf - 1
2286 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2288 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2290 good = numpy.where(numpy.isfinite(skyLevels))
2291 skyMedian = numpy.median(skyLevels[good])
2292 flatness = (skyLevels[good] - skyMedian) / skyMedian
2293 flatness_rms = numpy.std(flatness)
2294 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2296 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2297 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2298 nX, nY, flatness_pp, flatness_rms)
2300 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2301 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2302 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2303 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2304 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2307 """Set an approximate magnitude zero point for the exposure. 2311 exposure : `lsst.afw.image.Exposure` 2312 Exposure to process. 2314 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2315 if filterName
in self.config.fluxMag0T1:
2316 fluxMag0 = self.config.fluxMag0T1[filterName]
2318 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2319 fluxMag0 = self.config.defaultFluxMag0T1
2321 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2323 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2326 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2327 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2330 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2334 ccdExposure : `lsst.afw.image.Exposure` 2335 Exposure to process. 2336 fpPolygon : `lsst.afw.geom.Polygon` 2337 Polygon in focal plane coordinates. 2340 ccd = ccdExposure.getDetector()
2341 fpCorners = ccd.getCorners(FOCAL_PLANE)
2342 ccdPolygon = Polygon(fpCorners)
2345 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2348 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2349 validPolygon = Polygon(ccdPoints)
2350 ccdExposure.getInfo().setValidPolygon(validPolygon)
2354 """Context manager that applies and removes flats and darks, 2355 if the task is configured to apply them. 2359 exp : `lsst.afw.image.Exposure` 2360 Exposure to process. 2361 flat : `lsst.afw.image.Exposure` 2362 Flat exposure the same size as ``exp``. 2363 dark : `lsst.afw.image.Exposure`, optional 2364 Dark exposure the same size as ``exp``. 2368 exp : `lsst.afw.image.Exposure` 2369 The flat and dark corrected exposure. 2371 if self.config.doDark
and dark
is not None:
2373 if self.config.doFlat:
2378 if self.config.doFlat:
2380 if self.config.doDark
and dark
is not None:
2384 """Utility function to examine ISR exposure at different stages. 2388 exposure : `lsst.afw.image.Exposure` 2391 State of processing to view. 2393 frame = getDebugFrame(self._display, stepname)
2395 display = getDisplay(frame)
2396 display.scale(
'asinh',
'zscale')
2397 display.mtv(exposure)
2398 prompt =
"Press Enter to continue [c]... " 2400 ans = input(prompt).lower()
2401 if ans
in (
"",
"c",):
2406 """A Detector-like object that supports returning gain and saturation level 2408 This is used when the input exposure does not have a detector. 2412 exposure : `lsst.afw.image.Exposure` 2413 Exposure to generate a fake amplifier for. 2414 config : `lsst.ip.isr.isrTaskConfig` 2415 Configuration to apply to the fake amplifier. 2419 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2421 self.
_gain = config.gain
2451 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2455 """Task to wrap the default IsrTask to allow it to be retargeted. 2457 The standard IsrTask can be called directly from a command line 2458 program, but doing so removes the ability of the task to be 2459 retargeted. As most cameras override some set of the IsrTask 2460 methods, this would remove those data-specific methods in the 2461 output post-ISR images. This wrapping class fixes the issue, 2462 allowing identical post-ISR images to be generated by both the 2463 processCcd and isrTask code. 2465 ConfigClass = RunIsrConfig
2466 _DefaultName =
"runIsr" 2470 self.makeSubtask(
"isr")
2476 dataRef : `lsst.daf.persistence.ButlerDataRef` 2477 data reference of the detector data to be processed 2481 result : `pipeBase.Struct` 2482 Result struct with component: 2484 - exposure : `lsst.afw.image.Exposure` 2485 Post-ISR processed exposure.
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def getRawHorizontalOverscanBBox(self)
def maskNan(self, exposure)
def getSuspectLevel(self)
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def __init__(self, config=None)
def flatContext(self, exp, flat, dark=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def maskAndInterpolateNan(self, exposure)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
doSaturationInterpolation
def __init__(self, exposure, config)