28 import lsst.pex.config
as pexConfig
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .straylight
import StrayLightTask
54 from .vignette
import VignetteTask
57 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
61 dimensions={
"instrument",
"visit",
"detector"},
63 ccdExposure = cT.PrerequisiteInput(
65 doc=
"Input exposure to process.",
66 storageClass=
"Exposure",
67 dimensions=[
"instrument",
"visit",
"detector"],
69 camera = cT.PrerequisiteInput(
71 storageClass=
"Camera",
72 doc=
"Input camera to construct complete exposures.",
73 dimensions=[
"instrument",
"calibration_label"],
75 bias = cT.PrerequisiteInput(
77 doc=
"Input bias calibration.",
78 storageClass=
"ImageF",
79 dimensions=[
"instrument",
"calibration_label",
"detector"],
81 dark = cT.PrerequisiteInput(
83 doc=
"Input dark calibration.",
84 storageClass=
"ImageF",
85 dimensions=[
"instrument",
"calibration_label",
"detector"],
87 flat = cT.PrerequisiteInput(
89 doc=
"Input flat calibration.",
90 storageClass=
"MaskedImageF",
91 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
93 fringes = cT.PrerequisiteInput(
95 doc=
"Input fringe calibration.",
96 storageClass=
"ExposureF",
97 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
99 strayLightData = cT.PrerequisiteInput(
101 doc=
"Input stray light calibration.",
102 storageClass=
"StrayLightData",
103 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
105 bfKernel = cT.PrerequisiteInput(
107 doc=
"Input brighter-fatter kernel.",
108 storageClass=
"NumpyArray",
109 dimensions=[
"instrument",
"calibration_label"],
111 newBFKernel = cT.PrerequisiteInput(
112 name=
'brighterFatterKernel',
113 doc=
"Newer complete kernel + gain solutions.",
114 storageClass=
"BrighterFatterKernel",
115 dimensions=[
"instrument",
"calibration_label",
"detector"],
117 defects = cT.PrerequisiteInput(
119 doc=
"Input defect tables.",
120 storageClass=
"DefectsList",
121 dimensions=[
"instrument",
"calibration_label",
"detector"],
123 opticsTransmission = cT.PrerequisiteInput(
124 name=
"transmission_optics",
125 storageClass=
"TransmissionCurve",
126 doc=
"Transmission curve due to the optics.",
127 dimensions=[
"instrument",
"calibration_label"],
129 filterTransmission = cT.PrerequisiteInput(
130 name=
"transmission_filter",
131 storageClass=
"TransmissionCurve",
132 doc=
"Transmission curve due to the filter.",
133 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
135 sensorTransmission = cT.PrerequisiteInput(
136 name=
"transmission_sensor",
137 storageClass=
"TransmissionCurve",
138 doc=
"Transmission curve due to the sensor.",
139 dimensions=[
"instrument",
"calibration_label",
"detector"],
141 atmosphereTransmission = cT.PrerequisiteInput(
142 name=
"transmission_atmosphere",
143 storageClass=
"TransmissionCurve",
144 doc=
"Transmission curve due to the atmosphere.",
145 dimensions=[
"instrument"],
147 illumMaskedImage = cT.PrerequisiteInput(
149 doc=
"Input illumination correction.",
150 storageClass=
"MaskedImageF",
151 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
154 outputExposure = cT.Output(
156 doc=
"Output ISR processed exposure.",
157 storageClass=
"ExposureF",
158 dimensions=[
"instrument",
"visit",
"detector"],
160 preInterpExposure = cT.Output(
161 name=
'preInterpISRCCD',
162 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
163 storageClass=
"ExposureF",
164 dimensions=[
"instrument",
"visit",
"detector"],
166 outputOssThumbnail = cT.Output(
168 doc=
"Output Overscan-subtracted thumbnail image.",
169 storageClass=
"Thumbnail",
170 dimensions=[
"instrument",
"visit",
"detector"],
172 outputFlattenedThumbnail = cT.Output(
173 name=
"FlattenedThumb",
174 doc=
"Output flat-corrected thumbnail image.",
175 storageClass=
"Thumbnail",
176 dimensions=[
"instrument",
"visit",
"detector"],
182 if config.doBias
is not True:
183 self.prerequisiteInputs.discard(
"bias")
184 if config.doLinearize
is not True:
185 self.prerequisiteInputs.discard(
"linearizer")
186 if config.doCrosstalk
is not True:
187 self.prerequisiteInputs.discard(
"crosstalkSources")
188 if config.doBrighterFatter
is not True:
189 self.prerequisiteInputs.discard(
"bfKernel")
190 self.prerequisiteInputs.discard(
"newBFKernel")
191 if config.doDefect
is not True:
192 self.prerequisiteInputs.discard(
"defects")
193 if config.doDark
is not True:
194 self.prerequisiteInputs.discard(
"dark")
195 if config.doFlat
is not True:
196 self.prerequisiteInputs.discard(
"flat")
197 if config.doAttachTransmissionCurve
is not True:
198 self.prerequisiteInputs.discard(
"opticsTransmission")
199 self.prerequisiteInputs.discard(
"filterTransmission")
200 self.prerequisiteInputs.discard(
"sensorTransmission")
201 self.prerequisiteInputs.discard(
"atmosphereTransmission")
202 if config.doUseOpticsTransmission
is not True:
203 self.prerequisiteInputs.discard(
"opticsTransmission")
204 if config.doUseFilterTransmission
is not True:
205 self.prerequisiteInputs.discard(
"filterTransmission")
206 if config.doUseSensorTransmission
is not True:
207 self.prerequisiteInputs.discard(
"sensorTransmission")
208 if config.doUseAtmosphereTransmission
is not True:
209 self.prerequisiteInputs.discard(
"atmosphereTransmission")
210 if config.doIlluminationCorrection
is not True:
211 self.prerequisiteInputs.discard(
"illumMaskedImage")
213 if config.doWrite
is not True:
214 self.outputs.discard(
"outputExposure")
215 self.outputs.discard(
"preInterpExposure")
216 self.outputs.discard(
"outputFlattenedThumbnail")
217 self.outputs.discard(
"outputOssThumbnail")
218 if config.doSaveInterpPixels
is not True:
219 self.outputs.discard(
"preInterpExposure")
220 if config.qa.doThumbnailOss
is not True:
221 self.outputs.discard(
"outputOssThumbnail")
222 if config.qa.doThumbnailFlattened
is not True:
223 self.outputs.discard(
"outputFlattenedThumbnail")
227 pipelineConnections=IsrTaskConnections):
228 """Configuration parameters for IsrTask. 230 Items are grouped in the order in which they are executed by the task. 232 datasetType = pexConfig.Field(
234 doc=
"Dataset type for input data; users will typically leave this alone, " 235 "but camera-specific ISR tasks will override it",
239 fallbackFilterName = pexConfig.Field(
241 doc=
"Fallback default filter name for calibrations.",
244 useFallbackDate = pexConfig.Field(
246 doc=
"Pass observation date when using fallback filter.",
249 expectWcs = pexConfig.Field(
252 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 254 fwhm = pexConfig.Field(
256 doc=
"FWHM of PSF in arcseconds.",
259 qa = pexConfig.ConfigField(
261 doc=
"QA related configuration options.",
265 doConvertIntToFloat = pexConfig.Field(
267 doc=
"Convert integer raw images to floating point values?",
272 doSaturation = pexConfig.Field(
274 doc=
"Mask saturated pixels? NB: this is totally independent of the" 275 " interpolation option - this is ONLY setting the bits in the mask." 276 " To have them interpolated make sure doSaturationInterpolation=True",
279 saturatedMaskName = pexConfig.Field(
281 doc=
"Name of mask plane to use in saturation detection and interpolation",
284 saturation = pexConfig.Field(
286 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
287 default=float(
"NaN"),
289 growSaturationFootprintSize = pexConfig.Field(
291 doc=
"Number of pixels by which to grow the saturation footprints",
296 doSuspect = pexConfig.Field(
298 doc=
"Mask suspect pixels?",
301 suspectMaskName = pexConfig.Field(
303 doc=
"Name of mask plane to use for suspect pixels",
306 numEdgeSuspect = pexConfig.Field(
308 doc=
"Number of edge pixels to be flagged as untrustworthy.",
313 doSetBadRegions = pexConfig.Field(
315 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
318 badStatistic = pexConfig.ChoiceField(
320 doc=
"How to estimate the average value for BAD regions.",
323 "MEANCLIP":
"Correct using the (clipped) mean of good data",
324 "MEDIAN":
"Correct using the median of the good data",
329 doOverscan = pexConfig.Field(
331 doc=
"Do overscan subtraction?",
334 overscanFitType = pexConfig.ChoiceField(
336 doc=
"The method for fitting the overscan bias level.",
339 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
340 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
341 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
342 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
343 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
344 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
345 "MEAN":
"Correct using the mean of the overscan region",
346 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
347 "MEDIAN":
"Correct using the median of the overscan region",
348 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
351 overscanOrder = pexConfig.Field(
353 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
354 "or number of spline knots if overscan fit type is a spline."),
357 overscanNumSigmaClip = pexConfig.Field(
359 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
362 overscanIsInt = pexConfig.Field(
364 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" +
365 " and overscan.FitType=MEDIAN_PER_ROW.",
368 overscanNumLeadingColumnsToSkip = pexConfig.Field(
370 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
373 overscanNumTrailingColumnsToSkip = pexConfig.Field(
375 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
378 overscanMaxDev = pexConfig.Field(
380 doc=
"Maximum deviation from the median for overscan",
381 default=1000.0, check=
lambda x: x > 0
383 overscanBiasJump = pexConfig.Field(
385 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
388 overscanBiasJumpKeyword = pexConfig.Field(
390 doc=
"Header keyword containing information about devices.",
391 default=
"NO_SUCH_KEY",
393 overscanBiasJumpDevices = pexConfig.ListField(
395 doc=
"List of devices that need piecewise overscan correction.",
398 overscanBiasJumpLocation = pexConfig.Field(
400 doc=
"Location of bias jump along y-axis.",
405 doAssembleCcd = pexConfig.Field(
408 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 410 assembleCcd = pexConfig.ConfigurableField(
411 target=AssembleCcdTask,
412 doc=
"CCD assembly task",
416 doAssembleIsrExposures = pexConfig.Field(
419 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 421 doTrimToMatchCalib = pexConfig.Field(
424 doc=
"Trim raw data to match calibration bounding boxes?" 428 doBias = pexConfig.Field(
430 doc=
"Apply bias frame correction?",
433 biasDataProductName = pexConfig.Field(
435 doc=
"Name of the bias data product",
440 doVariance = pexConfig.Field(
442 doc=
"Calculate variance?",
445 gain = pexConfig.Field(
447 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
448 default=float(
"NaN"),
450 readNoise = pexConfig.Field(
452 doc=
"The read noise to use if no Detector is present in the Exposure",
455 doEmpiricalReadNoise = pexConfig.Field(
458 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 462 doLinearize = pexConfig.Field(
464 doc=
"Correct for nonlinearity of the detector's response?",
469 doCrosstalk = pexConfig.Field(
471 doc=
"Apply intra-CCD crosstalk correction?",
474 doCrosstalkBeforeAssemble = pexConfig.Field(
476 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
479 crosstalk = pexConfig.ConfigurableField(
480 target=CrosstalkTask,
481 doc=
"Intra-CCD crosstalk correction",
485 doDefect = pexConfig.Field(
487 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
490 doNanMasking = pexConfig.Field(
492 doc=
"Mask NAN pixels?",
495 doWidenSaturationTrails = pexConfig.Field(
497 doc=
"Widen bleed trails based on their width?",
502 doBrighterFatter = pexConfig.Field(
505 doc=
"Apply the brighter fatter correction" 507 brighterFatterLevel = pexConfig.ChoiceField(
510 doc=
"The level at which to correct for brighter-fatter.",
512 "AMP":
"Every amplifier treated separately.",
513 "DETECTOR":
"One kernel per detector",
516 brighterFatterMaxIter = pexConfig.Field(
519 doc=
"Maximum number of iterations for the brighter fatter correction" 521 brighterFatterThreshold = pexConfig.Field(
524 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 525 " absolute value of the difference between the current corrected image and the one" 526 " from the previous iteration summed over all the pixels." 528 brighterFatterApplyGain = pexConfig.Field(
531 doc=
"Should the gain be applied when applying the brighter fatter correction?" 533 brighterFatterMaskGrowSize = pexConfig.Field(
536 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate " 537 " when brighter-fatter correction is applied." 541 doDark = pexConfig.Field(
543 doc=
"Apply dark frame correction?",
546 darkDataProductName = pexConfig.Field(
548 doc=
"Name of the dark data product",
553 doStrayLight = pexConfig.Field(
555 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
558 strayLight = pexConfig.ConfigurableField(
559 target=StrayLightTask,
560 doc=
"y-band stray light correction" 564 doFlat = pexConfig.Field(
566 doc=
"Apply flat field correction?",
569 flatDataProductName = pexConfig.Field(
571 doc=
"Name of the flat data product",
574 flatScalingType = pexConfig.ChoiceField(
576 doc=
"The method for scaling the flat on the fly.",
579 "USER":
"Scale by flatUserScale",
580 "MEAN":
"Scale by the inverse of the mean",
581 "MEDIAN":
"Scale by the inverse of the median",
584 flatUserScale = pexConfig.Field(
586 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
589 doTweakFlat = pexConfig.Field(
591 doc=
"Tweak flats to match observed amplifier ratios?",
596 doApplyGains = pexConfig.Field(
598 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
601 normalizeGains = pexConfig.Field(
603 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
608 doFringe = pexConfig.Field(
610 doc=
"Apply fringe correction?",
613 fringe = pexConfig.ConfigurableField(
615 doc=
"Fringe subtraction task",
617 fringeAfterFlat = pexConfig.Field(
619 doc=
"Do fringe subtraction after flat-fielding?",
624 doAddDistortionModel = pexConfig.Field(
626 doc=
"Apply a distortion model based on camera geometry to the WCS?",
628 deprecated=(
"Camera geometry is incorporated when reading the raw files." 629 " This option no longer is used, and will be removed after v19.")
633 doMeasureBackground = pexConfig.Field(
635 doc=
"Measure the background level on the reduced image?",
640 doCameraSpecificMasking = pexConfig.Field(
642 doc=
"Mask camera-specific bad regions?",
645 masking = pexConfig.ConfigurableField(
652 doInterpolate = pexConfig.Field(
654 doc=
"Interpolate masked pixels?",
657 doSaturationInterpolation = pexConfig.Field(
659 doc=
"Perform interpolation over pixels masked as saturated?" 660 " NB: This is independent of doSaturation; if that is False this plane" 661 " will likely be blank, resulting in a no-op here.",
664 doNanInterpolation = pexConfig.Field(
666 doc=
"Perform interpolation over pixels masked as NaN?" 667 " NB: This is independent of doNanMasking; if that is False this plane" 668 " will likely be blank, resulting in a no-op here.",
671 doNanInterpAfterFlat = pexConfig.Field(
673 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 674 "also have to interpolate them before flat-fielding."),
677 maskListToInterpolate = pexConfig.ListField(
679 doc=
"List of mask planes that should be interpolated.",
680 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
682 doSaveInterpPixels = pexConfig.Field(
684 doc=
"Save a copy of the pre-interpolated pixel values?",
689 fluxMag0T1 = pexConfig.DictField(
692 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
693 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
696 defaultFluxMag0T1 = pexConfig.Field(
698 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
699 default=pow(10.0, 0.4*28.0)
703 doVignette = pexConfig.Field(
705 doc=
"Apply vignetting parameters?",
708 vignette = pexConfig.ConfigurableField(
710 doc=
"Vignetting task.",
714 doAttachTransmissionCurve = pexConfig.Field(
717 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 719 doUseOpticsTransmission = pexConfig.Field(
722 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 724 doUseFilterTransmission = pexConfig.Field(
727 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 729 doUseSensorTransmission = pexConfig.Field(
732 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 734 doUseAtmosphereTransmission = pexConfig.Field(
737 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 741 doIlluminationCorrection = pexConfig.Field(
744 doc=
"Perform illumination correction?" 746 illuminationCorrectionDataProductName = pexConfig.Field(
748 doc=
"Name of the illumination correction data product.",
751 illumScale = pexConfig.Field(
753 doc=
"Scale factor for the illumination correction.",
756 illumFilters = pexConfig.ListField(
759 doc=
"Only perform illumination correction for these filters." 763 doWrite = pexConfig.Field(
765 doc=
"Persist postISRCCD?",
772 raise ValueError(
"You may not specify both doFlat and doApplyGains")
774 self.config.maskListToInterpolate.append(
"SAT")
776 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
779 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
780 """Apply common instrument signature correction algorithms to a raw frame. 782 The process for correcting imaging data is very similar from 783 camera to camera. This task provides a vanilla implementation of 784 doing these corrections, including the ability to turn certain 785 corrections off if they are not needed. The inputs to the primary 786 method, `run()`, are a raw exposure to be corrected and the 787 calibration data products. The raw input is a single chip sized 788 mosaic of all amps including overscans and other non-science 789 pixels. The method `runDataRef()` identifies and defines the 790 calibration data products, and is intended for use by a 791 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 792 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 793 subclassed for different camera, although the most camera specific 794 methods have been split into subtasks that can be redirected 797 The __init__ method sets up the subtasks for ISR processing, using 798 the defaults from `lsst.ip.isr`. 803 Positional arguments passed to the Task constructor. None used at this time. 804 kwargs : `dict`, optional 805 Keyword arguments passed on to the Task constructor. None used at this time. 807 ConfigClass = IsrTaskConfig
812 self.makeSubtask(
"assembleCcd")
813 self.makeSubtask(
"crosstalk")
814 self.makeSubtask(
"strayLight")
815 self.makeSubtask(
"fringe")
816 self.makeSubtask(
"masking")
817 self.makeSubtask(
"vignette")
820 inputs = butlerQC.get(inputRefs)
823 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
824 except Exception
as e:
825 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
828 inputs[
'isGen3'] =
True 830 detector = inputs[
'ccdExposure'].getDetector()
833 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
835 linearizer.fromYaml(inputs[
'linearizer'])
838 inputs[
'linearizer'] = linearizer
840 if self.config.doDefect
is True:
841 if "defects" in inputs
and inputs[
'defects']
is not None:
844 if not isinstance(inputs[
"defects"], Defects):
845 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
849 if self.config.doBrighterFatter:
850 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
851 if brighterFatterKernel
is None:
852 brighterFatterKernel = inputs.get(
'bfKernel',
None)
854 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
855 detId = detector.getId()
856 inputs[
'bfGains'] = brighterFatterKernel.gain
859 if self.config.brighterFatterLevel ==
'DETECTOR':
860 if brighterFatterKernel.detectorKernel:
861 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
862 elif brighterFatterKernel.detectorKernelFromAmpKernels:
863 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
865 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
868 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
876 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
877 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
878 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
880 assembler=self.assembleCcd
881 if self.config.doAssembleIsrExposures
else None)
883 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
885 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
886 if 'strayLightData' not in inputs:
887 inputs[
'strayLightData'] =
None 889 outputs = self.
run(**inputs)
890 butlerQC.put(outputs, outputRefs)
893 """!Retrieve necessary frames for instrument signature removal. 895 Pre-fetching all required ISR data products limits the IO 896 required by the ISR. Any conflict between the calibration data 897 available and that needed for ISR is also detected prior to 898 doing processing, allowing it to fail quickly. 902 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 903 Butler reference of the detector data to be processed 904 rawExposure : `afw.image.Exposure` 905 The raw exposure that will later be corrected with the 906 retrieved calibration data; should not be modified in this 911 result : `lsst.pipe.base.Struct` 912 Result struct with components (which may be `None`): 913 - ``bias``: bias calibration frame (`afw.image.Exposure`) 914 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 915 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 916 - ``dark``: dark calibration frame (`afw.image.Exposure`) 917 - ``flat``: flat calibration frame (`afw.image.Exposure`) 918 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 919 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 920 - ``fringes``: `lsst.pipe.base.Struct` with components: 921 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 922 - ``seed``: random seed derived from the ccdExposureId for random 923 number generator (`uint32`). 924 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 925 A ``TransmissionCurve`` that represents the throughput of the optics, 926 to be evaluated in focal-plane coordinates. 927 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 928 A ``TransmissionCurve`` that represents the throughput of the filter 929 itself, to be evaluated in focal-plane coordinates. 930 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 931 A ``TransmissionCurve`` that represents the throughput of the sensor 932 itself, to be evaluated in post-assembly trimmed detector coordinates. 933 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 934 A ``TransmissionCurve`` that represents the throughput of the 935 atmosphere, assumed to be spatially constant. 936 - ``strayLightData`` : `object` 937 An opaque object containing calibration information for 938 stray-light correction. If `None`, no correction will be 940 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 944 NotImplementedError : 945 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 948 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
949 dateObs = dateObs.toPython().isoformat()
951 self.log.warn(
"Unable to identify dateObs for rawExposure.")
954 ccd = rawExposure.getDetector()
955 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
956 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
957 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
958 if self.config.doBias
else None)
960 linearizer = (dataRef.get(
"linearizer", immediate=
True)
962 if isinstance(linearizer, numpy.ndarray):
964 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
965 if self.config.doCrosstalk
else None)
966 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
967 if self.config.doDark
else None)
968 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
970 if self.config.doFlat
else None)
972 brighterFatterKernel =
None 973 brighterFatterGains =
None 974 if self.config.doBrighterFatter
is True:
979 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
980 brighterFatterGains = brighterFatterKernel.gain
981 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
984 brighterFatterKernel = dataRef.get(
"bfKernel")
985 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
987 brighterFatterKernel =
None 988 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
991 if self.config.brighterFatterLevel ==
'DETECTOR':
992 if brighterFatterKernel.detectorKernel:
993 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
994 elif brighterFatterKernel.detectorKernelFromAmpKernels:
995 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
997 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1000 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1002 defectList = (dataRef.get(
"defects")
1003 if self.config.doDefect
else None)
1004 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1005 if self.config.doAssembleIsrExposures
else None)
1006 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1007 else pipeBase.Struct(fringes=
None))
1009 if self.config.doAttachTransmissionCurve:
1010 opticsTransmission = (dataRef.get(
"transmission_optics")
1011 if self.config.doUseOpticsTransmission
else None)
1012 filterTransmission = (dataRef.get(
"transmission_filter")
1013 if self.config.doUseFilterTransmission
else None)
1014 sensorTransmission = (dataRef.get(
"transmission_sensor")
1015 if self.config.doUseSensorTransmission
else None)
1016 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1017 if self.config.doUseAtmosphereTransmission
else None)
1019 opticsTransmission =
None 1020 filterTransmission =
None 1021 sensorTransmission =
None 1022 atmosphereTransmission =
None 1024 if self.config.doStrayLight:
1025 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1027 strayLightData =
None 1030 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1031 if (self.config.doIlluminationCorrection
and 1032 filterName
in self.config.illumFilters)
1036 return pipeBase.Struct(bias=biasExposure,
1037 linearizer=linearizer,
1038 crosstalkSources=crosstalkSources,
1041 bfKernel=brighterFatterKernel,
1042 bfGains=brighterFatterGains,
1044 fringes=fringeStruct,
1045 opticsTransmission=opticsTransmission,
1046 filterTransmission=filterTransmission,
1047 sensorTransmission=sensorTransmission,
1048 atmosphereTransmission=atmosphereTransmission,
1049 strayLightData=strayLightData,
1050 illumMaskedImage=illumMaskedImage
1053 @pipeBase.timeMethod
1054 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1055 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1056 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1057 sensorTransmission=
None, atmosphereTransmission=
None,
1058 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1061 """!Perform instrument signature removal on an exposure. 1063 Steps included in the ISR processing, in order performed, are: 1064 - saturation and suspect pixel masking 1065 - overscan subtraction 1066 - CCD assembly of individual amplifiers 1068 - variance image construction 1069 - linearization of non-linear response 1071 - brighter-fatter correction 1074 - stray light subtraction 1076 - masking of known defects and camera specific features 1077 - vignette calculation 1078 - appending transmission curve and distortion model 1082 ccdExposure : `lsst.afw.image.Exposure` 1083 The raw exposure that is to be run through ISR. The 1084 exposure is modified by this method. 1085 camera : `lsst.afw.cameraGeom.Camera`, optional 1086 The camera geometry for this exposure. Used to select the 1087 distortion model appropriate for this data. 1088 bias : `lsst.afw.image.Exposure`, optional 1089 Bias calibration frame. 1090 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 1091 Functor for linearization. 1092 crosstalkSources : `list`, optional 1093 List of possible crosstalk sources. 1094 dark : `lsst.afw.image.Exposure`, optional 1095 Dark calibration frame. 1096 flat : `lsst.afw.image.Exposure`, optional 1097 Flat calibration frame. 1098 bfKernel : `numpy.ndarray`, optional 1099 Brighter-fatter kernel. 1100 bfGains : `dict` of `float`, optional 1101 Gains used to override the detector's nominal gains for the 1102 brighter-fatter correction. A dict keyed by amplifier name for 1103 the detector in question. 1104 defects : `lsst.meas.algorithms.Defects`, optional 1106 fringes : `lsst.pipe.base.Struct`, optional 1107 Struct containing the fringe correction data, with 1109 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 1110 - ``seed``: random seed derived from the ccdExposureId for random 1111 number generator (`uint32`) 1112 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 1113 A ``TransmissionCurve`` that represents the throughput of the optics, 1114 to be evaluated in focal-plane coordinates. 1115 filterTransmission : `lsst.afw.image.TransmissionCurve` 1116 A ``TransmissionCurve`` that represents the throughput of the filter 1117 itself, to be evaluated in focal-plane coordinates. 1118 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1119 A ``TransmissionCurve`` that represents the throughput of the sensor 1120 itself, to be evaluated in post-assembly trimmed detector coordinates. 1121 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1122 A ``TransmissionCurve`` that represents the throughput of the 1123 atmosphere, assumed to be spatially constant. 1124 detectorNum : `int`, optional 1125 The integer number for the detector to process. 1126 isGen3 : bool, optional 1127 Flag this call to run() as using the Gen3 butler environment. 1128 strayLightData : `object`, optional 1129 Opaque object containing calibration information for stray-light 1130 correction. If `None`, no correction will be performed. 1131 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 1132 Illumination correction image. 1136 result : `lsst.pipe.base.Struct` 1137 Result struct with component: 1138 - ``exposure`` : `afw.image.Exposure` 1139 The fully ISR corrected exposure. 1140 - ``outputExposure`` : `afw.image.Exposure` 1141 An alias for `exposure` 1142 - ``ossThumb`` : `numpy.ndarray` 1143 Thumbnail image of the exposure after overscan subtraction. 1144 - ``flattenedThumb`` : `numpy.ndarray` 1145 Thumbnail image of the exposure after flat-field correction. 1150 Raised if a configuration option is set to True, but the 1151 required calibration data has not been specified. 1155 The current processed exposure can be viewed by setting the 1156 appropriate lsstDebug entries in the `debug.display` 1157 dictionary. The names of these entries correspond to some of 1158 the IsrTaskConfig Boolean options, with the value denoting the 1159 frame to use. The exposure is shown inside the matching 1160 option check and after the processing of that step has 1161 finished. The steps with debug points are: 1172 In addition, setting the "postISRCCD" entry displays the 1173 exposure after all ISR processing has finished. 1181 if detectorNum
is None:
1182 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1184 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1189 if isinstance(ccdExposure, ButlerDataRef):
1192 ccd = ccdExposure.getDetector()
1193 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1196 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd." 1197 ccd = [
FakeAmp(ccdExposure, self.config)]
1200 if self.config.doBias
and bias
is None:
1201 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1203 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1204 if self.config.doBrighterFatter
and bfKernel
is None:
1205 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1206 if self.config.doDark
and dark
is None:
1207 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1208 if self.config.doFlat
and flat
is None:
1209 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1210 if self.config.doDefect
and defects
is None:
1211 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1212 if (self.config.doFringe
and filterName
in self.fringe.config.filters
and 1213 fringes.fringes
is None):
1218 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1219 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
and 1220 illumMaskedImage
is None):
1221 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1224 if self.config.doConvertIntToFloat:
1225 self.log.info(
"Converting exposure to floating point values.")
1232 if ccdExposure.getBBox().contains(amp.getBBox()):
1236 if self.config.doOverscan
and not badAmp:
1239 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1240 if overscanResults
is not None and \
1241 self.config.qa
is not None and self.config.qa.saveStats
is True:
1242 if isinstance(overscanResults.overscanFit, float):
1243 qaMedian = overscanResults.overscanFit
1244 qaStdev = float(
"NaN")
1246 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1247 afwMath.MEDIAN | afwMath.STDEVCLIP)
1248 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1249 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1251 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1252 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1253 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1254 amp.getName(), qaMedian, qaStdev)
1255 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1258 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1259 overscanResults =
None 1261 overscans.append(overscanResults
if overscanResults
is not None else None)
1263 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1265 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1266 self.log.info(
"Applying crosstalk correction.")
1267 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1268 self.
debugView(ccdExposure,
"doCrosstalk")
1270 if self.config.doAssembleCcd:
1271 self.log.info(
"Assembling CCD from amplifiers.")
1272 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1274 if self.config.expectWcs
and not ccdExposure.getWcs():
1275 self.log.warn(
"No WCS found in input exposure.")
1276 self.
debugView(ccdExposure,
"doAssembleCcd")
1279 if self.config.qa.doThumbnailOss:
1280 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1282 if self.config.doBias:
1283 self.log.info(
"Applying bias correction.")
1284 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1285 trimToFit=self.config.doTrimToMatchCalib)
1288 if self.config.doVariance:
1289 for amp, overscanResults
in zip(ccd, overscans):
1290 if ccdExposure.getBBox().contains(amp.getBBox()):
1291 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1292 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1293 if overscanResults
is not None:
1295 overscanImage=overscanResults.overscanImage)
1299 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1300 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1301 afwMath.MEDIAN | afwMath.STDEVCLIP)
1302 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1303 qaStats.getValue(afwMath.MEDIAN))
1304 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1305 qaStats.getValue(afwMath.STDEVCLIP))
1306 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1307 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1308 qaStats.getValue(afwMath.STDEVCLIP))
1311 self.log.info(
"Applying linearizer.")
1312 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1313 detector=ccd, log=self.log)
1315 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1316 self.log.info(
"Applying crosstalk correction.")
1317 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=
True)
1318 self.
debugView(ccdExposure,
"doCrosstalk")
1322 if self.config.doDefect:
1323 self.log.info(
"Masking defects.")
1326 if self.config.numEdgeSuspect > 0:
1327 self.log.info(
"Masking edges as SUSPECT.")
1328 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1329 maskPlane=
"SUSPECT")
1331 if self.config.doNanMasking:
1332 self.log.info(
"Masking NAN value pixels.")
1335 if self.config.doWidenSaturationTrails:
1336 self.log.info(
"Widening saturation trails.")
1337 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1339 if self.config.doCameraSpecificMasking:
1340 self.log.info(
"Masking regions for camera specific reasons.")
1341 self.masking.
run(ccdExposure)
1343 if self.config.doBrighterFatter:
1352 interpExp = ccdExposure.clone()
1354 isrFunctions.interpolateFromMask(
1355 maskedImage=interpExp.getMaskedImage(),
1356 fwhm=self.config.fwhm,
1357 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1358 maskNameList=self.config.maskListToInterpolate
1360 bfExp = interpExp.clone()
1362 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1363 type(bfKernel), type(bfGains))
1364 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1365 self.config.brighterFatterMaxIter,
1366 self.config.brighterFatterThreshold,
1367 self.config.brighterFatterApplyGain,
1369 if bfResults[1] == self.config.brighterFatterMaxIter:
1370 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1373 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1375 image = ccdExposure.getMaskedImage().getImage()
1376 bfCorr = bfExp.getMaskedImage().getImage()
1377 bfCorr -= interpExp.getMaskedImage().getImage()
1386 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1387 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1390 if self.config.brighterFatterMaskGrowSize > 0:
1391 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1392 for maskPlane
in self.config.maskListToInterpolate:
1393 isrFunctions.growMasks(ccdExposure.getMask(),
1394 radius=self.config.brighterFatterMaskGrowSize,
1395 maskNameList=maskPlane,
1396 maskValue=maskPlane)
1398 self.
debugView(ccdExposure,
"doBrighterFatter")
1400 if self.config.doDark:
1401 self.log.info(
"Applying dark correction.")
1405 if self.config.doFringe
and not self.config.fringeAfterFlat:
1406 self.log.info(
"Applying fringe correction before flat.")
1407 self.fringe.
run(ccdExposure, **fringes.getDict())
1410 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1411 self.log.info(
"Checking strayLight correction.")
1412 self.strayLight.
run(ccdExposure, strayLightData)
1413 self.
debugView(ccdExposure,
"doStrayLight")
1415 if self.config.doFlat:
1416 self.log.info(
"Applying flat correction.")
1420 if self.config.doApplyGains:
1421 self.log.info(
"Applying gain correction instead of flat.")
1422 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1424 if self.config.doFringe
and self.config.fringeAfterFlat:
1425 self.log.info(
"Applying fringe correction after flat.")
1426 self.fringe.
run(ccdExposure, **fringes.getDict())
1428 if self.config.doVignette:
1429 self.log.info(
"Constructing Vignette polygon.")
1432 if self.config.vignette.doWriteVignettePolygon:
1435 if self.config.doAttachTransmissionCurve:
1436 self.log.info(
"Adding transmission curves.")
1437 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1438 filterTransmission=filterTransmission,
1439 sensorTransmission=sensorTransmission,
1440 atmosphereTransmission=atmosphereTransmission)
1442 flattenedThumb =
None 1443 if self.config.qa.doThumbnailFlattened:
1444 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1446 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1447 self.log.info(
"Performing illumination correction.")
1448 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1449 illumMaskedImage, illumScale=self.config.illumScale,
1450 trimToFit=self.config.doTrimToMatchCalib)
1453 if self.config.doSaveInterpPixels:
1454 preInterpExp = ccdExposure.clone()
1469 if self.config.doSetBadRegions:
1470 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1471 if badPixelCount > 0:
1472 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1474 if self.config.doInterpolate:
1475 self.log.info(
"Interpolating masked pixels.")
1476 isrFunctions.interpolateFromMask(
1477 maskedImage=ccdExposure.getMaskedImage(),
1478 fwhm=self.config.fwhm,
1479 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1480 maskNameList=list(self.config.maskListToInterpolate)
1485 if self.config.doMeasureBackground:
1486 self.log.info(
"Measuring background level.")
1489 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1491 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1492 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1493 afwMath.MEDIAN | afwMath.STDEVCLIP)
1494 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1495 qaStats.getValue(afwMath.MEDIAN))
1496 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1497 qaStats.getValue(afwMath.STDEVCLIP))
1498 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1499 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1500 qaStats.getValue(afwMath.STDEVCLIP))
1502 self.
debugView(ccdExposure,
"postISRCCD")
1504 return pipeBase.Struct(
1505 exposure=ccdExposure,
1507 flattenedThumb=flattenedThumb,
1509 preInterpolatedExposure=preInterpExp,
1510 outputExposure=ccdExposure,
1511 outputOssThumbnail=ossThumb,
1512 outputFlattenedThumbnail=flattenedThumb,
1515 @pipeBase.timeMethod
1517 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1519 This method contains the `CmdLineTask` interface to the ISR 1520 processing. All IO is handled here, freeing the `run()` method 1521 to manage only pixel-level calculations. The steps performed 1523 - Read in necessary detrending/isr/calibration data. 1524 - Process raw exposure in `run()`. 1525 - Persist the ISR-corrected exposure as "postISRCCD" if 1526 config.doWrite=True. 1530 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1531 DataRef of the detector data to be processed 1535 result : `lsst.pipe.base.Struct` 1536 Result struct with component: 1537 - ``exposure`` : `afw.image.Exposure` 1538 The fully ISR corrected exposure. 1543 Raised if a configuration option is set to True, but the 1544 required calibration data does not exist. 1547 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1549 ccdExposure = sensorRef.get(self.config.datasetType)
1551 camera = sensorRef.get(
"camera")
1552 isrData = self.
readIsrData(sensorRef, ccdExposure)
1554 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1556 if self.config.doWrite:
1557 sensorRef.put(result.exposure,
"postISRCCD")
1558 if result.preInterpolatedExposure
is not None:
1559 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1560 if result.ossThumb
is not None:
1561 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1562 if result.flattenedThumb
is not None:
1563 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1568 """!Retrieve a calibration dataset for removing instrument signature. 1573 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1574 DataRef of the detector data to find calibration datasets 1577 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1578 dateObs : `str`, optional 1579 Date of the observation. Used to correct butler failures 1580 when using fallback filters. 1582 If True, disable butler proxies to enable error handling 1583 within this routine. 1587 exposure : `lsst.afw.image.Exposure` 1588 Requested calibration frame. 1593 Raised if no matching calibration frame can be found. 1596 exp = dataRef.get(datasetType, immediate=immediate)
1597 except Exception
as exc1:
1598 if not self.config.fallbackFilterName:
1599 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1601 if self.config.useFallbackDate
and dateObs:
1602 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1603 dateObs=dateObs, immediate=immediate)
1605 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1606 except Exception
as exc2:
1607 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1608 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1609 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1611 if self.config.doAssembleIsrExposures:
1612 exp = self.assembleCcd.assembleCcd(exp)
1616 """Ensure that the data returned by Butler is a fully constructed exposure. 1618 ISR requires exposure-level image data for historical reasons, so if we did 1619 not recieve that from Butler, construct it from what we have, modifying the 1624 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1625 `lsst.afw.image.ImageF` 1626 The input data structure obtained from Butler. 1627 camera : `lsst.afw.cameraGeom.camera` 1628 The camera associated with the image. Used to find the appropriate 1631 The detector this exposure should match. 1635 inputExp : `lsst.afw.image.Exposure` 1636 The re-constructed exposure, with appropriate detector parameters. 1641 Raised if the input data cannot be used to construct an exposure. 1643 if isinstance(inputExp, afwImage.DecoratedImageU):
1644 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1645 elif isinstance(inputExp, afwImage.ImageF):
1646 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1647 elif isinstance(inputExp, afwImage.MaskedImageF):
1648 inputExp = afwImage.makeExposure(inputExp)
1649 elif isinstance(inputExp, afwImage.Exposure):
1651 elif inputExp
is None:
1655 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1658 if inputExp.getDetector()
is None:
1659 inputExp.setDetector(camera[detectorNum])
1664 """Convert exposure image from uint16 to float. 1666 If the exposure does not need to be converted, the input is 1667 immediately returned. For exposures that are converted to use 1668 floating point pixels, the variance is set to unity and the 1673 exposure : `lsst.afw.image.Exposure` 1674 The raw exposure to be converted. 1678 newexposure : `lsst.afw.image.Exposure` 1679 The input ``exposure``, converted to floating point pixels. 1684 Raised if the exposure type cannot be converted to float. 1687 if isinstance(exposure, afwImage.ExposureF):
1689 self.log.debug(
"Exposure already of type float.")
1691 if not hasattr(exposure,
"convertF"):
1692 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1694 newexposure = exposure.convertF()
1695 newexposure.variance[:] = 1
1696 newexposure.mask[:] = 0x0
1701 """Identify bad amplifiers, saturated and suspect pixels. 1705 ccdExposure : `lsst.afw.image.Exposure` 1706 Input exposure to be masked. 1707 amp : `lsst.afw.table.AmpInfoCatalog` 1708 Catalog of parameters defining the amplifier on this 1710 defects : `lsst.meas.algorithms.Defects` 1711 List of defects. Used to determine if the entire 1717 If this is true, the entire amplifier area is covered by 1718 defects and unusable. 1721 maskedImage = ccdExposure.getMaskedImage()
1727 if defects
is not None:
1728 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1733 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1735 maskView = dataView.getMask()
1736 maskView |= maskView.getPlaneBitMask(
"BAD")
1743 if self.config.doSaturation
and not badAmp:
1744 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1745 if self.config.doSuspect
and not badAmp:
1746 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1747 if math.isfinite(self.config.saturation):
1748 limits.update({self.config.saturatedMaskName: self.config.saturation})
1750 for maskName, maskThreshold
in limits.items():
1751 if not math.isnan(maskThreshold):
1752 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1753 isrFunctions.makeThresholdMask(
1754 maskedImage=dataView,
1755 threshold=maskThreshold,
1761 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1763 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1764 self.config.suspectMaskName])
1765 if numpy.all(maskView.getArray() & maskVal > 0):
1767 maskView |= maskView.getPlaneBitMask(
"BAD")
1772 """Apply overscan correction in place. 1774 This method does initial pixel rejection of the overscan 1775 region. The overscan can also be optionally segmented to 1776 allow for discontinuous overscan responses to be fit 1777 separately. The actual overscan subtraction is performed by 1778 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1779 which is called here after the amplifier is preprocessed. 1783 ccdExposure : `lsst.afw.image.Exposure` 1784 Exposure to have overscan correction performed. 1785 amp : `lsst.afw.table.AmpInfoCatalog` 1786 The amplifier to consider while correcting the overscan. 1790 overscanResults : `lsst.pipe.base.Struct` 1791 Result struct with components: 1792 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1793 Value or fit subtracted from the amplifier image data. 1794 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1795 Value or fit subtracted from the overscan image data. 1796 - ``overscanImage`` : `lsst.afw.image.Image` 1797 Image of the overscan region with the overscan 1798 correction applied. This quantity is used to estimate 1799 the amplifier read noise empirically. 1804 Raised if the ``amp`` does not contain raw pixel information. 1808 lsst.ip.isr.isrFunctions.overscanCorrection 1810 if not amp.getHasRawInfo():
1811 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1813 if amp.getRawHorizontalOverscanBBox().isEmpty():
1814 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1817 statControl = afwMath.StatisticsControl()
1818 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1821 dataBBox = amp.getRawDataBBox()
1822 oscanBBox = amp.getRawHorizontalOverscanBBox()
1826 prescanBBox = amp.getRawPrescanBBox()
1827 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1828 dx0 += self.config.overscanNumLeadingColumnsToSkip
1829 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1831 dx0 += self.config.overscanNumTrailingColumnsToSkip
1832 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1838 if ((self.config.overscanBiasJump
and 1839 self.config.overscanBiasJumpLocation)
and 1840 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1841 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1842 self.config.overscanBiasJumpDevices)):
1843 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1844 yLower = self.config.overscanBiasJumpLocation
1845 yUpper = dataBBox.getHeight() - yLower
1847 yUpper = self.config.overscanBiasJumpLocation
1848 yLower = dataBBox.getHeight() - yUpper
1867 oscanBBox.getHeight())))
1870 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1871 ampImage = ccdExposure.maskedImage[imageBBox]
1872 overscanImage = ccdExposure.maskedImage[overscanBBox]
1874 overscanArray = overscanImage.image.array
1875 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1876 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1877 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1879 statControl = afwMath.StatisticsControl()
1880 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1882 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1883 overscanImage=overscanImage,
1884 fitType=self.config.overscanFitType,
1885 order=self.config.overscanOrder,
1886 collapseRej=self.config.overscanNumSigmaClip,
1887 statControl=statControl,
1888 overscanIsInt=self.config.overscanIsInt
1892 levelStat = afwMath.MEDIAN
1893 sigmaStat = afwMath.STDEVCLIP
1895 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1896 self.config.qa.flatness.nIter)
1897 metadata = ccdExposure.getMetadata()
1898 ampNum = amp.getName()
1899 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1900 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1901 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1903 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1904 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1905 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1907 return overscanResults
1910 """Set the variance plane using the amplifier gain and read noise 1912 The read noise is calculated from the ``overscanImage`` if the 1913 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1914 the value from the amplifier data is used. 1918 ampExposure : `lsst.afw.image.Exposure` 1919 Exposure to process. 1920 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1921 Amplifier detector data. 1922 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1923 Image of overscan, required only for empirical read noise. 1927 lsst.ip.isr.isrFunctions.updateVariance 1929 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1930 gain = amp.getGain()
1932 if math.isnan(gain):
1934 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1937 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
1938 amp.getName(), gain, patchedGain)
1941 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1942 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
1944 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1945 stats = afwMath.StatisticsControl()
1946 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1947 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1948 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
1949 amp.getName(), readNoise)
1951 readNoise = amp.getReadNoise()
1953 isrFunctions.updateVariance(
1954 maskedImage=ampExposure.getMaskedImage(),
1956 readNoise=readNoise,
1960 """!Apply dark correction in place. 1964 exposure : `lsst.afw.image.Exposure` 1965 Exposure to process. 1966 darkExposure : `lsst.afw.image.Exposure` 1967 Dark exposure of the same size as ``exposure``. 1968 invert : `Bool`, optional 1969 If True, re-add the dark to an already corrected image. 1974 Raised if either ``exposure`` or ``darkExposure`` do not 1975 have their dark time defined. 1979 lsst.ip.isr.isrFunctions.darkCorrection 1981 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1982 if math.isnan(expScale):
1983 raise RuntimeError(
"Exposure darktime is NAN.")
1984 if darkExposure.getInfo().getVisitInfo()
is not None:
1985 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1989 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1992 if math.isnan(darkScale):
1993 raise RuntimeError(
"Dark calib darktime is NAN.")
1994 isrFunctions.darkCorrection(
1995 maskedImage=exposure.getMaskedImage(),
1996 darkMaskedImage=darkExposure.getMaskedImage(),
1998 darkScale=darkScale,
2000 trimToFit=self.config.doTrimToMatchCalib
2004 """!Check if linearization is needed for the detector cameraGeom. 2006 Checks config.doLinearize and the linearity type of the first 2011 detector : `lsst.afw.cameraGeom.Detector` 2012 Detector to get linearity type from. 2016 doLinearize : `Bool` 2017 If True, linearization should be performed. 2019 return self.config.doLinearize
and \
2020 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2023 """!Apply flat correction in place. 2027 exposure : `lsst.afw.image.Exposure` 2028 Exposure to process. 2029 flatExposure : `lsst.afw.image.Exposure` 2030 Flat exposure of the same size as ``exposure``. 2031 invert : `Bool`, optional 2032 If True, unflatten an already flattened image. 2036 lsst.ip.isr.isrFunctions.flatCorrection 2038 isrFunctions.flatCorrection(
2039 maskedImage=exposure.getMaskedImage(),
2040 flatMaskedImage=flatExposure.getMaskedImage(),
2041 scalingType=self.config.flatScalingType,
2042 userScale=self.config.flatUserScale,
2044 trimToFit=self.config.doTrimToMatchCalib
2048 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 2052 exposure : `lsst.afw.image.Exposure` 2053 Exposure to process. Only the amplifier DataSec is processed. 2054 amp : `lsst.afw.table.AmpInfoCatalog` 2055 Amplifier detector data. 2059 lsst.ip.isr.isrFunctions.makeThresholdMask 2061 if not math.isnan(amp.getSaturation()):
2062 maskedImage = exposure.getMaskedImage()
2063 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2064 isrFunctions.makeThresholdMask(
2065 maskedImage=dataView,
2066 threshold=amp.getSaturation(),
2068 maskName=self.config.saturatedMaskName,
2072 """!Interpolate over saturated pixels, in place. 2074 This method should be called after `saturationDetection`, to 2075 ensure that the saturated pixels have been identified in the 2076 SAT mask. It should also be called after `assembleCcd`, since 2077 saturated regions may cross amplifier boundaries. 2081 exposure : `lsst.afw.image.Exposure` 2082 Exposure to process. 2086 lsst.ip.isr.isrTask.saturationDetection 2087 lsst.ip.isr.isrFunctions.interpolateFromMask 2089 isrFunctions.interpolateFromMask(
2090 maskedImage=exposure.getMaskedImage(),
2091 fwhm=self.config.fwhm,
2092 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2093 maskNameList=list(self.config.saturatedMaskName),
2097 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 2101 exposure : `lsst.afw.image.Exposure` 2102 Exposure to process. Only the amplifier DataSec is processed. 2103 amp : `lsst.afw.table.AmpInfoCatalog` 2104 Amplifier detector data. 2108 lsst.ip.isr.isrFunctions.makeThresholdMask 2112 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 2113 This is intended to indicate pixels that may be affected by unknown systematics; 2114 for example if non-linearity corrections above a certain level are unstable 2115 then that would be a useful value for suspectLevel. A value of `nan` indicates 2116 that no such level exists and no pixels are to be masked as suspicious. 2118 suspectLevel = amp.getSuspectLevel()
2119 if math.isnan(suspectLevel):
2122 maskedImage = exposure.getMaskedImage()
2123 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2124 isrFunctions.makeThresholdMask(
2125 maskedImage=dataView,
2126 threshold=suspectLevel,
2128 maskName=self.config.suspectMaskName,
2132 """!Mask defects using mask plane "BAD", in place. 2136 exposure : `lsst.afw.image.Exposure` 2137 Exposure to process. 2138 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 2139 `lsst.afw.image.DefectBase`. 2140 List of defects to mask. 2144 Call this after CCD assembly, since defects may cross amplifier boundaries. 2146 maskedImage = exposure.getMaskedImage()
2147 if not isinstance(defectBaseList, Defects):
2149 defectList = Defects(defectBaseList)
2151 defectList = defectBaseList
2152 defectList.maskPixels(maskedImage, maskName=
"BAD")
2154 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2155 """!Mask edge pixels with applicable mask plane. 2159 exposure : `lsst.afw.image.Exposure` 2160 Exposure to process. 2161 numEdgePixels : `int`, optional 2162 Number of edge pixels to mask. 2163 maskPlane : `str`, optional 2164 Mask plane name to use. 2166 maskedImage = exposure.getMaskedImage()
2167 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2169 if numEdgePixels > 0:
2170 goodBBox = maskedImage.getBBox()
2172 goodBBox.grow(-numEdgePixels)
2174 SourceDetectionTask.setEdgeBits(
2181 """Mask and interpolate defects using mask plane "BAD", in place. 2185 exposure : `lsst.afw.image.Exposure` 2186 Exposure to process. 2187 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 2188 `lsst.afw.image.DefectBase`. 2189 List of defects to mask and interpolate. 2193 lsst.ip.isr.isrTask.maskDefect() 2196 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2197 maskPlane=
"SUSPECT")
2198 isrFunctions.interpolateFromMask(
2199 maskedImage=exposure.getMaskedImage(),
2200 fwhm=self.config.fwhm,
2201 growSaturatedFootprints=0,
2202 maskNameList=[
"BAD"],
2206 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 2210 exposure : `lsst.afw.image.Exposure` 2211 Exposure to process. 2215 We mask over all NaNs, including those that are masked with 2216 other bits (because those may or may not be interpolated over 2217 later, and we want to remove all NaNs). Despite this 2218 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 2219 the historical name. 2221 maskedImage = exposure.getMaskedImage()
2224 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2225 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2226 numNans =
maskNans(maskedImage, maskVal)
2227 self.metadata.set(
"NUMNANS", numNans)
2229 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2232 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 2236 exposure : `lsst.afw.image.Exposure` 2237 Exposure to process. 2241 lsst.ip.isr.isrTask.maskNan() 2244 isrFunctions.interpolateFromMask(
2245 maskedImage=exposure.getMaskedImage(),
2246 fwhm=self.config.fwhm,
2247 growSaturatedFootprints=0,
2248 maskNameList=[
"UNMASKEDNAN"],
2252 """Measure the image background in subgrids, for quality control purposes. 2256 exposure : `lsst.afw.image.Exposure` 2257 Exposure to process. 2258 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 2259 Configuration object containing parameters on which background 2260 statistics and subgrids to use. 2262 if IsrQaConfig
is not None:
2263 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2264 IsrQaConfig.flatness.nIter)
2265 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2266 statsControl.setAndMask(maskVal)
2267 maskedImage = exposure.getMaskedImage()
2268 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2269 skyLevel = stats.getValue(afwMath.MEDIAN)
2270 skySigma = stats.getValue(afwMath.STDEVCLIP)
2271 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2272 metadata = exposure.getMetadata()
2273 metadata.set(
'SKYLEVEL', skyLevel)
2274 metadata.set(
'SKYSIGMA', skySigma)
2277 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2278 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2279 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2280 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2281 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2282 skyLevels = numpy.zeros((nX, nY))
2285 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2287 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2289 xLLC = xc - meshXHalf
2290 yLLC = yc - meshYHalf
2291 xURC = xc + meshXHalf - 1
2292 yURC = yc + meshYHalf - 1
2295 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2297 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2299 good = numpy.where(numpy.isfinite(skyLevels))
2300 skyMedian = numpy.median(skyLevels[good])
2301 flatness = (skyLevels[good] - skyMedian) / skyMedian
2302 flatness_rms = numpy.std(flatness)
2303 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2305 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2306 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2307 nX, nY, flatness_pp, flatness_rms)
2309 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2310 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2311 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2312 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2313 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2316 """Set an approximate magnitude zero point for the exposure. 2320 exposure : `lsst.afw.image.Exposure` 2321 Exposure to process. 2323 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2324 if filterName
in self.config.fluxMag0T1:
2325 fluxMag0 = self.config.fluxMag0T1[filterName]
2327 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2328 fluxMag0 = self.config.defaultFluxMag0T1
2330 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2332 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2335 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2336 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2339 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2343 ccdExposure : `lsst.afw.image.Exposure` 2344 Exposure to process. 2345 fpPolygon : `lsst.afw.geom.Polygon` 2346 Polygon in focal plane coordinates. 2349 ccd = ccdExposure.getDetector()
2350 fpCorners = ccd.getCorners(FOCAL_PLANE)
2351 ccdPolygon = Polygon(fpCorners)
2354 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2357 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2358 validPolygon = Polygon(ccdPoints)
2359 ccdExposure.getInfo().setValidPolygon(validPolygon)
2363 """Context manager that applies and removes flats and darks, 2364 if the task is configured to apply them. 2368 exp : `lsst.afw.image.Exposure` 2369 Exposure to process. 2370 flat : `lsst.afw.image.Exposure` 2371 Flat exposure the same size as ``exp``. 2372 dark : `lsst.afw.image.Exposure`, optional 2373 Dark exposure the same size as ``exp``. 2377 exp : `lsst.afw.image.Exposure` 2378 The flat and dark corrected exposure. 2380 if self.config.doDark
and dark
is not None:
2382 if self.config.doFlat:
2387 if self.config.doFlat:
2389 if self.config.doDark
and dark
is not None:
2393 """Utility function to examine ISR exposure at different stages. 2397 exposure : `lsst.afw.image.Exposure` 2400 State of processing to view. 2402 frame = getDebugFrame(self._display, stepname)
2404 display = getDisplay(frame)
2405 display.scale(
'asinh',
'zscale')
2406 display.mtv(exposure)
2407 prompt =
"Press Enter to continue [c]... " 2409 ans = input(prompt).lower()
2410 if ans
in (
"",
"c",):
2415 """A Detector-like object that supports returning gain and saturation level 2417 This is used when the input exposure does not have a detector. 2421 exposure : `lsst.afw.image.Exposure` 2422 Exposure to generate a fake amplifier for. 2423 config : `lsst.ip.isr.isrTaskConfig` 2424 Configuration to apply to the fake amplifier. 2428 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2430 self.
_gain = config.gain
2460 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2464 """Task to wrap the default IsrTask to allow it to be retargeted. 2466 The standard IsrTask can be called directly from a command line 2467 program, but doing so removes the ability of the task to be 2468 retargeted. As most cameras override some set of the IsrTask 2469 methods, this would remove those data-specific methods in the 2470 output post-ISR images. This wrapping class fixes the issue, 2471 allowing identical post-ISR images to be generated by both the 2472 processCcd and isrTask code. 2474 ConfigClass = RunIsrConfig
2475 _DefaultName =
"runIsr" 2479 self.makeSubtask(
"isr")
2485 dataRef : `lsst.daf.persistence.ButlerDataRef` 2486 data reference of the detector data to be processed 2490 result : `pipeBase.Struct` 2491 Result struct with component: 2493 - exposure : `lsst.afw.image.Exposure` 2494 Post-ISR processed exposure.
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def getRawHorizontalOverscanBBox(self)
def maskNan(self, exposure)
def getSuspectLevel(self)
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def __init__(self, config=None)
def flatContext(self, exp, flat, dark=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def maskAndInterpolateNan(self, exposure)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
doSaturationInterpolation
def __init__(self, exposure, config)