28 import lsst.pex.config
as pexConfig
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .straylight
import StrayLightTask
54 from .vignette
import VignetteTask
57 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
61 dimensions={
"instrument",
"visit",
"detector"},
63 ccdExposure = cT.Input(
65 doc=
"Input exposure to process.",
66 storageClass=
"Exposure",
67 dimensions=[
"instrument",
"visit",
"detector",
"exposure"],
69 camera = cT.PrerequisiteInput(
71 storageClass=
"Camera",
72 doc=
"Input camera to construct complete exposures.",
73 dimensions=[
"instrument",
"calibration_label"],
75 bias = cT.PrerequisiteInput(
77 doc=
"Input bias calibration.",
78 storageClass=
"ExposureF",
79 dimensions=[
"instrument",
"calibration_label",
"detector"],
81 dark = cT.PrerequisiteInput(
83 doc=
"Input dark calibration.",
84 storageClass=
"ExposureF",
85 dimensions=[
"instrument",
"calibration_label",
"detector"],
87 flat = cT.PrerequisiteInput(
89 doc=
"Input flat calibration.",
90 storageClass=
"ExposureF",
91 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
93 fringes = cT.PrerequisiteInput(
95 doc=
"Input fringe calibration.",
96 storageClass=
"ExposureF",
97 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
99 strayLightData = cT.PrerequisiteInput(
101 doc=
"Input stray light calibration.",
102 storageClass=
"StrayLightData",
103 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
105 bfKernel = cT.PrerequisiteInput(
107 doc=
"Input brighter-fatter kernel.",
108 storageClass=
"NumpyArray",
109 dimensions=[
"instrument",
"calibration_label"],
111 newBFKernel = cT.PrerequisiteInput(
112 name=
'brighterFatterKernel',
113 doc=
"Newer complete kernel + gain solutions.",
114 storageClass=
"BrighterFatterKernel",
115 dimensions=[
"instrument",
"calibration_label",
"detector"],
117 defects = cT.PrerequisiteInput(
119 doc=
"Input defect tables.",
120 storageClass=
"Defects",
121 dimensions=[
"instrument",
"calibration_label",
"detector"],
123 opticsTransmission = cT.PrerequisiteInput(
124 name=
"transmission_optics",
125 storageClass=
"TransmissionCurve",
126 doc=
"Transmission curve due to the optics.",
127 dimensions=[
"instrument",
"calibration_label"],
129 filterTransmission = cT.PrerequisiteInput(
130 name=
"transmission_filter",
131 storageClass=
"TransmissionCurve",
132 doc=
"Transmission curve due to the filter.",
133 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
135 sensorTransmission = cT.PrerequisiteInput(
136 name=
"transmission_sensor",
137 storageClass=
"TransmissionCurve",
138 doc=
"Transmission curve due to the sensor.",
139 dimensions=[
"instrument",
"calibration_label",
"detector"],
141 atmosphereTransmission = cT.PrerequisiteInput(
142 name=
"transmission_atmosphere",
143 storageClass=
"TransmissionCurve",
144 doc=
"Transmission curve due to the atmosphere.",
145 dimensions=[
"instrument"],
147 illumMaskedImage = cT.PrerequisiteInput(
149 doc=
"Input illumination correction.",
150 storageClass=
"MaskedImageF",
151 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
154 outputExposure = cT.Output(
156 doc=
"Output ISR processed exposure.",
157 storageClass=
"ExposureF",
158 dimensions=[
"instrument",
"visit",
"detector"],
160 preInterpExposure = cT.Output(
161 name=
'preInterpISRCCD',
162 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
163 storageClass=
"ExposureF",
164 dimensions=[
"instrument",
"visit",
"detector"],
166 outputOssThumbnail = cT.Output(
168 doc=
"Output Overscan-subtracted thumbnail image.",
169 storageClass=
"Thumbnail",
170 dimensions=[
"instrument",
"visit",
"detector"],
172 outputFlattenedThumbnail = cT.Output(
173 name=
"FlattenedThumb",
174 doc=
"Output flat-corrected thumbnail image.",
175 storageClass=
"Thumbnail",
176 dimensions=[
"instrument",
"visit",
"detector"],
182 if config.doBias
is not True:
183 self.prerequisiteInputs.discard(
"bias")
184 if config.doLinearize
is not True:
185 self.prerequisiteInputs.discard(
"linearizer")
186 if config.doCrosstalk
is not True:
187 self.prerequisiteInputs.discard(
"crosstalkSources")
188 if config.doBrighterFatter
is not True:
189 self.prerequisiteInputs.discard(
"bfKernel")
190 self.prerequisiteInputs.discard(
"newBFKernel")
191 if config.doDefect
is not True:
192 self.prerequisiteInputs.discard(
"defects")
193 if config.doDark
is not True:
194 self.prerequisiteInputs.discard(
"dark")
195 if config.doFlat
is not True:
196 self.prerequisiteInputs.discard(
"flat")
197 if config.doAttachTransmissionCurve
is not True:
198 self.prerequisiteInputs.discard(
"opticsTransmission")
199 self.prerequisiteInputs.discard(
"filterTransmission")
200 self.prerequisiteInputs.discard(
"sensorTransmission")
201 self.prerequisiteInputs.discard(
"atmosphereTransmission")
202 if config.doUseOpticsTransmission
is not True:
203 self.prerequisiteInputs.discard(
"opticsTransmission")
204 if config.doUseFilterTransmission
is not True:
205 self.prerequisiteInputs.discard(
"filterTransmission")
206 if config.doUseSensorTransmission
is not True:
207 self.prerequisiteInputs.discard(
"sensorTransmission")
208 if config.doUseAtmosphereTransmission
is not True:
209 self.prerequisiteInputs.discard(
"atmosphereTransmission")
210 if config.doIlluminationCorrection
is not True:
211 self.prerequisiteInputs.discard(
"illumMaskedImage")
213 if config.doWrite
is not True:
214 self.outputs.discard(
"outputExposure")
215 self.outputs.discard(
"preInterpExposure")
216 self.outputs.discard(
"outputFlattenedThumbnail")
217 self.outputs.discard(
"outputOssThumbnail")
218 if config.doSaveInterpPixels
is not True:
219 self.outputs.discard(
"preInterpExposure")
220 if config.qa.doThumbnailOss
is not True:
221 self.outputs.discard(
"outputOssThumbnail")
222 if config.qa.doThumbnailFlattened
is not True:
223 self.outputs.discard(
"outputFlattenedThumbnail")
227 pipelineConnections=IsrTaskConnections):
228 """Configuration parameters for IsrTask. 230 Items are grouped in the order in which they are executed by the task. 232 datasetType = pexConfig.Field(
234 doc=
"Dataset type for input data; users will typically leave this alone, " 235 "but camera-specific ISR tasks will override it",
239 fallbackFilterName = pexConfig.Field(
241 doc=
"Fallback default filter name for calibrations.",
244 useFallbackDate = pexConfig.Field(
246 doc=
"Pass observation date when using fallback filter.",
249 expectWcs = pexConfig.Field(
252 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 254 fwhm = pexConfig.Field(
256 doc=
"FWHM of PSF in arcseconds.",
259 qa = pexConfig.ConfigField(
261 doc=
"QA related configuration options.",
265 doConvertIntToFloat = pexConfig.Field(
267 doc=
"Convert integer raw images to floating point values?",
272 doSaturation = pexConfig.Field(
274 doc=
"Mask saturated pixels? NB: this is totally independent of the" 275 " interpolation option - this is ONLY setting the bits in the mask." 276 " To have them interpolated make sure doSaturationInterpolation=True",
279 saturatedMaskName = pexConfig.Field(
281 doc=
"Name of mask plane to use in saturation detection and interpolation",
284 saturation = pexConfig.Field(
286 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
287 default=float(
"NaN"),
289 growSaturationFootprintSize = pexConfig.Field(
291 doc=
"Number of pixels by which to grow the saturation footprints",
296 doSuspect = pexConfig.Field(
298 doc=
"Mask suspect pixels?",
301 suspectMaskName = pexConfig.Field(
303 doc=
"Name of mask plane to use for suspect pixels",
306 numEdgeSuspect = pexConfig.Field(
308 doc=
"Number of edge pixels to be flagged as untrustworthy.",
313 doSetBadRegions = pexConfig.Field(
315 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
318 badStatistic = pexConfig.ChoiceField(
320 doc=
"How to estimate the average value for BAD regions.",
323 "MEANCLIP":
"Correct using the (clipped) mean of good data",
324 "MEDIAN":
"Correct using the median of the good data",
329 doOverscan = pexConfig.Field(
331 doc=
"Do overscan subtraction?",
334 overscanFitType = pexConfig.ChoiceField(
336 doc=
"The method for fitting the overscan bias level.",
339 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
340 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
341 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
342 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
343 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
344 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
345 "MEAN":
"Correct using the mean of the overscan region",
346 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
347 "MEDIAN":
"Correct using the median of the overscan region",
348 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
351 overscanOrder = pexConfig.Field(
353 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
354 "or number of spline knots if overscan fit type is a spline."),
357 overscanNumSigmaClip = pexConfig.Field(
359 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
362 overscanIsInt = pexConfig.Field(
364 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" +
365 " and overscan.FitType=MEDIAN_PER_ROW.",
368 overscanNumLeadingColumnsToSkip = pexConfig.Field(
370 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
373 overscanNumTrailingColumnsToSkip = pexConfig.Field(
375 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
378 overscanMaxDev = pexConfig.Field(
380 doc=
"Maximum deviation from the median for overscan",
381 default=1000.0, check=
lambda x: x > 0
383 overscanBiasJump = pexConfig.Field(
385 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
388 overscanBiasJumpKeyword = pexConfig.Field(
390 doc=
"Header keyword containing information about devices.",
391 default=
"NO_SUCH_KEY",
393 overscanBiasJumpDevices = pexConfig.ListField(
395 doc=
"List of devices that need piecewise overscan correction.",
398 overscanBiasJumpLocation = pexConfig.Field(
400 doc=
"Location of bias jump along y-axis.",
405 doAssembleCcd = pexConfig.Field(
408 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 410 assembleCcd = pexConfig.ConfigurableField(
411 target=AssembleCcdTask,
412 doc=
"CCD assembly task",
416 doAssembleIsrExposures = pexConfig.Field(
419 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 421 doTrimToMatchCalib = pexConfig.Field(
424 doc=
"Trim raw data to match calibration bounding boxes?" 428 doBias = pexConfig.Field(
430 doc=
"Apply bias frame correction?",
433 biasDataProductName = pexConfig.Field(
435 doc=
"Name of the bias data product",
440 doVariance = pexConfig.Field(
442 doc=
"Calculate variance?",
445 gain = pexConfig.Field(
447 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
448 default=float(
"NaN"),
450 readNoise = pexConfig.Field(
452 doc=
"The read noise to use if no Detector is present in the Exposure",
455 doEmpiricalReadNoise = pexConfig.Field(
458 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 462 doLinearize = pexConfig.Field(
464 doc=
"Correct for nonlinearity of the detector's response?",
469 doCrosstalk = pexConfig.Field(
471 doc=
"Apply intra-CCD crosstalk correction?",
474 doCrosstalkBeforeAssemble = pexConfig.Field(
476 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
479 crosstalk = pexConfig.ConfigurableField(
480 target=CrosstalkTask,
481 doc=
"Intra-CCD crosstalk correction",
485 doDefect = pexConfig.Field(
487 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
490 doNanMasking = pexConfig.Field(
492 doc=
"Mask NAN pixels?",
495 doWidenSaturationTrails = pexConfig.Field(
497 doc=
"Widen bleed trails based on their width?",
502 doBrighterFatter = pexConfig.Field(
505 doc=
"Apply the brighter fatter correction" 507 brighterFatterLevel = pexConfig.ChoiceField(
510 doc=
"The level at which to correct for brighter-fatter.",
512 "AMP":
"Every amplifier treated separately.",
513 "DETECTOR":
"One kernel per detector",
516 brighterFatterMaxIter = pexConfig.Field(
519 doc=
"Maximum number of iterations for the brighter fatter correction" 521 brighterFatterThreshold = pexConfig.Field(
524 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 525 " absolute value of the difference between the current corrected image and the one" 526 " from the previous iteration summed over all the pixels." 528 brighterFatterApplyGain = pexConfig.Field(
531 doc=
"Should the gain be applied when applying the brighter fatter correction?" 533 brighterFatterMaskGrowSize = pexConfig.Field(
536 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate " 537 " when brighter-fatter correction is applied." 541 doDark = pexConfig.Field(
543 doc=
"Apply dark frame correction?",
546 darkDataProductName = pexConfig.Field(
548 doc=
"Name of the dark data product",
553 doStrayLight = pexConfig.Field(
555 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
558 strayLight = pexConfig.ConfigurableField(
559 target=StrayLightTask,
560 doc=
"y-band stray light correction" 564 doFlat = pexConfig.Field(
566 doc=
"Apply flat field correction?",
569 flatDataProductName = pexConfig.Field(
571 doc=
"Name of the flat data product",
574 flatScalingType = pexConfig.ChoiceField(
576 doc=
"The method for scaling the flat on the fly.",
579 "USER":
"Scale by flatUserScale",
580 "MEAN":
"Scale by the inverse of the mean",
581 "MEDIAN":
"Scale by the inverse of the median",
584 flatUserScale = pexConfig.Field(
586 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
589 doTweakFlat = pexConfig.Field(
591 doc=
"Tweak flats to match observed amplifier ratios?",
596 doApplyGains = pexConfig.Field(
598 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
601 normalizeGains = pexConfig.Field(
603 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
608 doFringe = pexConfig.Field(
610 doc=
"Apply fringe correction?",
613 fringe = pexConfig.ConfigurableField(
615 doc=
"Fringe subtraction task",
617 fringeAfterFlat = pexConfig.Field(
619 doc=
"Do fringe subtraction after flat-fielding?",
624 doAddDistortionModel = pexConfig.Field(
626 doc=
"Apply a distortion model based on camera geometry to the WCS?",
628 deprecated=(
"Camera geometry is incorporated when reading the raw files." 629 " This option no longer is used, and will be removed after v19.")
633 doMeasureBackground = pexConfig.Field(
635 doc=
"Measure the background level on the reduced image?",
640 doCameraSpecificMasking = pexConfig.Field(
642 doc=
"Mask camera-specific bad regions?",
645 masking = pexConfig.ConfigurableField(
652 doInterpolate = pexConfig.Field(
654 doc=
"Interpolate masked pixels?",
657 doSaturationInterpolation = pexConfig.Field(
659 doc=
"Perform interpolation over pixels masked as saturated?" 660 " NB: This is independent of doSaturation; if that is False this plane" 661 " will likely be blank, resulting in a no-op here.",
664 doNanInterpolation = pexConfig.Field(
666 doc=
"Perform interpolation over pixels masked as NaN?" 667 " NB: This is independent of doNanMasking; if that is False this plane" 668 " will likely be blank, resulting in a no-op here.",
671 doNanInterpAfterFlat = pexConfig.Field(
673 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 674 "also have to interpolate them before flat-fielding."),
677 maskListToInterpolate = pexConfig.ListField(
679 doc=
"List of mask planes that should be interpolated.",
680 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
682 doSaveInterpPixels = pexConfig.Field(
684 doc=
"Save a copy of the pre-interpolated pixel values?",
689 fluxMag0T1 = pexConfig.DictField(
692 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
693 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
696 defaultFluxMag0T1 = pexConfig.Field(
698 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
699 default=pow(10.0, 0.4*28.0)
703 doVignette = pexConfig.Field(
705 doc=
"Apply vignetting parameters?",
708 vignette = pexConfig.ConfigurableField(
710 doc=
"Vignetting task.",
714 doAttachTransmissionCurve = pexConfig.Field(
717 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 719 doUseOpticsTransmission = pexConfig.Field(
722 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 724 doUseFilterTransmission = pexConfig.Field(
727 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 729 doUseSensorTransmission = pexConfig.Field(
732 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 734 doUseAtmosphereTransmission = pexConfig.Field(
737 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 741 doIlluminationCorrection = pexConfig.Field(
744 doc=
"Perform illumination correction?" 746 illuminationCorrectionDataProductName = pexConfig.Field(
748 doc=
"Name of the illumination correction data product.",
751 illumScale = pexConfig.Field(
753 doc=
"Scale factor for the illumination correction.",
756 illumFilters = pexConfig.ListField(
759 doc=
"Only perform illumination correction for these filters." 763 doWrite = pexConfig.Field(
765 doc=
"Persist postISRCCD?",
772 raise ValueError(
"You may not specify both doFlat and doApplyGains")
774 self.config.maskListToInterpolate.append(
"SAT")
776 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
779 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
780 """Apply common instrument signature correction algorithms to a raw frame. 782 The process for correcting imaging data is very similar from 783 camera to camera. This task provides a vanilla implementation of 784 doing these corrections, including the ability to turn certain 785 corrections off if they are not needed. The inputs to the primary 786 method, `run()`, are a raw exposure to be corrected and the 787 calibration data products. The raw input is a single chip sized 788 mosaic of all amps including overscans and other non-science 789 pixels. The method `runDataRef()` identifies and defines the 790 calibration data products, and is intended for use by a 791 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 792 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 793 subclassed for different camera, although the most camera specific 794 methods have been split into subtasks that can be redirected 797 The __init__ method sets up the subtasks for ISR processing, using 798 the defaults from `lsst.ip.isr`. 803 Positional arguments passed to the Task constructor. None used at this time. 804 kwargs : `dict`, optional 805 Keyword arguments passed on to the Task constructor. None used at this time. 807 ConfigClass = IsrTaskConfig
812 self.makeSubtask(
"assembleCcd")
813 self.makeSubtask(
"crosstalk")
814 self.makeSubtask(
"strayLight")
815 self.makeSubtask(
"fringe")
816 self.makeSubtask(
"masking")
817 self.makeSubtask(
"vignette")
820 inputs = butlerQC.get(inputRefs)
823 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
824 except Exception
as e:
825 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
828 inputs[
'isGen3'] =
True 830 detector = inputs[
'ccdExposure'].getDetector()
833 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
835 linearizer.fromYaml(inputs[
'linearizer'])
839 inputs[
'linearizer'] = linearizer
841 if self.config.doDefect
is True:
842 if "defects" in inputs
and inputs[
'defects']
is not None:
845 if not isinstance(inputs[
"defects"], Defects):
846 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
850 if self.config.doBrighterFatter:
851 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
852 if brighterFatterKernel
is None:
853 brighterFatterKernel = inputs.get(
'bfKernel',
None)
855 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
856 detId = detector.getId()
857 inputs[
'bfGains'] = brighterFatterKernel.gain
860 if self.config.brighterFatterLevel ==
'DETECTOR':
861 if brighterFatterKernel.detectorKernel:
862 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
863 elif brighterFatterKernel.detectorKernelFromAmpKernels:
864 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
866 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
869 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
877 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
878 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
879 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
881 assembler=self.assembleCcd
882 if self.config.doAssembleIsrExposures
else None)
884 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
886 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
887 if 'strayLightData' not in inputs:
888 inputs[
'strayLightData'] =
None 890 outputs = self.
run(**inputs)
891 butlerQC.put(outputs, outputRefs)
894 """!Retrieve necessary frames for instrument signature removal. 896 Pre-fetching all required ISR data products limits the IO 897 required by the ISR. Any conflict between the calibration data 898 available and that needed for ISR is also detected prior to 899 doing processing, allowing it to fail quickly. 903 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 904 Butler reference of the detector data to be processed 905 rawExposure : `afw.image.Exposure` 906 The raw exposure that will later be corrected with the 907 retrieved calibration data; should not be modified in this 912 result : `lsst.pipe.base.Struct` 913 Result struct with components (which may be `None`): 914 - ``bias``: bias calibration frame (`afw.image.Exposure`) 915 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 916 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 917 - ``dark``: dark calibration frame (`afw.image.Exposure`) 918 - ``flat``: flat calibration frame (`afw.image.Exposure`) 919 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 920 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 921 - ``fringes``: `lsst.pipe.base.Struct` with components: 922 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 923 - ``seed``: random seed derived from the ccdExposureId for random 924 number generator (`uint32`). 925 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 926 A ``TransmissionCurve`` that represents the throughput of the optics, 927 to be evaluated in focal-plane coordinates. 928 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 929 A ``TransmissionCurve`` that represents the throughput of the filter 930 itself, to be evaluated in focal-plane coordinates. 931 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 932 A ``TransmissionCurve`` that represents the throughput of the sensor 933 itself, to be evaluated in post-assembly trimmed detector coordinates. 934 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 935 A ``TransmissionCurve`` that represents the throughput of the 936 atmosphere, assumed to be spatially constant. 937 - ``strayLightData`` : `object` 938 An opaque object containing calibration information for 939 stray-light correction. If `None`, no correction will be 941 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`) 945 NotImplementedError : 946 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 949 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
950 dateObs = dateObs.toPython().isoformat()
952 self.log.warn(
"Unable to identify dateObs for rawExposure.")
955 ccd = rawExposure.getDetector()
956 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
957 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
958 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
959 if self.config.doBias
else None)
961 linearizer = (dataRef.get(
"linearizer", immediate=
True)
963 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
964 linearizer.log = self.log
965 if isinstance(linearizer, numpy.ndarray):
967 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
968 if self.config.doCrosstalk
else None)
969 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
970 if self.config.doDark
else None)
971 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
973 if self.config.doFlat
else None)
975 brighterFatterKernel =
None 976 brighterFatterGains =
None 977 if self.config.doBrighterFatter
is True:
982 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
983 brighterFatterGains = brighterFatterKernel.gain
984 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
987 brighterFatterKernel = dataRef.get(
"bfKernel")
988 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
990 brighterFatterKernel =
None 991 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
994 if self.config.brighterFatterLevel ==
'DETECTOR':
995 if brighterFatterKernel.detectorKernel:
996 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
997 elif brighterFatterKernel.detectorKernelFromAmpKernels:
998 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1000 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1003 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1005 defectList = (dataRef.get(
"defects")
1006 if self.config.doDefect
else None)
1007 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1008 if self.config.doAssembleIsrExposures
else None)
1009 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1010 else pipeBase.Struct(fringes=
None))
1012 if self.config.doAttachTransmissionCurve:
1013 opticsTransmission = (dataRef.get(
"transmission_optics")
1014 if self.config.doUseOpticsTransmission
else None)
1015 filterTransmission = (dataRef.get(
"transmission_filter")
1016 if self.config.doUseFilterTransmission
else None)
1017 sensorTransmission = (dataRef.get(
"transmission_sensor")
1018 if self.config.doUseSensorTransmission
else None)
1019 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1020 if self.config.doUseAtmosphereTransmission
else None)
1022 opticsTransmission =
None 1023 filterTransmission =
None 1024 sensorTransmission =
None 1025 atmosphereTransmission =
None 1027 if self.config.doStrayLight:
1028 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1030 strayLightData =
None 1033 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1034 if (self.config.doIlluminationCorrection
and 1035 filterName
in self.config.illumFilters)
1039 return pipeBase.Struct(bias=biasExposure,
1040 linearizer=linearizer,
1041 crosstalkSources=crosstalkSources,
1044 bfKernel=brighterFatterKernel,
1045 bfGains=brighterFatterGains,
1047 fringes=fringeStruct,
1048 opticsTransmission=opticsTransmission,
1049 filterTransmission=filterTransmission,
1050 sensorTransmission=sensorTransmission,
1051 atmosphereTransmission=atmosphereTransmission,
1052 strayLightData=strayLightData,
1053 illumMaskedImage=illumMaskedImage
1056 @pipeBase.timeMethod
1057 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1058 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1059 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1060 sensorTransmission=
None, atmosphereTransmission=
None,
1061 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1064 """!Perform instrument signature removal on an exposure. 1066 Steps included in the ISR processing, in order performed, are: 1067 - saturation and suspect pixel masking 1068 - overscan subtraction 1069 - CCD assembly of individual amplifiers 1071 - variance image construction 1072 - linearization of non-linear response 1074 - brighter-fatter correction 1077 - stray light subtraction 1079 - masking of known defects and camera specific features 1080 - vignette calculation 1081 - appending transmission curve and distortion model 1085 ccdExposure : `lsst.afw.image.Exposure` 1086 The raw exposure that is to be run through ISR. The 1087 exposure is modified by this method. 1088 camera : `lsst.afw.cameraGeom.Camera`, optional 1089 The camera geometry for this exposure. Used to select the 1090 distortion model appropriate for this data. 1091 bias : `lsst.afw.image.Exposure`, optional 1092 Bias calibration frame. 1093 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 1094 Functor for linearization. 1095 crosstalkSources : `list`, optional 1096 List of possible crosstalk sources. 1097 dark : `lsst.afw.image.Exposure`, optional 1098 Dark calibration frame. 1099 flat : `lsst.afw.image.Exposure`, optional 1100 Flat calibration frame. 1101 bfKernel : `numpy.ndarray`, optional 1102 Brighter-fatter kernel. 1103 bfGains : `dict` of `float`, optional 1104 Gains used to override the detector's nominal gains for the 1105 brighter-fatter correction. A dict keyed by amplifier name for 1106 the detector in question. 1107 defects : `lsst.meas.algorithms.Defects`, optional 1109 fringes : `lsst.pipe.base.Struct`, optional 1110 Struct containing the fringe correction data, with 1112 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 1113 - ``seed``: random seed derived from the ccdExposureId for random 1114 number generator (`uint32`) 1115 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 1116 A ``TransmissionCurve`` that represents the throughput of the optics, 1117 to be evaluated in focal-plane coordinates. 1118 filterTransmission : `lsst.afw.image.TransmissionCurve` 1119 A ``TransmissionCurve`` that represents the throughput of the filter 1120 itself, to be evaluated in focal-plane coordinates. 1121 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1122 A ``TransmissionCurve`` that represents the throughput of the sensor 1123 itself, to be evaluated in post-assembly trimmed detector coordinates. 1124 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1125 A ``TransmissionCurve`` that represents the throughput of the 1126 atmosphere, assumed to be spatially constant. 1127 detectorNum : `int`, optional 1128 The integer number for the detector to process. 1129 isGen3 : bool, optional 1130 Flag this call to run() as using the Gen3 butler environment. 1131 strayLightData : `object`, optional 1132 Opaque object containing calibration information for stray-light 1133 correction. If `None`, no correction will be performed. 1134 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 1135 Illumination correction image. 1139 result : `lsst.pipe.base.Struct` 1140 Result struct with component: 1141 - ``exposure`` : `afw.image.Exposure` 1142 The fully ISR corrected exposure. 1143 - ``outputExposure`` : `afw.image.Exposure` 1144 An alias for `exposure` 1145 - ``ossThumb`` : `numpy.ndarray` 1146 Thumbnail image of the exposure after overscan subtraction. 1147 - ``flattenedThumb`` : `numpy.ndarray` 1148 Thumbnail image of the exposure after flat-field correction. 1153 Raised if a configuration option is set to True, but the 1154 required calibration data has not been specified. 1158 The current processed exposure can be viewed by setting the 1159 appropriate lsstDebug entries in the `debug.display` 1160 dictionary. The names of these entries correspond to some of 1161 the IsrTaskConfig Boolean options, with the value denoting the 1162 frame to use. The exposure is shown inside the matching 1163 option check and after the processing of that step has 1164 finished. The steps with debug points are: 1175 In addition, setting the "postISRCCD" entry displays the 1176 exposure after all ISR processing has finished. 1184 if detectorNum
is None:
1185 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1187 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1192 if isinstance(ccdExposure, ButlerDataRef):
1195 ccd = ccdExposure.getDetector()
1196 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1199 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd." 1200 ccd = [
FakeAmp(ccdExposure, self.config)]
1203 if self.config.doBias
and bias
is None:
1204 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1206 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1207 if self.config.doBrighterFatter
and bfKernel
is None:
1208 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1209 if self.config.doDark
and dark
is None:
1210 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1211 if self.config.doFlat
and flat
is None:
1212 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1213 if self.config.doDefect
and defects
is None:
1214 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1215 if (self.config.doFringe
and filterName
in self.fringe.config.filters
and 1216 fringes.fringes
is None):
1221 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1222 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
and 1223 illumMaskedImage
is None):
1224 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1227 if self.config.doConvertIntToFloat:
1228 self.log.info(
"Converting exposure to floating point values.")
1235 if ccdExposure.getBBox().contains(amp.getBBox()):
1239 if self.config.doOverscan
and not badAmp:
1242 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1243 if overscanResults
is not None and \
1244 self.config.qa
is not None and self.config.qa.saveStats
is True:
1245 if isinstance(overscanResults.overscanFit, float):
1246 qaMedian = overscanResults.overscanFit
1247 qaStdev = float(
"NaN")
1249 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1250 afwMath.MEDIAN | afwMath.STDEVCLIP)
1251 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1252 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1254 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1255 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1256 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1257 amp.getName(), qaMedian, qaStdev)
1258 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1261 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1262 overscanResults =
None 1264 overscans.append(overscanResults
if overscanResults
is not None else None)
1266 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1268 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1269 self.log.info(
"Applying crosstalk correction.")
1270 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1271 self.
debugView(ccdExposure,
"doCrosstalk")
1273 if self.config.doAssembleCcd:
1274 self.log.info(
"Assembling CCD from amplifiers.")
1275 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1277 if self.config.expectWcs
and not ccdExposure.getWcs():
1278 self.log.warn(
"No WCS found in input exposure.")
1279 self.
debugView(ccdExposure,
"doAssembleCcd")
1282 if self.config.qa.doThumbnailOss:
1283 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1285 if self.config.doBias:
1286 self.log.info(
"Applying bias correction.")
1287 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1288 trimToFit=self.config.doTrimToMatchCalib)
1291 if self.config.doVariance:
1292 for amp, overscanResults
in zip(ccd, overscans):
1293 if ccdExposure.getBBox().contains(amp.getBBox()):
1294 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1295 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1296 if overscanResults
is not None:
1298 overscanImage=overscanResults.overscanImage)
1302 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1303 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1304 afwMath.MEDIAN | afwMath.STDEVCLIP)
1305 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1306 qaStats.getValue(afwMath.MEDIAN))
1307 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1308 qaStats.getValue(afwMath.STDEVCLIP))
1309 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1310 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1311 qaStats.getValue(afwMath.STDEVCLIP))
1314 self.log.info(
"Applying linearizer.")
1315 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1316 detector=ccd, log=self.log)
1318 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1319 self.log.info(
"Applying crosstalk correction.")
1320 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=
True)
1321 self.
debugView(ccdExposure,
"doCrosstalk")
1325 if self.config.doDefect:
1326 self.log.info(
"Masking defects.")
1329 if self.config.numEdgeSuspect > 0:
1330 self.log.info(
"Masking edges as SUSPECT.")
1331 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1332 maskPlane=
"SUSPECT")
1334 if self.config.doNanMasking:
1335 self.log.info(
"Masking NAN value pixels.")
1338 if self.config.doWidenSaturationTrails:
1339 self.log.info(
"Widening saturation trails.")
1340 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1342 if self.config.doCameraSpecificMasking:
1343 self.log.info(
"Masking regions for camera specific reasons.")
1344 self.masking.
run(ccdExposure)
1346 if self.config.doBrighterFatter:
1355 interpExp = ccdExposure.clone()
1357 isrFunctions.interpolateFromMask(
1358 maskedImage=interpExp.getMaskedImage(),
1359 fwhm=self.config.fwhm,
1360 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1361 maskNameList=self.config.maskListToInterpolate
1363 bfExp = interpExp.clone()
1365 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1366 type(bfKernel), type(bfGains))
1367 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1368 self.config.brighterFatterMaxIter,
1369 self.config.brighterFatterThreshold,
1370 self.config.brighterFatterApplyGain,
1372 if bfResults[1] == self.config.brighterFatterMaxIter:
1373 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1376 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1378 image = ccdExposure.getMaskedImage().getImage()
1379 bfCorr = bfExp.getMaskedImage().getImage()
1380 bfCorr -= interpExp.getMaskedImage().getImage()
1389 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1390 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1393 if self.config.brighterFatterMaskGrowSize > 0:
1394 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1395 for maskPlane
in self.config.maskListToInterpolate:
1396 isrFunctions.growMasks(ccdExposure.getMask(),
1397 radius=self.config.brighterFatterMaskGrowSize,
1398 maskNameList=maskPlane,
1399 maskValue=maskPlane)
1401 self.
debugView(ccdExposure,
"doBrighterFatter")
1403 if self.config.doDark:
1404 self.log.info(
"Applying dark correction.")
1408 if self.config.doFringe
and not self.config.fringeAfterFlat:
1409 self.log.info(
"Applying fringe correction before flat.")
1410 self.fringe.
run(ccdExposure, **fringes.getDict())
1413 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1414 self.log.info(
"Checking strayLight correction.")
1415 self.strayLight.
run(ccdExposure, strayLightData)
1416 self.
debugView(ccdExposure,
"doStrayLight")
1418 if self.config.doFlat:
1419 self.log.info(
"Applying flat correction.")
1423 if self.config.doApplyGains:
1424 self.log.info(
"Applying gain correction instead of flat.")
1425 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1427 if self.config.doFringe
and self.config.fringeAfterFlat:
1428 self.log.info(
"Applying fringe correction after flat.")
1429 self.fringe.
run(ccdExposure, **fringes.getDict())
1431 if self.config.doVignette:
1432 self.log.info(
"Constructing Vignette polygon.")
1435 if self.config.vignette.doWriteVignettePolygon:
1438 if self.config.doAttachTransmissionCurve:
1439 self.log.info(
"Adding transmission curves.")
1440 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1441 filterTransmission=filterTransmission,
1442 sensorTransmission=sensorTransmission,
1443 atmosphereTransmission=atmosphereTransmission)
1445 flattenedThumb =
None 1446 if self.config.qa.doThumbnailFlattened:
1447 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1449 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1450 self.log.info(
"Performing illumination correction.")
1451 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1452 illumMaskedImage, illumScale=self.config.illumScale,
1453 trimToFit=self.config.doTrimToMatchCalib)
1456 if self.config.doSaveInterpPixels:
1457 preInterpExp = ccdExposure.clone()
1472 if self.config.doSetBadRegions:
1473 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1474 if badPixelCount > 0:
1475 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1477 if self.config.doInterpolate:
1478 self.log.info(
"Interpolating masked pixels.")
1479 isrFunctions.interpolateFromMask(
1480 maskedImage=ccdExposure.getMaskedImage(),
1481 fwhm=self.config.fwhm,
1482 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1483 maskNameList=list(self.config.maskListToInterpolate)
1488 if self.config.doMeasureBackground:
1489 self.log.info(
"Measuring background level.")
1492 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1494 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1495 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1496 afwMath.MEDIAN | afwMath.STDEVCLIP)
1497 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1498 qaStats.getValue(afwMath.MEDIAN))
1499 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1500 qaStats.getValue(afwMath.STDEVCLIP))
1501 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1502 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1503 qaStats.getValue(afwMath.STDEVCLIP))
1505 self.
debugView(ccdExposure,
"postISRCCD")
1507 return pipeBase.Struct(
1508 exposure=ccdExposure,
1510 flattenedThumb=flattenedThumb,
1512 preInterpolatedExposure=preInterpExp,
1513 outputExposure=ccdExposure,
1514 outputOssThumbnail=ossThumb,
1515 outputFlattenedThumbnail=flattenedThumb,
1518 @pipeBase.timeMethod
1520 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1522 This method contains the `CmdLineTask` interface to the ISR 1523 processing. All IO is handled here, freeing the `run()` method 1524 to manage only pixel-level calculations. The steps performed 1526 - Read in necessary detrending/isr/calibration data. 1527 - Process raw exposure in `run()`. 1528 - Persist the ISR-corrected exposure as "postISRCCD" if 1529 config.doWrite=True. 1533 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1534 DataRef of the detector data to be processed 1538 result : `lsst.pipe.base.Struct` 1539 Result struct with component: 1540 - ``exposure`` : `afw.image.Exposure` 1541 The fully ISR corrected exposure. 1546 Raised if a configuration option is set to True, but the 1547 required calibration data does not exist. 1550 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1552 ccdExposure = sensorRef.get(self.config.datasetType)
1554 camera = sensorRef.get(
"camera")
1555 isrData = self.
readIsrData(sensorRef, ccdExposure)
1557 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1559 if self.config.doWrite:
1560 sensorRef.put(result.exposure,
"postISRCCD")
1561 if result.preInterpolatedExposure
is not None:
1562 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1563 if result.ossThumb
is not None:
1564 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1565 if result.flattenedThumb
is not None:
1566 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1571 """!Retrieve a calibration dataset for removing instrument signature. 1576 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1577 DataRef of the detector data to find calibration datasets 1580 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1581 dateObs : `str`, optional 1582 Date of the observation. Used to correct butler failures 1583 when using fallback filters. 1585 If True, disable butler proxies to enable error handling 1586 within this routine. 1590 exposure : `lsst.afw.image.Exposure` 1591 Requested calibration frame. 1596 Raised if no matching calibration frame can be found. 1599 exp = dataRef.get(datasetType, immediate=immediate)
1600 except Exception
as exc1:
1601 if not self.config.fallbackFilterName:
1602 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1604 if self.config.useFallbackDate
and dateObs:
1605 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1606 dateObs=dateObs, immediate=immediate)
1608 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1609 except Exception
as exc2:
1610 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1611 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1612 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1614 if self.config.doAssembleIsrExposures:
1615 exp = self.assembleCcd.assembleCcd(exp)
1619 """Ensure that the data returned by Butler is a fully constructed exposure. 1621 ISR requires exposure-level image data for historical reasons, so if we did 1622 not recieve that from Butler, construct it from what we have, modifying the 1627 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1628 `lsst.afw.image.ImageF` 1629 The input data structure obtained from Butler. 1630 camera : `lsst.afw.cameraGeom.camera` 1631 The camera associated with the image. Used to find the appropriate 1634 The detector this exposure should match. 1638 inputExp : `lsst.afw.image.Exposure` 1639 The re-constructed exposure, with appropriate detector parameters. 1644 Raised if the input data cannot be used to construct an exposure. 1646 if isinstance(inputExp, afwImage.DecoratedImageU):
1647 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1648 elif isinstance(inputExp, afwImage.ImageF):
1649 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1650 elif isinstance(inputExp, afwImage.MaskedImageF):
1651 inputExp = afwImage.makeExposure(inputExp)
1652 elif isinstance(inputExp, afwImage.Exposure):
1654 elif inputExp
is None:
1658 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1661 if inputExp.getDetector()
is None:
1662 inputExp.setDetector(camera[detectorNum])
1667 """Convert exposure image from uint16 to float. 1669 If the exposure does not need to be converted, the input is 1670 immediately returned. For exposures that are converted to use 1671 floating point pixels, the variance is set to unity and the 1676 exposure : `lsst.afw.image.Exposure` 1677 The raw exposure to be converted. 1681 newexposure : `lsst.afw.image.Exposure` 1682 The input ``exposure``, converted to floating point pixels. 1687 Raised if the exposure type cannot be converted to float. 1690 if isinstance(exposure, afwImage.ExposureF):
1692 self.log.debug(
"Exposure already of type float.")
1694 if not hasattr(exposure,
"convertF"):
1695 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1697 newexposure = exposure.convertF()
1698 newexposure.variance[:] = 1
1699 newexposure.mask[:] = 0x0
1704 """Identify bad amplifiers, saturated and suspect pixels. 1708 ccdExposure : `lsst.afw.image.Exposure` 1709 Input exposure to be masked. 1710 amp : `lsst.afw.table.AmpInfoCatalog` 1711 Catalog of parameters defining the amplifier on this 1713 defects : `lsst.meas.algorithms.Defects` 1714 List of defects. Used to determine if the entire 1720 If this is true, the entire amplifier area is covered by 1721 defects and unusable. 1724 maskedImage = ccdExposure.getMaskedImage()
1730 if defects
is not None:
1731 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1736 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1738 maskView = dataView.getMask()
1739 maskView |= maskView.getPlaneBitMask(
"BAD")
1746 if self.config.doSaturation
and not badAmp:
1747 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1748 if self.config.doSuspect
and not badAmp:
1749 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1750 if math.isfinite(self.config.saturation):
1751 limits.update({self.config.saturatedMaskName: self.config.saturation})
1753 for maskName, maskThreshold
in limits.items():
1754 if not math.isnan(maskThreshold):
1755 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1756 isrFunctions.makeThresholdMask(
1757 maskedImage=dataView,
1758 threshold=maskThreshold,
1764 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1766 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1767 self.config.suspectMaskName])
1768 if numpy.all(maskView.getArray() & maskVal > 0):
1770 maskView |= maskView.getPlaneBitMask(
"BAD")
1775 """Apply overscan correction in place. 1777 This method does initial pixel rejection of the overscan 1778 region. The overscan can also be optionally segmented to 1779 allow for discontinuous overscan responses to be fit 1780 separately. The actual overscan subtraction is performed by 1781 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1782 which is called here after the amplifier is preprocessed. 1786 ccdExposure : `lsst.afw.image.Exposure` 1787 Exposure to have overscan correction performed. 1788 amp : `lsst.afw.table.AmpInfoCatalog` 1789 The amplifier to consider while correcting the overscan. 1793 overscanResults : `lsst.pipe.base.Struct` 1794 Result struct with components: 1795 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1796 Value or fit subtracted from the amplifier image data. 1797 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1798 Value or fit subtracted from the overscan image data. 1799 - ``overscanImage`` : `lsst.afw.image.Image` 1800 Image of the overscan region with the overscan 1801 correction applied. This quantity is used to estimate 1802 the amplifier read noise empirically. 1807 Raised if the ``amp`` does not contain raw pixel information. 1811 lsst.ip.isr.isrFunctions.overscanCorrection 1813 if not amp.getHasRawInfo():
1814 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1816 if amp.getRawHorizontalOverscanBBox().isEmpty():
1817 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1820 statControl = afwMath.StatisticsControl()
1821 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1824 dataBBox = amp.getRawDataBBox()
1825 oscanBBox = amp.getRawHorizontalOverscanBBox()
1829 prescanBBox = amp.getRawPrescanBBox()
1830 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1831 dx0 += self.config.overscanNumLeadingColumnsToSkip
1832 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1834 dx0 += self.config.overscanNumTrailingColumnsToSkip
1835 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1841 if ((self.config.overscanBiasJump
and 1842 self.config.overscanBiasJumpLocation)
and 1843 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1844 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1845 self.config.overscanBiasJumpDevices)):
1846 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1847 yLower = self.config.overscanBiasJumpLocation
1848 yUpper = dataBBox.getHeight() - yLower
1850 yUpper = self.config.overscanBiasJumpLocation
1851 yLower = dataBBox.getHeight() - yUpper
1870 oscanBBox.getHeight())))
1873 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1874 ampImage = ccdExposure.maskedImage[imageBBox]
1875 overscanImage = ccdExposure.maskedImage[overscanBBox]
1877 overscanArray = overscanImage.image.array
1878 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1879 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1880 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1882 statControl = afwMath.StatisticsControl()
1883 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1885 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1886 overscanImage=overscanImage,
1887 fitType=self.config.overscanFitType,
1888 order=self.config.overscanOrder,
1889 collapseRej=self.config.overscanNumSigmaClip,
1890 statControl=statControl,
1891 overscanIsInt=self.config.overscanIsInt
1895 levelStat = afwMath.MEDIAN
1896 sigmaStat = afwMath.STDEVCLIP
1898 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1899 self.config.qa.flatness.nIter)
1900 metadata = ccdExposure.getMetadata()
1901 ampNum = amp.getName()
1902 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1903 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1904 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1906 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1907 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1908 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1910 return overscanResults
1913 """Set the variance plane using the amplifier gain and read noise 1915 The read noise is calculated from the ``overscanImage`` if the 1916 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1917 the value from the amplifier data is used. 1921 ampExposure : `lsst.afw.image.Exposure` 1922 Exposure to process. 1923 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1924 Amplifier detector data. 1925 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1926 Image of overscan, required only for empirical read noise. 1930 lsst.ip.isr.isrFunctions.updateVariance 1932 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1933 gain = amp.getGain()
1935 if math.isnan(gain):
1937 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1940 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
1941 amp.getName(), gain, patchedGain)
1944 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1945 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
1947 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1948 stats = afwMath.StatisticsControl()
1949 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1950 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1951 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
1952 amp.getName(), readNoise)
1954 readNoise = amp.getReadNoise()
1956 isrFunctions.updateVariance(
1957 maskedImage=ampExposure.getMaskedImage(),
1959 readNoise=readNoise,
1963 """!Apply dark correction in place. 1967 exposure : `lsst.afw.image.Exposure` 1968 Exposure to process. 1969 darkExposure : `lsst.afw.image.Exposure` 1970 Dark exposure of the same size as ``exposure``. 1971 invert : `Bool`, optional 1972 If True, re-add the dark to an already corrected image. 1977 Raised if either ``exposure`` or ``darkExposure`` do not 1978 have their dark time defined. 1982 lsst.ip.isr.isrFunctions.darkCorrection 1984 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1985 if math.isnan(expScale):
1986 raise RuntimeError(
"Exposure darktime is NAN.")
1987 if darkExposure.getInfo().getVisitInfo()
is not None \
1988 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
1989 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1993 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1996 isrFunctions.darkCorrection(
1997 maskedImage=exposure.getMaskedImage(),
1998 darkMaskedImage=darkExposure.getMaskedImage(),
2000 darkScale=darkScale,
2002 trimToFit=self.config.doTrimToMatchCalib
2006 """!Check if linearization is needed for the detector cameraGeom. 2008 Checks config.doLinearize and the linearity type of the first 2013 detector : `lsst.afw.cameraGeom.Detector` 2014 Detector to get linearity type from. 2018 doLinearize : `Bool` 2019 If True, linearization should be performed. 2021 return self.config.doLinearize
and \
2022 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2025 """!Apply flat correction in place. 2029 exposure : `lsst.afw.image.Exposure` 2030 Exposure to process. 2031 flatExposure : `lsst.afw.image.Exposure` 2032 Flat exposure of the same size as ``exposure``. 2033 invert : `Bool`, optional 2034 If True, unflatten an already flattened image. 2038 lsst.ip.isr.isrFunctions.flatCorrection 2040 isrFunctions.flatCorrection(
2041 maskedImage=exposure.getMaskedImage(),
2042 flatMaskedImage=flatExposure.getMaskedImage(),
2043 scalingType=self.config.flatScalingType,
2044 userScale=self.config.flatUserScale,
2046 trimToFit=self.config.doTrimToMatchCalib
2050 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 2054 exposure : `lsst.afw.image.Exposure` 2055 Exposure to process. Only the amplifier DataSec is processed. 2056 amp : `lsst.afw.table.AmpInfoCatalog` 2057 Amplifier detector data. 2061 lsst.ip.isr.isrFunctions.makeThresholdMask 2063 if not math.isnan(amp.getSaturation()):
2064 maskedImage = exposure.getMaskedImage()
2065 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2066 isrFunctions.makeThresholdMask(
2067 maskedImage=dataView,
2068 threshold=amp.getSaturation(),
2070 maskName=self.config.saturatedMaskName,
2074 """!Interpolate over saturated pixels, in place. 2076 This method should be called after `saturationDetection`, to 2077 ensure that the saturated pixels have been identified in the 2078 SAT mask. It should also be called after `assembleCcd`, since 2079 saturated regions may cross amplifier boundaries. 2083 exposure : `lsst.afw.image.Exposure` 2084 Exposure to process. 2088 lsst.ip.isr.isrTask.saturationDetection 2089 lsst.ip.isr.isrFunctions.interpolateFromMask 2091 isrFunctions.interpolateFromMask(
2092 maskedImage=exposure.getMaskedImage(),
2093 fwhm=self.config.fwhm,
2094 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2095 maskNameList=list(self.config.saturatedMaskName),
2099 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 2103 exposure : `lsst.afw.image.Exposure` 2104 Exposure to process. Only the amplifier DataSec is processed. 2105 amp : `lsst.afw.table.AmpInfoCatalog` 2106 Amplifier detector data. 2110 lsst.ip.isr.isrFunctions.makeThresholdMask 2114 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 2115 This is intended to indicate pixels that may be affected by unknown systematics; 2116 for example if non-linearity corrections above a certain level are unstable 2117 then that would be a useful value for suspectLevel. A value of `nan` indicates 2118 that no such level exists and no pixels are to be masked as suspicious. 2120 suspectLevel = amp.getSuspectLevel()
2121 if math.isnan(suspectLevel):
2124 maskedImage = exposure.getMaskedImage()
2125 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2126 isrFunctions.makeThresholdMask(
2127 maskedImage=dataView,
2128 threshold=suspectLevel,
2130 maskName=self.config.suspectMaskName,
2134 """!Mask defects using mask plane "BAD", in place. 2138 exposure : `lsst.afw.image.Exposure` 2139 Exposure to process. 2140 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 2141 `lsst.afw.image.DefectBase`. 2142 List of defects to mask. 2146 Call this after CCD assembly, since defects may cross amplifier boundaries. 2148 maskedImage = exposure.getMaskedImage()
2149 if not isinstance(defectBaseList, Defects):
2151 defectList = Defects(defectBaseList)
2153 defectList = defectBaseList
2154 defectList.maskPixels(maskedImage, maskName=
"BAD")
2156 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2157 """!Mask edge pixels with applicable mask plane. 2161 exposure : `lsst.afw.image.Exposure` 2162 Exposure to process. 2163 numEdgePixels : `int`, optional 2164 Number of edge pixels to mask. 2165 maskPlane : `str`, optional 2166 Mask plane name to use. 2168 maskedImage = exposure.getMaskedImage()
2169 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2171 if numEdgePixels > 0:
2172 goodBBox = maskedImage.getBBox()
2174 goodBBox.grow(-numEdgePixels)
2176 SourceDetectionTask.setEdgeBits(
2183 """Mask and interpolate defects using mask plane "BAD", in place. 2187 exposure : `lsst.afw.image.Exposure` 2188 Exposure to process. 2189 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 2190 `lsst.afw.image.DefectBase`. 2191 List of defects to mask and interpolate. 2195 lsst.ip.isr.isrTask.maskDefect() 2198 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2199 maskPlane=
"SUSPECT")
2200 isrFunctions.interpolateFromMask(
2201 maskedImage=exposure.getMaskedImage(),
2202 fwhm=self.config.fwhm,
2203 growSaturatedFootprints=0,
2204 maskNameList=[
"BAD"],
2208 """Mask NaNs using mask plane "UNMASKEDNAN", in place. 2212 exposure : `lsst.afw.image.Exposure` 2213 Exposure to process. 2217 We mask over all NaNs, including those that are masked with 2218 other bits (because those may or may not be interpolated over 2219 later, and we want to remove all NaNs). Despite this 2220 behaviour, the "UNMASKEDNAN" mask plane is used to preserve 2221 the historical name. 2223 maskedImage = exposure.getMaskedImage()
2226 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2227 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2228 numNans =
maskNans(maskedImage, maskVal)
2229 self.metadata.set(
"NUMNANS", numNans)
2231 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2234 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place. 2238 exposure : `lsst.afw.image.Exposure` 2239 Exposure to process. 2243 lsst.ip.isr.isrTask.maskNan() 2246 isrFunctions.interpolateFromMask(
2247 maskedImage=exposure.getMaskedImage(),
2248 fwhm=self.config.fwhm,
2249 growSaturatedFootprints=0,
2250 maskNameList=[
"UNMASKEDNAN"],
2254 """Measure the image background in subgrids, for quality control purposes. 2258 exposure : `lsst.afw.image.Exposure` 2259 Exposure to process. 2260 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 2261 Configuration object containing parameters on which background 2262 statistics and subgrids to use. 2264 if IsrQaConfig
is not None:
2265 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2266 IsrQaConfig.flatness.nIter)
2267 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2268 statsControl.setAndMask(maskVal)
2269 maskedImage = exposure.getMaskedImage()
2270 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2271 skyLevel = stats.getValue(afwMath.MEDIAN)
2272 skySigma = stats.getValue(afwMath.STDEVCLIP)
2273 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2274 metadata = exposure.getMetadata()
2275 metadata.set(
'SKYLEVEL', skyLevel)
2276 metadata.set(
'SKYSIGMA', skySigma)
2279 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2280 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2281 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2282 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2283 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2284 skyLevels = numpy.zeros((nX, nY))
2287 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2289 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2291 xLLC = xc - meshXHalf
2292 yLLC = yc - meshYHalf
2293 xURC = xc + meshXHalf - 1
2294 yURC = yc + meshYHalf - 1
2297 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2299 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2301 good = numpy.where(numpy.isfinite(skyLevels))
2302 skyMedian = numpy.median(skyLevels[good])
2303 flatness = (skyLevels[good] - skyMedian) / skyMedian
2304 flatness_rms = numpy.std(flatness)
2305 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2307 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2308 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2309 nX, nY, flatness_pp, flatness_rms)
2311 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2312 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2313 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2314 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2315 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2318 """Set an approximate magnitude zero point for the exposure. 2322 exposure : `lsst.afw.image.Exposure` 2323 Exposure to process. 2325 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2326 if filterName
in self.config.fluxMag0T1:
2327 fluxMag0 = self.config.fluxMag0T1[filterName]
2329 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2330 fluxMag0 = self.config.defaultFluxMag0T1
2332 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2334 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2337 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2338 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2341 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2345 ccdExposure : `lsst.afw.image.Exposure` 2346 Exposure to process. 2347 fpPolygon : `lsst.afw.geom.Polygon` 2348 Polygon in focal plane coordinates. 2351 ccd = ccdExposure.getDetector()
2352 fpCorners = ccd.getCorners(FOCAL_PLANE)
2353 ccdPolygon = Polygon(fpCorners)
2356 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2359 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2360 validPolygon = Polygon(ccdPoints)
2361 ccdExposure.getInfo().setValidPolygon(validPolygon)
2365 """Context manager that applies and removes flats and darks, 2366 if the task is configured to apply them. 2370 exp : `lsst.afw.image.Exposure` 2371 Exposure to process. 2372 flat : `lsst.afw.image.Exposure` 2373 Flat exposure the same size as ``exp``. 2374 dark : `lsst.afw.image.Exposure`, optional 2375 Dark exposure the same size as ``exp``. 2379 exp : `lsst.afw.image.Exposure` 2380 The flat and dark corrected exposure. 2382 if self.config.doDark
and dark
is not None:
2384 if self.config.doFlat:
2389 if self.config.doFlat:
2391 if self.config.doDark
and dark
is not None:
2395 """Utility function to examine ISR exposure at different stages. 2399 exposure : `lsst.afw.image.Exposure` 2402 State of processing to view. 2404 frame = getDebugFrame(self._display, stepname)
2406 display = getDisplay(frame)
2407 display.scale(
'asinh',
'zscale')
2408 display.mtv(exposure)
2409 prompt =
"Press Enter to continue [c]... " 2411 ans = input(prompt).lower()
2412 if ans
in (
"",
"c",):
2417 """A Detector-like object that supports returning gain and saturation level 2419 This is used when the input exposure does not have a detector. 2423 exposure : `lsst.afw.image.Exposure` 2424 Exposure to generate a fake amplifier for. 2425 config : `lsst.ip.isr.isrTaskConfig` 2426 Configuration to apply to the fake amplifier. 2430 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2432 self.
_gain = config.gain
2462 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2466 """Task to wrap the default IsrTask to allow it to be retargeted. 2468 The standard IsrTask can be called directly from a command line 2469 program, but doing so removes the ability of the task to be 2470 retargeted. As most cameras override some set of the IsrTask 2471 methods, this would remove those data-specific methods in the 2472 output post-ISR images. This wrapping class fixes the issue, 2473 allowing identical post-ISR images to be generated by both the 2474 processCcd and isrTask code. 2476 ConfigClass = RunIsrConfig
2477 _DefaultName =
"runIsr" 2481 self.makeSubtask(
"isr")
2487 dataRef : `lsst.daf.persistence.ButlerDataRef` 2488 data reference of the detector data to be processed 2492 result : `pipeBase.Struct` 2493 Result struct with component: 2495 - exposure : `lsst.afw.image.Exposure` 2496 Post-ISR processed exposure.
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def getRawHorizontalOverscanBBox(self)
def maskNan(self, exposure)
def getSuspectLevel(self)
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def __init__(self, config=None)
def flatContext(self, exp, flat, dark=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def maskAndInterpolateNan(self, exposure)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
doSaturationInterpolation
def __init__(self, exposure, config)