32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
47 from .assembleCcdTask
import AssembleCcdTask
48 from .crosstalk
import CrosstalkTask
49 from .fringe
import FringeTask
50 from .isr
import maskNans
51 from .masking
import MaskingTask
52 from .straylight
import StrayLightTask
53 from .vignette
import VignetteTask
55 __all__ = [
"IsrTask",
"RunIsrTask"]
59 """Configuration parameters for IsrTask. 61 Items are grouped in the order in which they are executed by the task. 66 isrName = pexConfig.Field(
73 ccdExposure = pipeBase.InputDatasetField(
74 doc=
"Input exposure to process",
77 storageClass=
"ExposureU",
78 dimensions=[
"Instrument",
"Exposure",
"Detector"],
80 camera = pipeBase.InputDatasetField(
81 doc=
"Input camera to construct complete exposures.",
84 storageClass=
"TablePersistableCamera",
85 dimensions=[
"Instrument",
"CalibrationLabel"],
87 bias = pipeBase.InputDatasetField(
88 doc=
"Input bias calibration.",
91 storageClass=
"ImageF",
92 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
94 dark = pipeBase.InputDatasetField(
95 doc=
"Input dark calibration.",
98 storageClass=
"ImageF",
99 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
101 flat = pipeBase.InputDatasetField(
102 doc=
"Input flat calibration.",
105 storageClass=
"MaskedImageF",
106 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel",
"Detector"],
108 bfKernel = pipeBase.InputDatasetField(
109 doc=
"Input brighter-fatter kernel.",
112 storageClass=
"NumpyArray",
113 dimensions=[
"Instrument",
"CalibrationLabel"],
115 defects = pipeBase.InputDatasetField(
116 doc=
"Input defect tables.",
119 storageClass=
"DefectsList",
120 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
122 opticsTransmission = pipeBase.InputDatasetField(
123 doc=
"Transmission curve due to the optics.",
124 name=
"transmission_optics",
126 storageClass=
"TablePersistableTransmissionCurve",
127 dimensions=[
"Instrument",
"CalibrationLabel"],
129 filterTransmission = pipeBase.InputDatasetField(
130 doc=
"Transmission curve due to the filter.",
131 name=
"transmission_filter",
133 storageClass=
"TablePersistableTransmissionCurve",
134 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel"],
136 sensorTransmission = pipeBase.InputDatasetField(
137 doc=
"Transmission curve due to the sensor.",
138 name=
"transmission_sensor",
140 storageClass=
"TablePersistableTransmissionCurve",
141 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
143 atmosphereTransmission = pipeBase.InputDatasetField(
144 doc=
"Transmission curve due to the atmosphere.",
145 name=
"transmission_atmosphere",
147 storageClass=
"TablePersistableTransmissionCurve",
148 dimensions=[
"Instrument"],
152 outputExposure = pipeBase.OutputDatasetField(
153 doc=
"Output ISR processed exposure.",
156 storageClass=
"ExposureF",
157 dimensions=[
"Instrument",
"Visit",
"Detector"],
159 outputOssThumbnail = pipeBase.OutputDatasetField(
160 doc=
"Output Overscan-subtracted thumbnail image.",
163 storageClass=
"Thumbnail",
164 dimensions=[
"Instrument",
"Visit",
"Detector"],
166 outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167 doc=
"Output flat-corrected thumbnail image.",
168 name=
"FlattenedThumb",
170 storageClass=
"TextStorage",
171 dimensions=[
"Instrument",
"Visit",
"Detector"],
174 quantum = pipeBase.QuantumConfig(
175 dimensions=[
"Visit",
"Detector",
"Instrument"],
179 datasetType = pexConfig.Field(
181 doc=
"Dataset type for input data; users will typically leave this alone, " 182 "but camera-specific ISR tasks will override it",
186 fallbackFilterName = pexConfig.Field(
188 doc=
"Fallback default filter name for calibrations.",
191 expectWcs = pexConfig.Field(
194 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 196 fwhm = pexConfig.Field(
198 doc=
"FWHM of PSF in arcseconds.",
201 qa = pexConfig.ConfigField(
203 doc=
"QA related configuration options.",
207 doConvertIntToFloat = pexConfig.Field(
209 doc=
"Convert integer raw images to floating point values?",
214 doSaturation = pexConfig.Field(
216 doc=
"Mask saturated pixels? NB: this is totally independent of the" 217 " interpolation option - this is ONLY setting the bits in the mask." 218 " To have them interpolated make sure doSaturationInterpolation=True",
221 saturatedMaskName = pexConfig.Field(
223 doc=
"Name of mask plane to use in saturation detection and interpolation",
226 saturation = pexConfig.Field(
228 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229 default=float(
"NaN"),
231 growSaturationFootprintSize = pexConfig.Field(
233 doc=
"Number of pixels by which to grow the saturation footprints",
238 doSuspect = pexConfig.Field(
240 doc=
"Mask suspect pixels?",
243 suspectMaskName = pexConfig.Field(
245 doc=
"Name of mask plane to use for suspect pixels",
248 numEdgeSuspect = pexConfig.Field(
250 doc=
"Number of edge pixels to be flagged as untrustworthy.",
255 doSetBadRegions = pexConfig.Field(
257 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
260 badStatistic = pexConfig.ChoiceField(
262 doc=
"How to estimate the average value for BAD regions.",
265 "MEANCLIP":
"Correct using the (clipped) mean of good data",
266 "MEDIAN":
"Correct using the median of the good data",
271 doOverscan = pexConfig.Field(
273 doc=
"Do overscan subtraction?",
276 overscanFitType = pexConfig.ChoiceField(
278 doc=
"The method for fitting the overscan bias level.",
281 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
282 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
283 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
284 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
285 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
286 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
287 "MEAN":
"Correct using the mean of the overscan region",
288 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
289 "MEDIAN":
"Correct using the median of the overscan region",
292 overscanOrder = pexConfig.Field(
294 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
295 "or number of spline knots if overscan fit type is a spline."),
298 overscanNumSigmaClip = pexConfig.Field(
300 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
303 overscanIsInt = pexConfig.Field(
305 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
308 overscanNumLeadingColumnsToSkip = pexConfig.Field(
310 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
313 overscanNumTrailingColumnsToSkip = pexConfig.Field(
315 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
318 overscanMaxDev = pexConfig.Field(
320 doc=
"Maximum deviation from the median for overscan",
321 default=1000.0, check=
lambda x: x > 0
323 overscanBiasJump = pexConfig.Field(
325 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
328 overscanBiasJumpKeyword = pexConfig.Field(
330 doc=
"Header keyword containing information about devices.",
331 default=
"NO_SUCH_KEY",
333 overscanBiasJumpDevices = pexConfig.ListField(
335 doc=
"List of devices that need piecewise overscan correction.",
338 overscanBiasJumpLocation = pexConfig.Field(
340 doc=
"Location of bias jump along y-axis.",
345 doAssembleCcd = pexConfig.Field(
348 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 350 assembleCcd = pexConfig.ConfigurableField(
351 target=AssembleCcdTask,
352 doc=
"CCD assembly task",
356 doAssembleIsrExposures = pexConfig.Field(
359 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 361 doTrimToMatchCalib = pexConfig.Field(
364 doc=
"Trim raw data to match calibration bounding boxes?" 368 doBias = pexConfig.Field(
370 doc=
"Apply bias frame correction?",
373 biasDataProductName = pexConfig.Field(
375 doc=
"Name of the bias data product",
380 doVariance = pexConfig.Field(
382 doc=
"Calculate variance?",
385 gain = pexConfig.Field(
387 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388 default=float(
"NaN"),
390 readNoise = pexConfig.Field(
392 doc=
"The read noise to use if no Detector is present in the Exposure",
395 doEmpiricalReadNoise = pexConfig.Field(
398 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 402 doLinearize = pexConfig.Field(
404 doc=
"Correct for nonlinearity of the detector's response?",
409 doCrosstalk = pexConfig.Field(
411 doc=
"Apply intra-CCD crosstalk correction?",
414 doCrosstalkBeforeAssemble = pexConfig.Field(
416 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
419 crosstalk = pexConfig.ConfigurableField(
420 target=CrosstalkTask,
421 doc=
"Intra-CCD crosstalk correction",
425 doWidenSaturationTrails = pexConfig.Field(
427 doc=
"Widen bleed trails based on their width?",
432 doBrighterFatter = pexConfig.Field(
435 doc=
"Apply the brighter fatter correction" 437 brighterFatterLevel = pexConfig.ChoiceField(
440 doc=
"The level at which to correct for brighter-fatter.",
442 "AMP":
"Every amplifier treated separately.",
443 "DETECTOR":
"One kernel per detector",
446 brighterFatterKernelFile = pexConfig.Field(
449 doc=
"Kernel file used for the brighter fatter correction" 451 brighterFatterMaxIter = pexConfig.Field(
454 doc=
"Maximum number of iterations for the brighter fatter correction" 456 brighterFatterThreshold = pexConfig.Field(
459 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 460 " absolute value of the difference between the current corrected image and the one" 461 " from the previous iteration summed over all the pixels." 463 brighterFatterApplyGain = pexConfig.Field(
466 doc=
"Should the gain be applied when applying the brighter fatter correction?" 470 doDefect = pexConfig.Field(
472 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
475 doSaturationInterpolation = pexConfig.Field(
477 doc=
"Perform interpolation over pixels masked as saturated?" 478 " NB: This is independent of doSaturation; if that is False this plane" 479 " will likely be blank, resulting in a no-op here.",
482 numEdgeSuspect = pexConfig.Field(
484 doc=
"Number of edge pixels to be flagged as untrustworthy.",
489 doDark = pexConfig.Field(
491 doc=
"Apply dark frame correction?",
494 darkDataProductName = pexConfig.Field(
496 doc=
"Name of the dark data product",
501 doStrayLight = pexConfig.Field(
503 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
506 strayLight = pexConfig.ConfigurableField(
507 target=StrayLightTask,
508 doc=
"y-band stray light correction" 512 doFlat = pexConfig.Field(
514 doc=
"Apply flat field correction?",
517 flatDataProductName = pexConfig.Field(
519 doc=
"Name of the flat data product",
522 flatScalingType = pexConfig.ChoiceField(
524 doc=
"The method for scaling the flat on the fly.",
527 "USER":
"Scale by flatUserScale",
528 "MEAN":
"Scale by the inverse of the mean",
529 "MEDIAN":
"Scale by the inverse of the median",
532 flatUserScale = pexConfig.Field(
534 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
537 doTweakFlat = pexConfig.Field(
539 doc=
"Tweak flats to match observed amplifier ratios?",
544 doApplyGains = pexConfig.Field(
546 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
549 normalizeGains = pexConfig.Field(
551 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
556 doFringe = pexConfig.Field(
558 doc=
"Apply fringe correction?",
561 fringe = pexConfig.ConfigurableField(
563 doc=
"Fringe subtraction task",
565 fringeAfterFlat = pexConfig.Field(
567 doc=
"Do fringe subtraction after flat-fielding?",
572 doNanInterpAfterFlat = pexConfig.Field(
574 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 575 "also have to interpolate them before flat-fielding."),
580 doAddDistortionModel = pexConfig.Field(
582 doc=
"Apply a distortion model based on camera geometry to the WCS?",
587 doMeasureBackground = pexConfig.Field(
589 doc=
"Measure the background level on the reduced image?",
594 doCameraSpecificMasking = pexConfig.Field(
596 doc=
"Mask camera-specific bad regions?",
599 masking = pexConfig.ConfigurableField(
605 fluxMag0T1 = pexConfig.DictField(
608 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
612 defaultFluxMag0T1 = pexConfig.Field(
614 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
615 default=pow(10.0, 0.4*28.0)
619 doVignette = pexConfig.Field(
621 doc=
"Apply vignetting parameters?",
624 vignette = pexConfig.ConfigurableField(
626 doc=
"Vignetting task.",
630 doAttachTransmissionCurve = pexConfig.Field(
633 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 635 doUseOpticsTransmission = pexConfig.Field(
638 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 640 doUseFilterTransmission = pexConfig.Field(
643 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 645 doUseSensorTransmission = pexConfig.Field(
648 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 650 doUseAtmosphereTransmission = pexConfig.Field(
653 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 657 doWrite = pexConfig.Field(
659 doc=
"Persist postISRCCD?",
666 raise ValueError(
"You may not specify both doFlat and doApplyGains")
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670 r"""Apply common instrument signature correction algorithms to a raw frame. 672 The process for correcting imaging data is very similar from 673 camera to camera. This task provides a vanilla implementation of 674 doing these corrections, including the ability to turn certain 675 corrections off if they are not needed. The inputs to the primary 676 method, `run()`, are a raw exposure to be corrected and the 677 calibration data products. The raw input is a single chip sized 678 mosaic of all amps including overscans and other non-science 679 pixels. The method `runDataRef()` identifies and defines the 680 calibration data products, and is intended for use by a 681 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 682 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 683 subclassed for different camera, although the most camera specific 684 methods have been split into subtasks that can be redirected 687 The __init__ method sets up the subtasks for ISR processing, using 688 the defaults from `lsst.ip.isr`. 693 Positional arguments passed to the Task constructor. None used at this time. 694 kwargs : `dict`, optional 695 Keyword arguments passed on to the Task constructor. None used at this time. 697 ConfigClass = IsrTaskConfig
702 self.makeSubtask(
"assembleCcd")
703 self.makeSubtask(
"crosstalk")
704 self.makeSubtask(
"strayLight")
705 self.makeSubtask(
"fringe")
706 self.makeSubtask(
"masking")
707 self.makeSubtask(
"vignette")
716 if config.doBias
is not True:
717 inputTypeDict.pop(
"bias",
None)
718 if config.doLinearize
is not True:
719 inputTypeDict.pop(
"linearizer",
None)
720 if config.doCrosstalk
is not True:
721 inputTypeDict.pop(
"crosstalkSources",
None)
722 if config.doBrighterFatter
is not True:
723 inputTypeDict.pop(
"bfKernel",
None)
724 if config.doDefect
is not True:
725 inputTypeDict.pop(
"defects",
None)
726 if config.doDark
is not True:
727 inputTypeDict.pop(
"dark",
None)
728 if config.doFlat
is not True:
729 inputTypeDict.pop(
"flat",
None)
730 if config.doAttachTransmissionCurve
is not True:
731 inputTypeDict.pop(
"opticsTransmission",
None)
732 inputTypeDict.pop(
"filterTransmission",
None)
733 inputTypeDict.pop(
"sensorTransmission",
None)
734 inputTypeDict.pop(
"atmosphereTransmission",
None)
735 if config.doUseOpticsTransmission
is not True:
736 inputTypeDict.pop(
"opticsTransmission",
None)
737 if config.doUseFilterTransmission
is not True:
738 inputTypeDict.pop(
"filterTransmission",
None)
739 if config.doUseSensorTransmission
is not True:
740 inputTypeDict.pop(
"sensorTransmission",
None)
741 if config.doUseAtmosphereTransmission
is not True:
742 inputTypeDict.pop(
"atmosphereTransmission",
None)
750 if config.qa.doThumbnailOss
is not True:
751 outputTypeDict.pop(
"outputOssThumbnail",
None)
752 if config.qa.doThumbnailFlattened
is not True:
753 outputTypeDict.pop(
"outputFlattenedThumbnail",
None)
754 if config.doWrite
is not True:
755 outputTypeDict.pop(
"outputExposure",
None)
757 return outputTypeDict
766 names.remove(
"ccdExposure")
775 return frozenset([
"CalibrationLabel"])
779 inputData[
'detectorNum'] = int(inputDataIds[
'ccdExposure'][
'detector'])
780 except Exception
as e:
781 raise ValueError(f
"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
783 inputData[
'isGen3'] =
True 785 if self.config.doLinearize
is True:
786 if 'linearizer' not in inputData.keys():
787 detector = inputData[
'camera'][inputData[
'detectorNum']]
788 linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
789 inputData[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
791 if inputData[
'defects']
is not None:
794 if not isinstance(inputData[
"defects"], Defects):
795 inputData[
"defects"] = Defects.fromTable(inputData[
"defects"])
812 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
818 """!Retrieve necessary frames for instrument signature removal. 820 Pre-fetching all required ISR data products limits the IO 821 required by the ISR. Any conflict between the calibration data 822 available and that needed for ISR is also detected prior to 823 doing processing, allowing it to fail quickly. 827 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 828 Butler reference of the detector data to be processed 829 rawExposure : `afw.image.Exposure` 830 The raw exposure that will later be corrected with the 831 retrieved calibration data; should not be modified in this 836 result : `lsst.pipe.base.Struct` 837 Result struct with components (which may be `None`): 838 - ``bias``: bias calibration frame (`afw.image.Exposure`) 839 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 840 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 841 - ``dark``: dark calibration frame (`afw.image.Exposure`) 842 - ``flat``: flat calibration frame (`afw.image.Exposure`) 843 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 844 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 845 - ``fringes``: `lsst.pipe.base.Struct` with components: 846 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 847 - ``seed``: random seed derived from the ccdExposureId for random 848 number generator (`uint32`) 849 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 850 A ``TransmissionCurve`` that represents the throughput of the optics, 851 to be evaluated in focal-plane coordinates. 852 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 853 A ``TransmissionCurve`` that represents the throughput of the filter 854 itself, to be evaluated in focal-plane coordinates. 855 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 856 A ``TransmissionCurve`` that represents the throughput of the sensor 857 itself, to be evaluated in post-assembly trimmed detector coordinates. 858 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 859 A ``TransmissionCurve`` that represents the throughput of the 860 atmosphere, assumed to be spatially constant. 861 - ``strayLightData`` : `object` 862 An opaque object containing calibration information for 863 stray-light correction. If `None`, no correction will be 868 NotImplementedError : 869 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 871 ccd = rawExposure.getDetector()
872 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
873 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
874 if self.config.doBias
else None)
876 linearizer = (dataRef.get(
"linearizer", immediate=
True)
878 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
879 if self.config.doCrosstalk
else None)
880 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
881 if self.config.doDark
else None)
882 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
883 if self.config.doFlat
else None)
885 brighterFatterKernel =
None 886 if self.config.doBrighterFatter
is True:
890 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
894 brighterFatterKernel = dataRef.get(
"bfKernel")
896 brighterFatterKernel =
None 897 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
900 if self.config.brighterFatterLevel ==
'DETECTOR':
901 brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
904 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
906 defectList = (dataRef.get(
"defects")
907 if self.config.doDefect
else None)
908 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
909 if self.config.doAssembleIsrExposures
else None)
910 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
911 else pipeBase.Struct(fringes=
None))
913 if self.config.doAttachTransmissionCurve:
914 opticsTransmission = (dataRef.get(
"transmission_optics")
915 if self.config.doUseOpticsTransmission
else None)
916 filterTransmission = (dataRef.get(
"transmission_filter")
917 if self.config.doUseFilterTransmission
else None)
918 sensorTransmission = (dataRef.get(
"transmission_sensor")
919 if self.config.doUseSensorTransmission
else None)
920 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
921 if self.config.doUseAtmosphereTransmission
else None)
923 opticsTransmission =
None 924 filterTransmission =
None 925 sensorTransmission =
None 926 atmosphereTransmission =
None 928 if self.config.doStrayLight:
929 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
931 strayLightData =
None 934 return pipeBase.Struct(bias=biasExposure,
935 linearizer=linearizer,
936 crosstalkSources=crosstalkSources,
939 bfKernel=brighterFatterKernel,
941 fringes=fringeStruct,
942 opticsTransmission=opticsTransmission,
943 filterTransmission=filterTransmission,
944 sensorTransmission=sensorTransmission,
945 atmosphereTransmission=atmosphereTransmission,
946 strayLightData=strayLightData
950 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
951 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
952 opticsTransmission=None, filterTransmission=None,
953 sensorTransmission=None, atmosphereTransmission=None,
954 detectorNum=None, strayLightData=None, isGen3=False,
956 """!Perform instrument signature removal on an exposure. 958 Steps included in the ISR processing, in order performed, are: 959 - saturation and suspect pixel masking 960 - overscan subtraction 961 - CCD assembly of individual amplifiers 963 - variance image construction 964 - linearization of non-linear response 966 - brighter-fatter correction 969 - stray light subtraction 971 - masking of known defects and camera specific features 972 - vignette calculation 973 - appending transmission curve and distortion model 977 ccdExposure : `lsst.afw.image.Exposure` 978 The raw exposure that is to be run through ISR. The 979 exposure is modified by this method. 980 camera : `lsst.afw.cameraGeom.Camera`, optional 981 The camera geometry for this exposure. Used to select the 982 distortion model appropriate for this data. 983 bias : `lsst.afw.image.Exposure`, optional 984 Bias calibration frame. 985 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 986 Functor for linearization. 987 crosstalkSources : `list`, optional 988 List of possible crosstalk sources. 989 dark : `lsst.afw.image.Exposure`, optional 990 Dark calibration frame. 991 flat : `lsst.afw.image.Exposure`, optional 992 Flat calibration frame. 993 bfKernel : `numpy.ndarray`, optional 994 Brighter-fatter kernel. 995 defects : `lsst.meas.algorithms.Defects`, optional 997 fringes : `lsst.pipe.base.Struct`, optional 998 Struct containing the fringe correction data, with 1000 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 1001 - ``seed``: random seed derived from the ccdExposureId for random 1002 number generator (`uint32`) 1003 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 1004 A ``TransmissionCurve`` that represents the throughput of the optics, 1005 to be evaluated in focal-plane coordinates. 1006 filterTransmission : `lsst.afw.image.TransmissionCurve` 1007 A ``TransmissionCurve`` that represents the throughput of the filter 1008 itself, to be evaluated in focal-plane coordinates. 1009 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1010 A ``TransmissionCurve`` that represents the throughput of the sensor 1011 itself, to be evaluated in post-assembly trimmed detector coordinates. 1012 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1013 A ``TransmissionCurve`` that represents the throughput of the 1014 atmosphere, assumed to be spatially constant. 1015 detectorNum : `int`, optional 1016 The integer number for the detector to process. 1017 isGen3 : bool, optional 1018 Flag this call to run() as using the Gen3 butler environment. 1019 strayLightData : `object`, optional 1020 Opaque object containing calibration information for stray-light 1021 correction. If `None`, no correction will be performed. 1025 result : `lsst.pipe.base.Struct` 1026 Result struct with component: 1027 - ``exposure`` : `afw.image.Exposure` 1028 The fully ISR corrected exposure. 1029 - ``outputExposure`` : `afw.image.Exposure` 1030 An alias for `exposure` 1031 - ``ossThumb`` : `numpy.ndarray` 1032 Thumbnail image of the exposure after overscan subtraction. 1033 - ``flattenedThumb`` : `numpy.ndarray` 1034 Thumbnail image of the exposure after flat-field correction. 1039 Raised if a configuration option is set to True, but the 1040 required calibration data has not been specified. 1044 The current processed exposure can be viewed by setting the 1045 appropriate lsstDebug entries in the `debug.display` 1046 dictionary. The names of these entries correspond to some of 1047 the IsrTaskConfig Boolean options, with the value denoting the 1048 frame to use. The exposure is shown inside the matching 1049 option check and after the processing of that step has 1050 finished. The steps with debug points are: 1061 In addition, setting the "postISRCCD" entry displays the 1062 exposure after all ISR processing has finished. 1070 self.config.doFringe =
False 1073 if detectorNum
is None:
1074 raise RuntimeError(
"Must supply the detectorNum if running as Gen3")
1076 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1081 if isinstance(ccdExposure, ButlerDataRef):
1084 ccd = ccdExposure.getDetector()
1087 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 1088 ccd = [
FakeAmp(ccdExposure, self.config)]
1091 if self.config.doBias
and bias
is None:
1092 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1094 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1095 if self.config.doBrighterFatter
and bfKernel
is None:
1096 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1097 if self.config.doDark
and dark
is None:
1098 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1100 fringes = pipeBase.Struct(fringes=
None)
1101 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
1102 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1103 if self.config.doFlat
and flat
is None:
1104 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1105 if self.config.doDefect
and defects
is None:
1106 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1107 if self.config.doAddDistortionModel
and camera
is None:
1108 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
1111 if self.config.doConvertIntToFloat:
1112 self.log.info(
"Converting exposure to floating point values")
1119 if ccdExposure.getBBox().contains(amp.getBBox()):
1123 if self.config.doOverscan
and not badAmp:
1126 self.log.debug(
"Corrected overscan for amplifier %s" % (amp.getName()))
1127 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1128 if isinstance(overscanResults.overscanFit, float):
1129 qaMedian = overscanResults.overscanFit
1130 qaStdev = float(
"NaN")
1132 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1133 afwMath.MEDIAN | afwMath.STDEVCLIP)
1134 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1135 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1137 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1138 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1139 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f" %
1140 (amp.getName(), qaMedian, qaStdev))
1141 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1143 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
1144 overscanResults =
None 1146 overscans.append(overscanResults
if overscanResults
is not None else None)
1148 self.log.info(
"Skipped OSCAN")
1150 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1151 self.log.info(
"Applying crosstalk correction.")
1152 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1153 self.
debugView(ccdExposure,
"doCrosstalk")
1155 if self.config.doAssembleCcd:
1156 self.log.info(
"Assembling CCD from amplifiers")
1157 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1159 if self.config.expectWcs
and not ccdExposure.getWcs():
1160 self.log.warn(
"No WCS found in input exposure")
1161 self.
debugView(ccdExposure,
"doAssembleCcd")
1164 if self.config.qa.doThumbnailOss:
1165 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1167 if self.config.doBias:
1168 self.log.info(
"Applying bias correction.")
1169 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1170 trimToFit=self.config.doTrimToMatchCalib)
1173 if self.config.doVariance:
1174 for amp, overscanResults
in zip(ccd, overscans):
1175 if ccdExposure.getBBox().contains(amp.getBBox()):
1176 self.log.debug(
"Constructing variance map for amplifer %s" % (amp.getName()))
1177 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1178 if overscanResults
is not None:
1180 overscanImage=overscanResults.overscanImage)
1184 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1185 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1186 afwMath.MEDIAN | afwMath.STDEVCLIP)
1187 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
1188 qaStats.getValue(afwMath.MEDIAN))
1189 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
1190 qaStats.getValue(afwMath.STDEVCLIP))
1191 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f" %
1192 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1193 qaStats.getValue(afwMath.STDEVCLIP)))
1196 self.log.info(
"Applying linearizer.")
1197 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1199 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1200 self.log.info(
"Applying crosstalk correction.")
1201 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=
True)
1202 self.
debugView(ccdExposure,
"doCrosstalk")
1204 if self.config.doWidenSaturationTrails:
1205 self.log.info(
"Widening saturation trails.")
1206 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1208 interpolationDone =
False 1209 if self.config.doBrighterFatter:
1215 if self.config.doDefect:
1218 if self.config.doSaturationInterpolation:
1222 interpolationDone =
True 1224 self.log.info(
"Applying brighter fatter correction.")
1225 isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1226 self.config.brighterFatterMaxIter,
1227 self.config.brighterFatterThreshold,
1228 self.config.brighterFatterApplyGain,
1230 self.
debugView(ccdExposure,
"doBrighterFatter")
1232 if self.config.doDark:
1233 self.log.info(
"Applying dark correction.")
1237 if self.config.doFringe
and not self.config.fringeAfterFlat:
1238 self.log.info(
"Applying fringe correction before flat.")
1239 self.fringe.
run(ccdExposure, **fringes.getDict())
1242 if self.config.doStrayLight:
1243 if strayLightData
is not None:
1244 self.log.info(
"Applying stray light correction.")
1245 self.strayLight.
run(ccdExposure, strayLightData)
1246 self.
debugView(ccdExposure,
"doStrayLight")
1248 self.log.debug(
"Skipping stray light correction: no data found for this image.")
1250 if self.config.doFlat:
1251 self.log.info(
"Applying flat correction.")
1255 if self.config.doApplyGains:
1256 self.log.info(
"Applying gain correction instead of flat.")
1257 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1259 if self.config.doDefect
and not interpolationDone:
1260 self.log.info(
"Masking and interpolating defects.")
1263 if self.config.doSaturationInterpolation
and not interpolationDone:
1264 self.log.info(
"Interpolating saturated pixels.")
1267 if self.config.doNanInterpAfterFlat
or not interpolationDone:
1268 self.log.info(
"Masking and interpolating NAN value pixels.")
1271 if self.config.doFringe
and self.config.fringeAfterFlat:
1272 self.log.info(
"Applying fringe correction after flat.")
1273 self.fringe.
run(ccdExposure, **fringes.getDict())
1275 if self.config.doSetBadRegions:
1276 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1277 if badPixelCount > 0:
1278 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1280 flattenedThumb =
None 1281 if self.config.qa.doThumbnailFlattened:
1282 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1284 if self.config.doCameraSpecificMasking:
1285 self.log.info(
"Masking regions for camera specific reasons.")
1286 self.masking.
run(ccdExposure)
1290 if self.config.doVignette:
1291 self.log.info(
"Constructing Vignette polygon.")
1294 if self.config.vignette.doWriteVignettePolygon:
1297 if self.config.doAttachTransmissionCurve:
1298 self.log.info(
"Adding transmission curves.")
1299 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1300 filterTransmission=filterTransmission,
1301 sensorTransmission=sensorTransmission,
1302 atmosphereTransmission=atmosphereTransmission)
1304 if self.config.doAddDistortionModel:
1305 self.log.info(
"Adding a distortion model to the WCS.")
1306 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1308 if self.config.doMeasureBackground:
1309 self.log.info(
"Measuring background level:")
1312 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1314 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1315 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1316 afwMath.MEDIAN | afwMath.STDEVCLIP)
1317 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1318 qaStats.getValue(afwMath.MEDIAN))
1319 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1320 qaStats.getValue(afwMath.STDEVCLIP))
1321 self.log.debug(
" Background stats for amplifer %s: %f +/- %f" %
1322 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1323 qaStats.getValue(afwMath.STDEVCLIP)))
1325 self.
debugView(ccdExposure,
"postISRCCD")
1327 return pipeBase.Struct(
1328 exposure=ccdExposure,
1330 flattenedThumb=flattenedThumb,
1332 outputExposure=ccdExposure,
1333 outputOssThumbnail=ossThumb,
1334 outputFlattenedThumbnail=flattenedThumb,
1337 @pipeBase.timeMethod
1339 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1341 This method contains the `CmdLineTask` interface to the ISR 1342 processing. All IO is handled here, freeing the `run()` method 1343 to manage only pixel-level calculations. The steps performed 1345 - Read in necessary detrending/isr/calibration data. 1346 - Process raw exposure in `run()`. 1347 - Persist the ISR-corrected exposure as "postISRCCD" if 1348 config.doWrite=True. 1352 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1353 DataRef of the detector data to be processed 1357 result : `lsst.pipe.base.Struct` 1358 Result struct with component: 1359 - ``exposure`` : `afw.image.Exposure` 1360 The fully ISR corrected exposure. 1365 Raised if a configuration option is set to True, but the 1366 required calibration data does not exist. 1369 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1371 ccdExposure = sensorRef.get(self.config.datasetType)
1373 camera = sensorRef.get(
"camera")
1374 if camera
is None and self.config.doAddDistortionModel:
1375 raise RuntimeError(
"config.doAddDistortionModel is True " 1376 "but could not get a camera from the butler")
1377 isrData = self.
readIsrData(sensorRef, ccdExposure)
1379 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1381 if self.config.doWrite:
1382 sensorRef.put(result.exposure,
"postISRCCD")
1383 if result.ossThumb
is not None:
1384 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1385 if result.flattenedThumb
is not None:
1386 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1391 """!Retrieve a calibration dataset for removing instrument signature. 1396 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1397 DataRef of the detector data to find calibration datasets 1400 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1402 If True, disable butler proxies to enable error handling 1403 within this routine. 1407 exposure : `lsst.afw.image.Exposure` 1408 Requested calibration frame. 1413 Raised if no matching calibration frame can be found. 1416 exp = dataRef.get(datasetType, immediate=immediate)
1417 except Exception
as exc1:
1418 if not self.config.fallbackFilterName:
1419 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1421 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1422 except Exception
as exc2:
1423 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1424 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1425 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1427 if self.config.doAssembleIsrExposures:
1428 exp = self.assembleCcd.assembleCcd(exp)
1432 """Ensure that the data returned by Butler is a fully constructed exposure. 1434 ISR requires exposure-level image data for historical reasons, so if we did 1435 not recieve that from Butler, construct it from what we have, modifying the 1440 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1441 `lsst.afw.image.ImageF` 1442 The input data structure obtained from Butler. 1443 camera : `lsst.afw.cameraGeom.camera` 1444 The camera associated with the image. Used to find the appropriate 1447 The detector this exposure should match. 1451 inputExp : `lsst.afw.image.Exposure` 1452 The re-constructed exposure, with appropriate detector parameters. 1457 Raised if the input data cannot be used to construct an exposure. 1459 if isinstance(inputExp, afwImage.DecoratedImageU):
1460 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1461 elif isinstance(inputExp, afwImage.ImageF):
1462 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1463 elif isinstance(inputExp, afwImage.MaskedImageF):
1464 inputExp = afwImage.makeExposure(inputExp)
1465 elif isinstance(inputExp, afwImage.Exposure):
1468 raise TypeError(f
"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1470 if inputExp.getDetector()
is None:
1471 inputExp.setDetector(camera[detectorNum])
1476 """Convert exposure image from uint16 to float. 1478 If the exposure does not need to be converted, the input is 1479 immediately returned. For exposures that are converted to use 1480 floating point pixels, the variance is set to unity and the 1485 exposure : `lsst.afw.image.Exposure` 1486 The raw exposure to be converted. 1490 newexposure : `lsst.afw.image.Exposure` 1491 The input ``exposure``, converted to floating point pixels. 1496 Raised if the exposure type cannot be converted to float. 1499 if isinstance(exposure, afwImage.ExposureF):
1502 if not hasattr(exposure,
"convertF"):
1503 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1505 newexposure = exposure.convertF()
1506 newexposure.variance[:] = 1
1507 newexposure.mask[:] = 0x0
1512 """Identify bad amplifiers, saturated and suspect pixels. 1516 ccdExposure : `lsst.afw.image.Exposure` 1517 Input exposure to be masked. 1518 amp : `lsst.afw.table.AmpInfoCatalog` 1519 Catalog of parameters defining the amplifier on this 1521 defects : `lsst.meas.algorithms.Defects` 1522 List of defects. Used to determine if the entire 1528 If this is true, the entire amplifier area is covered by 1529 defects and unusable. 1532 maskedImage = ccdExposure.getMaskedImage()
1538 if defects
is not None:
1539 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1544 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1546 maskView = dataView.getMask()
1547 maskView |= maskView.getPlaneBitMask(
"BAD")
1554 if self.config.doSaturation
and not badAmp:
1555 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1556 if self.config.doSuspect
and not badAmp:
1557 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1559 for maskName, maskThreshold
in limits.items():
1560 if not math.isnan(maskThreshold):
1561 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1562 isrFunctions.makeThresholdMask(
1563 maskedImage=dataView,
1564 threshold=maskThreshold,
1570 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1572 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1573 self.config.suspectMaskName])
1574 if numpy.all(maskView.getArray() & maskVal > 0):
1580 """Apply overscan correction in place. 1582 This method does initial pixel rejection of the overscan 1583 region. The overscan can also be optionally segmented to 1584 allow for discontinuous overscan responses to be fit 1585 separately. The actual overscan subtraction is performed by 1586 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1587 which is called here after the amplifier is preprocessed. 1591 ccdExposure : `lsst.afw.image.Exposure` 1592 Exposure to have overscan correction performed. 1593 amp : `lsst.afw.table.AmpInfoCatalog` 1594 The amplifier to consider while correcting the overscan. 1598 overscanResults : `lsst.pipe.base.Struct` 1599 Result struct with components: 1600 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1601 Value or fit subtracted from the amplifier image data. 1602 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1603 Value or fit subtracted from the overscan image data. 1604 - ``overscanImage`` : `lsst.afw.image.Image` 1605 Image of the overscan region with the overscan 1606 correction applied. This quantity is used to estimate 1607 the amplifier read noise empirically. 1612 Raised if the ``amp`` does not contain raw pixel information. 1616 lsst.ip.isr.isrFunctions.overscanCorrection 1618 if not amp.getHasRawInfo():
1619 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1621 if amp.getRawHorizontalOverscanBBox().isEmpty():
1622 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1625 statControl = afwMath.StatisticsControl()
1626 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1629 dataBBox = amp.getRawDataBBox()
1630 oscanBBox = amp.getRawHorizontalOverscanBBox()
1634 prescanBBox = amp.getRawPrescanBBox()
1635 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1636 dx0 += self.config.overscanNumLeadingColumnsToSkip
1637 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1639 dx0 += self.config.overscanNumTrailingColumnsToSkip
1640 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1646 if ((self.config.overscanBiasJump
and 1647 self.config.overscanBiasJumpLocation)
and 1648 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1649 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1650 self.config.overscanBiasJumpDevices)):
1651 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1652 yLower = self.config.overscanBiasJumpLocation
1653 yUpper = dataBBox.getHeight() - yLower
1655 yUpper = self.config.overscanBiasJumpLocation
1656 yLower = dataBBox.getHeight() - yUpper
1675 oscanBBox.getHeight())))
1678 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1679 ampImage = ccdExposure.maskedImage[imageBBox]
1680 overscanImage = ccdExposure.maskedImage[overscanBBox]
1682 overscanArray = overscanImage.image.array
1683 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1684 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1685 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1687 statControl = afwMath.StatisticsControl()
1688 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1690 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1691 overscanImage=overscanImage,
1692 fitType=self.config.overscanFitType,
1693 order=self.config.overscanOrder,
1694 collapseRej=self.config.overscanNumSigmaClip,
1695 statControl=statControl,
1696 overscanIsInt=self.config.overscanIsInt
1700 levelStat = afwMath.MEDIAN
1701 sigmaStat = afwMath.STDEVCLIP
1703 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1704 self.config.qa.flatness.nIter)
1705 metadata = ccdExposure.getMetadata()
1706 ampNum = amp.getName()
1707 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1708 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1709 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1711 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1712 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1713 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1715 return overscanResults
1718 """Set the variance plane using the amplifier gain and read noise 1720 The read noise is calculated from the ``overscanImage`` if the 1721 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1722 the value from the amplifier data is used. 1726 ampExposure : `lsst.afw.image.Exposure` 1727 Exposure to process. 1728 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1729 Amplifier detector data. 1730 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1731 Image of overscan, required only for empirical read noise. 1735 lsst.ip.isr.isrFunctions.updateVariance 1737 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1738 gain = amp.getGain()
1740 if math.isnan(gain):
1742 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1745 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1746 (amp.getName(), gain, patchedGain))
1749 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1750 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1752 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1753 stats = afwMath.StatisticsControl()
1754 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1755 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1756 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1758 readNoise = amp.getReadNoise()
1760 isrFunctions.updateVariance(
1761 maskedImage=ampExposure.getMaskedImage(),
1763 readNoise=readNoise,
1767 """!Apply dark correction in place. 1771 exposure : `lsst.afw.image.Exposure` 1772 Exposure to process. 1773 darkExposure : `lsst.afw.image.Exposure` 1774 Dark exposure of the same size as ``exposure``. 1775 invert : `Bool`, optional 1776 If True, re-add the dark to an already corrected image. 1781 Raised if either ``exposure`` or ``darkExposure`` do not 1782 have their dark time defined. 1786 lsst.ip.isr.isrFunctions.darkCorrection 1788 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1789 if math.isnan(expScale):
1790 raise RuntimeError(
"Exposure darktime is NAN")
1791 if darkExposure.getInfo().getVisitInfo()
is not None:
1792 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1798 if math.isnan(darkScale):
1799 raise RuntimeError(
"Dark calib darktime is NAN")
1800 isrFunctions.darkCorrection(
1801 maskedImage=exposure.getMaskedImage(),
1802 darkMaskedImage=darkExposure.getMaskedImage(),
1804 darkScale=darkScale,
1806 trimToFit=self.config.doTrimToMatchCalib
1810 """!Check if linearization is needed for the detector cameraGeom. 1812 Checks config.doLinearize and the linearity type of the first 1817 detector : `lsst.afw.cameraGeom.Detector` 1818 Detector to get linearity type from. 1822 doLinearize : `Bool` 1823 If True, linearization should be performed. 1825 return self.config.doLinearize
and \
1826 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1829 """!Apply flat correction in place. 1833 exposure : `lsst.afw.image.Exposure` 1834 Exposure to process. 1835 flatExposure : `lsst.afw.image.Exposure` 1836 Flat exposure of the same size as ``exposure``. 1837 invert : `Bool`, optional 1838 If True, unflatten an already flattened image. 1842 lsst.ip.isr.isrFunctions.flatCorrection 1844 isrFunctions.flatCorrection(
1845 maskedImage=exposure.getMaskedImage(),
1846 flatMaskedImage=flatExposure.getMaskedImage(),
1847 scalingType=self.config.flatScalingType,
1848 userScale=self.config.flatUserScale,
1850 trimToFit=self.config.doTrimToMatchCalib
1854 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1858 exposure : `lsst.afw.image.Exposure` 1859 Exposure to process. Only the amplifier DataSec is processed. 1860 amp : `lsst.afw.table.AmpInfoCatalog` 1861 Amplifier detector data. 1865 lsst.ip.isr.isrFunctions.makeThresholdMask 1867 if not math.isnan(amp.getSaturation()):
1868 maskedImage = exposure.getMaskedImage()
1869 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1870 isrFunctions.makeThresholdMask(
1871 maskedImage=dataView,
1872 threshold=amp.getSaturation(),
1874 maskName=self.config.saturatedMaskName,
1878 """!Interpolate over saturated pixels, in place. 1880 This method should be called after `saturationDetection`, to 1881 ensure that the saturated pixels have been identified in the 1882 SAT mask. It should also be called after `assembleCcd`, since 1883 saturated regions may cross amplifier boundaries. 1887 exposure : `lsst.afw.image.Exposure` 1888 Exposure to process. 1892 lsst.ip.isr.isrTask.saturationDetection 1893 lsst.ip.isr.isrFunctions.interpolateFromMask 1895 isrFunctions.interpolateFromMask(
1896 maskedImage=ccdExposure.getMaskedImage(),
1897 fwhm=self.config.fwhm,
1898 growFootprints=self.config.growSaturationFootprintSize,
1899 maskName=self.config.saturatedMaskName,
1903 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1907 exposure : `lsst.afw.image.Exposure` 1908 Exposure to process. Only the amplifier DataSec is processed. 1909 amp : `lsst.afw.table.AmpInfoCatalog` 1910 Amplifier detector data. 1914 lsst.ip.isr.isrFunctions.makeThresholdMask 1918 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1919 This is intended to indicate pixels that may be affected by unknown systematics; 1920 for example if non-linearity corrections above a certain level are unstable 1921 then that would be a useful value for suspectLevel. A value of `nan` indicates 1922 that no such level exists and no pixels are to be masked as suspicious. 1924 suspectLevel = amp.getSuspectLevel()
1925 if math.isnan(suspectLevel):
1928 maskedImage = exposure.getMaskedImage()
1929 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1930 isrFunctions.makeThresholdMask(
1931 maskedImage=dataView,
1932 threshold=suspectLevel,
1934 maskName=self.config.suspectMaskName,
1938 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1942 ccdExposure : `lsst.afw.image.Exposure` 1943 Exposure to process. 1944 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 1945 `lsst.afw.image.DefectBase`. 1946 List of defects to mask and interpolate. 1950 Call this after CCD assembly, since defects may cross amplifier boundaries. 1952 maskedImage = ccdExposure.getMaskedImage()
1953 if not isinstance(defectBaseList, Defects):
1955 defectList = Defects(defectBaseList)
1957 defectList = defectBaseList
1958 defectList.maskPixels(maskedImage, maskName=
"BAD")
1959 isrFunctions.interpolateDefectList(
1960 maskedImage=maskedImage,
1961 defectList=defectList,
1962 fwhm=self.config.fwhm,
1965 if self.config.numEdgeSuspect > 0:
1966 goodBBox = maskedImage.getBBox()
1968 goodBBox.grow(-self.config.numEdgeSuspect)
1970 SourceDetectionTask.setEdgeBits(
1973 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1977 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1981 exposure : `lsst.afw.image.Exposure` 1982 Exposure to process. 1986 We mask and interpolate over all NaNs, including those 1987 that are masked with other bits (because those may or may 1988 not be interpolated over later, and we want to remove all 1989 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1990 is used to preserve the historical name. 1992 maskedImage = exposure.getMaskedImage()
1995 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
1996 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
1997 numNans =
maskNans(maskedImage, maskVal)
1998 self.metadata.set(
"NUMNANS", numNans)
2002 self.log.warn(
"There were %i unmasked NaNs", numNans)
2003 nanDefectList = Defects.fromMask(
2004 maskedImage=maskedImage,
2005 maskName=
'UNMASKEDNAN',
2007 isrFunctions.interpolateDefectList(
2008 maskedImage=exposure.getMaskedImage(),
2009 defectList=nanDefectList,
2010 fwhm=self.config.fwhm,
2014 """Measure the image background in subgrids, for quality control purposes. 2018 exposure : `lsst.afw.image.Exposure` 2019 Exposure to process. 2020 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 2021 Configuration object containing parameters on which background 2022 statistics and subgrids to use. 2024 if IsrQaConfig
is not None:
2025 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2026 IsrQaConfig.flatness.nIter)
2027 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2028 statsControl.setAndMask(maskVal)
2029 maskedImage = exposure.getMaskedImage()
2030 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2031 skyLevel = stats.getValue(afwMath.MEDIAN)
2032 skySigma = stats.getValue(afwMath.STDEVCLIP)
2033 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2034 metadata = exposure.getMetadata()
2035 metadata.set(
'SKYLEVEL', skyLevel)
2036 metadata.set(
'SKYSIGMA', skySigma)
2039 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2040 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2041 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2042 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2043 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2044 skyLevels = numpy.zeros((nX, nY))
2047 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2049 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2051 xLLC = xc - meshXHalf
2052 yLLC = yc - meshYHalf
2053 xURC = xc + meshXHalf - 1
2054 yURC = yc + meshYHalf - 1
2057 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2059 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2061 good = numpy.where(numpy.isfinite(skyLevels))
2062 skyMedian = numpy.median(skyLevels[good])
2063 flatness = (skyLevels[good] - skyMedian) / skyMedian
2064 flatness_rms = numpy.std(flatness)
2065 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2067 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2068 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
2069 (nX, nY, flatness_pp, flatness_rms))
2071 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2072 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2073 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2074 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2075 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2078 """Set an approximate magnitude zero point for the exposure. 2082 exposure : `lsst.afw.image.Exposure` 2083 Exposure to process. 2085 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2086 if filterName
in self.config.fluxMag0T1:
2087 fluxMag0 = self.config.fluxMag0T1[filterName]
2089 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
2090 fluxMag0 = self.config.defaultFluxMag0T1
2092 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2094 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
2097 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2098 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2101 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2105 ccdExposure : `lsst.afw.image.Exposure` 2106 Exposure to process. 2107 fpPolygon : `lsst.afw.geom.Polygon` 2108 Polygon in focal plane coordinates. 2111 ccd = ccdExposure.getDetector()
2112 fpCorners = ccd.getCorners(FOCAL_PLANE)
2113 ccdPolygon = Polygon(fpCorners)
2116 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2119 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2120 validPolygon = Polygon(ccdPoints)
2121 ccdExposure.getInfo().setValidPolygon(validPolygon)
2125 """Context manager that applies and removes flats and darks, 2126 if the task is configured to apply them. 2130 exp : `lsst.afw.image.Exposure` 2131 Exposure to process. 2132 flat : `lsst.afw.image.Exposure` 2133 Flat exposure the same size as ``exp``. 2134 dark : `lsst.afw.image.Exposure`, optional 2135 Dark exposure the same size as ``exp``. 2139 exp : `lsst.afw.image.Exposure` 2140 The flat and dark corrected exposure. 2142 if self.config.doDark
and dark
is not None:
2144 if self.config.doFlat:
2149 if self.config.doFlat:
2151 if self.config.doDark
and dark
is not None:
2155 """Utility function to examine ISR exposure at different stages. 2159 exposure : `lsst.afw.image.Exposure` 2162 State of processing to view. 2164 frame = getDebugFrame(self._display, stepname)
2166 display = getDisplay(frame)
2167 display.scale(
'asinh',
'zscale')
2168 display.mtv(exposure)
2172 """A Detector-like object that supports returning gain and saturation level 2174 This is used when the input exposure does not have a detector. 2178 exposure : `lsst.afw.image.Exposure` 2179 Exposure to generate a fake amplifier for. 2180 config : `lsst.ip.isr.isrTaskConfig` 2181 Configuration to apply to the fake amplifier. 2185 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2187 self.
_gain = config.gain
2217 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2221 """Task to wrap the default IsrTask to allow it to be retargeted. 2223 The standard IsrTask can be called directly from a command line 2224 program, but doing so removes the ability of the task to be 2225 retargeted. As most cameras override some set of the IsrTask 2226 methods, this would remove those data-specific methods in the 2227 output post-ISR images. This wrapping class fixes the issue, 2228 allowing identical post-ISR images to be generated by both the 2229 processCcd and isrTask code. 2231 ConfigClass = RunIsrConfig
2232 _DefaultName =
"runIsr" 2236 self.makeSubtask(
"isr")
2242 dataRef : `lsst.daf.persistence.ButlerDataRef` 2243 data reference of the detector data to be processed 2247 result : `pipeBase.Struct` 2248 Result struct with component: 2250 - exposure : `lsst.afw.image.Exposure` 2251 Post-ISR processed exposure. def getInputDatasetTypes(cls, config)
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def getPrerequisiteDatasetTypes(cls, config)
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def getOutputDatasetTypes(cls, config)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def makeDatasetType(self, dsConfig)
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def getPerDatasetTypeDimensions(cls, config)
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, isGen3=False)
Perform instrument signature removal on an exposure.
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)