33 from contextlib
import contextmanager
34 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
47 from .assembleCcdTask
import AssembleCcdTask
48 from .crosstalk
import CrosstalkTask
49 from .fringe
import FringeTask
50 from .isr
import maskNans
51 from .masking
import MaskingTask
52 from .straylight
import StrayLightTask
53 from .vignette
import VignetteTask
55 __all__ = [
"IsrTask",
"RunIsrTask"]
59 """Configuration parameters for IsrTask. 61 Items are grouped in the order in which they are executed by the task. 66 isrName = pexConfig.Field(
73 ccdExposure = pipeBase.InputDatasetField(
74 doc=
"Input exposure to process",
77 storageClass=
"ExposureU",
78 dimensions=[
"Instrument",
"Exposure",
"Detector"],
80 camera = pipeBase.InputDatasetField(
81 doc=
"Input camera to construct complete exposures.",
84 storageClass=
"TablePersistableCamera",
85 dimensions=[
"Instrument",
"CalibrationLabel"],
87 bias = pipeBase.InputDatasetField(
88 doc=
"Input bias calibration.",
91 storageClass=
"ImageF",
92 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
94 dark = pipeBase.InputDatasetField(
95 doc=
"Input dark calibration.",
98 storageClass=
"ImageF",
99 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
101 flat = pipeBase.InputDatasetField(
102 doc=
"Input flat calibration.",
105 storageClass=
"MaskedImageF",
106 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel",
"Detector"],
108 bfKernel = pipeBase.InputDatasetField(
109 doc=
"Input brighter-fatter kernel.",
112 storageClass=
"NumpyArray",
113 dimensions=[
"Instrument",
"CalibrationLabel"],
115 defects = pipeBase.InputDatasetField(
116 doc=
"Input defect tables.",
119 storageClass=
"Catalog",
120 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
122 opticsTransmission = pipeBase.InputDatasetField(
123 doc=
"Transmission curve due to the optics.",
124 name=
"transmission_optics",
126 storageClass=
"TablePersistableTransmissionCurve",
127 dimensions=[
"Instrument",
"CalibrationLabel"],
129 filterTransmission = pipeBase.InputDatasetField(
130 doc=
"Transmission curve due to the filter.",
131 name=
"transmission_filter",
133 storageClass=
"TablePersistableTransmissionCurve",
134 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel"],
136 sensorTransmission = pipeBase.InputDatasetField(
137 doc=
"Transmission curve due to the sensor.",
138 name=
"transmission_sensor",
140 storageClass=
"TablePersistableTransmissionCurve",
141 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
143 atmosphereTransmission = pipeBase.InputDatasetField(
144 doc=
"Transmission curve due to the atmosphere.",
145 name=
"transmission_atmosphere",
147 storageClass=
"TablePersistableTransmissionCurve",
148 dimensions=[
"Instrument"],
152 outputExposure = pipeBase.OutputDatasetField(
153 doc=
"Output ISR processed exposure.",
156 storageClass=
"ExposureF",
157 dimensions=[
"Instrument",
"Visit",
"Detector"],
159 outputOssThumbnail = pipeBase.OutputDatasetField(
160 doc=
"Output Overscan-subtracted thumbnail image.",
163 storageClass=
"Thumbnail",
164 dimensions=[
"Instrument",
"Visit",
"Detector"],
166 outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167 doc=
"Output flat-corrected thumbnail image.",
168 name=
"FlattenedThumb",
170 storageClass=
"TextStorage",
171 dimensions=[
"Instrument",
"Visit",
"Detector"],
174 quantum = pipeBase.QuantumConfig(
175 dimensions=[
"Visit",
"Detector",
"Instrument"],
179 datasetType = pexConfig.Field(
181 doc=
"Dataset type for input data; users will typically leave this alone, " 182 "but camera-specific ISR tasks will override it",
186 fallbackFilterName = pexConfig.Field(
188 doc=
"Fallback default filter name for calibrations.",
191 expectWcs = pexConfig.Field(
194 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 196 fwhm = pexConfig.Field(
198 doc=
"FWHM of PSF in arcseconds.",
201 qa = pexConfig.ConfigField(
203 doc=
"QA related configuration options.",
207 doConvertIntToFloat = pexConfig.Field(
209 doc=
"Convert integer raw images to floating point values?",
214 doSaturation = pexConfig.Field(
216 doc=
"Mask saturated pixels? NB: this is totally independent of the" 217 " interpolation option - this is ONLY setting the bits in the mask." 218 " To have them interpolated make sure doSaturationInterpolation=True",
221 saturatedMaskName = pexConfig.Field(
223 doc=
"Name of mask plane to use in saturation detection and interpolation",
226 saturation = pexConfig.Field(
228 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229 default=float(
"NaN"),
231 growSaturationFootprintSize = pexConfig.Field(
233 doc=
"Number of pixels by which to grow the saturation footprints",
238 doSuspect = pexConfig.Field(
240 doc=
"Mask suspect pixels?",
243 suspectMaskName = pexConfig.Field(
245 doc=
"Name of mask plane to use for suspect pixels",
248 numEdgeSuspect = pexConfig.Field(
250 doc=
"Number of edge pixels to be flagged as untrustworthy.",
255 doSetBadRegions = pexConfig.Field(
257 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
260 badStatistic = pexConfig.ChoiceField(
262 doc=
"How to estimate the average value for BAD regions.",
265 "MEANCLIP":
"Correct using the (clipped) mean of good data",
266 "MEDIAN":
"Correct using the median of the good data",
271 doOverscan = pexConfig.Field(
273 doc=
"Do overscan subtraction?",
276 overscanFitType = pexConfig.ChoiceField(
278 doc=
"The method for fitting the overscan bias level.",
281 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
282 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
283 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
284 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
285 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
286 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
287 "MEAN":
"Correct using the mean of the overscan region",
288 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
289 "MEDIAN":
"Correct using the median of the overscan region",
292 overscanOrder = pexConfig.Field(
294 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
295 "or number of spline knots if overscan fit type is a spline."),
298 overscanNumSigmaClip = pexConfig.Field(
300 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
303 overscanIsInt = pexConfig.Field(
305 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
308 overscanNumLeadingColumnsToSkip = pexConfig.Field(
310 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
313 overscanNumTrailingColumnsToSkip = pexConfig.Field(
315 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
318 overscanMaxDev = pexConfig.Field(
320 doc=
"Maximum deviation from the median for overscan",
321 default=1000.0, check=
lambda x: x > 0
323 overscanBiasJump = pexConfig.Field(
325 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
328 overscanBiasJumpKeyword = pexConfig.Field(
330 doc=
"Header keyword containing information about devices.",
331 default=
"NO_SUCH_KEY",
333 overscanBiasJumpDevices = pexConfig.ListField(
335 doc=
"List of devices that need piecewise overscan correction.",
338 overscanBiasJumpLocation = pexConfig.Field(
340 doc=
"Location of bias jump along y-axis.",
345 doAssembleCcd = pexConfig.Field(
348 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 350 assembleCcd = pexConfig.ConfigurableField(
351 target=AssembleCcdTask,
352 doc=
"CCD assembly task",
356 doAssembleIsrExposures = pexConfig.Field(
359 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 361 doTrimToMatchCalib = pexConfig.Field(
364 doc=
"Trim raw data to match calibration bounding boxes?" 368 doBias = pexConfig.Field(
370 doc=
"Apply bias frame correction?",
373 biasDataProductName = pexConfig.Field(
375 doc=
"Name of the bias data product",
380 doVariance = pexConfig.Field(
382 doc=
"Calculate variance?",
385 gain = pexConfig.Field(
387 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388 default=float(
"NaN"),
390 readNoise = pexConfig.Field(
392 doc=
"The read noise to use if no Detector is present in the Exposure",
395 doEmpiricalReadNoise = pexConfig.Field(
398 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 402 doLinearize = pexConfig.Field(
404 doc=
"Correct for nonlinearity of the detector's response?",
409 doCrosstalk = pexConfig.Field(
411 doc=
"Apply intra-CCD crosstalk correction?",
414 doCrosstalkBeforeAssemble = pexConfig.Field(
416 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
419 crosstalk = pexConfig.ConfigurableField(
420 target=CrosstalkTask,
421 doc=
"Intra-CCD crosstalk correction",
425 doWidenSaturationTrails = pexConfig.Field(
427 doc=
"Widen bleed trails based on their width?",
432 doBrighterFatter = pexConfig.Field(
435 doc=
"Apply the brighter fatter correction" 437 brighterFatterLevel = pexConfig.ChoiceField(
440 doc=
"The level at which to correct for brighter-fatter.",
442 "AMP":
"Every amplifier treated separately.",
443 "DETECTOR":
"One kernel per detector",
446 brighterFatterKernelFile = pexConfig.Field(
449 doc=
"Kernel file used for the brighter fatter correction" 451 brighterFatterMaxIter = pexConfig.Field(
454 doc=
"Maximum number of iterations for the brighter fatter correction" 456 brighterFatterThreshold = pexConfig.Field(
459 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 460 " absolute value of the difference between the current corrected image and the one" 461 " from the previous iteration summed over all the pixels." 463 brighterFatterApplyGain = pexConfig.Field(
466 doc=
"Should the gain be applied when applying the brighter fatter correction?" 470 doDefect = pexConfig.Field(
472 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
475 doSaturationInterpolation = pexConfig.Field(
477 doc=
"Perform interpolation over pixels masked as saturated?" 478 " NB: This is independent of doSaturation; if that is False this plane" 479 " will likely be blank, resulting in a no-op here.",
482 numEdgeSuspect = pexConfig.Field(
484 doc=
"Number of edge pixels to be flagged as untrustworthy.",
489 doDark = pexConfig.Field(
491 doc=
"Apply dark frame correction?",
494 darkDataProductName = pexConfig.Field(
496 doc=
"Name of the dark data product",
501 doStrayLight = pexConfig.Field(
503 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
506 strayLight = pexConfig.ConfigurableField(
507 target=StrayLightTask,
508 doc=
"y-band stray light correction" 512 doFlat = pexConfig.Field(
514 doc=
"Apply flat field correction?",
517 flatDataProductName = pexConfig.Field(
519 doc=
"Name of the flat data product",
522 flatScalingType = pexConfig.ChoiceField(
524 doc=
"The method for scaling the flat on the fly.",
527 "USER":
"Scale by flatUserScale",
528 "MEAN":
"Scale by the inverse of the mean",
529 "MEDIAN":
"Scale by the inverse of the median",
532 flatUserScale = pexConfig.Field(
534 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
537 doTweakFlat = pexConfig.Field(
539 doc=
"Tweak flats to match observed amplifier ratios?",
544 doApplyGains = pexConfig.Field(
546 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
549 normalizeGains = pexConfig.Field(
551 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
556 doFringe = pexConfig.Field(
558 doc=
"Apply fringe correction?",
561 fringe = pexConfig.ConfigurableField(
563 doc=
"Fringe subtraction task",
565 fringeAfterFlat = pexConfig.Field(
567 doc=
"Do fringe subtraction after flat-fielding?",
572 doNanInterpAfterFlat = pexConfig.Field(
574 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 575 "also have to interpolate them before flat-fielding."),
580 doAddDistortionModel = pexConfig.Field(
582 doc=
"Apply a distortion model based on camera geometry to the WCS?",
587 doMeasureBackground = pexConfig.Field(
589 doc=
"Measure the background level on the reduced image?",
594 doCameraSpecificMasking = pexConfig.Field(
596 doc=
"Mask camera-specific bad regions?",
599 masking = pexConfig.ConfigurableField(
605 fluxMag0T1 = pexConfig.DictField(
608 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
612 defaultFluxMag0T1 = pexConfig.Field(
614 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
615 default=pow(10.0, 0.4*28.0)
619 doVignette = pexConfig.Field(
621 doc=
"Apply vignetting parameters?",
624 vignette = pexConfig.ConfigurableField(
626 doc=
"Vignetting task.",
630 doAttachTransmissionCurve = pexConfig.Field(
633 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 635 doUseOpticsTransmission = pexConfig.Field(
638 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 640 doUseFilterTransmission = pexConfig.Field(
643 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 645 doUseSensorTransmission = pexConfig.Field(
648 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 650 doUseAtmosphereTransmission = pexConfig.Field(
653 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 657 doWrite = pexConfig.Field(
659 doc=
"Persist postISRCCD?",
666 raise ValueError(
"You may not specify both doFlat and doApplyGains")
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670 r"""Apply common instrument signature correction algorithms to a raw frame. 672 The process for correcting imaging data is very similar from 673 camera to camera. This task provides a vanilla implementation of 674 doing these corrections, including the ability to turn certain 675 corrections off if they are not needed. The inputs to the primary 676 method, `run()`, are a raw exposure to be corrected and the 677 calibration data products. The raw input is a single chip sized 678 mosaic of all amps including overscans and other non-science 679 pixels. The method `runDataRef()` identifies and defines the 680 calibration data products, and is intended for use by a 681 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 682 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 683 subclassed for different camera, although the most camera specific 684 methods have been split into subtasks that can be redirected 687 The __init__ method sets up the subtasks for ISR processing, using 688 the defaults from `lsst.ip.isr`. 693 Positional arguments passed to the Task constructor. None used at this time. 694 kwargs : `dict`, optional 695 Keyword arguments passed on to the Task constructor. None used at this time. 697 ConfigClass = IsrTaskConfig
702 self.makeSubtask(
"assembleCcd")
703 self.makeSubtask(
"crosstalk")
704 self.makeSubtask(
"strayLight")
705 self.makeSubtask(
"fringe")
706 self.makeSubtask(
"masking")
707 self.makeSubtask(
"vignette")
716 if config.doBias
is not True:
717 inputTypeDict.pop(
"bias",
None)
718 if config.doLinearize
is not True:
719 inputTypeDict.pop(
"linearizer",
None)
720 if config.doCrosstalk
is not True:
721 inputTypeDict.pop(
"crosstalkSources",
None)
722 if config.doBrighterFatter
is not True:
723 inputTypeDict.pop(
"bfKernel",
None)
724 if config.doDefect
is not True:
725 inputTypeDict.pop(
"defects",
None)
726 if config.doDark
is not True:
727 inputTypeDict.pop(
"dark",
None)
728 if config.doFlat
is not True:
729 inputTypeDict.pop(
"flat",
None)
730 if config.doAttachTransmissionCurve
is not True:
731 inputTypeDict.pop(
"opticsTransmission",
None)
732 inputTypeDict.pop(
"filterTransmission",
None)
733 inputTypeDict.pop(
"sensorTransmission",
None)
734 inputTypeDict.pop(
"atmosphereTransmission",
None)
735 if config.doUseOpticsTransmission
is not True:
736 inputTypeDict.pop(
"opticsTransmission",
None)
737 if config.doUseFilterTransmission
is not True:
738 inputTypeDict.pop(
"filterTransmission",
None)
739 if config.doUseSensorTransmission
is not True:
740 inputTypeDict.pop(
"sensorTransmission",
None)
741 if config.doUseAtmosphereTransmission
is not True:
742 inputTypeDict.pop(
"atmosphereTransmission",
None)
750 if config.qa.doThumbnailOss
is not True:
751 outputTypeDict.pop(
"outputOssThumbnail",
None)
752 if config.qa.doThumbnailFlattened
is not True:
753 outputTypeDict.pop(
"outputFlattenedThumbnail",
None)
754 if config.doWrite
is not True:
755 outputTypeDict.pop(
"outputExposure",
None)
757 return outputTypeDict
766 names.remove(
"ccdExposure")
775 return frozenset([
"CalibrationLabel"])
779 inputData[
'detectorNum'] = int(inputDataIds[
'ccdExposure'][
'detector'])
780 except Exception
as e:
781 raise ValueError(f
"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
783 inputData[
'isGen3'] =
True 785 if self.config.doLinearize
is True:
786 if 'linearizer' not in inputData.keys():
787 detector = inputData[
'camera'][inputData[
'detectorNum']]
788 linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
789 inputData[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
791 if inputData[
'defects']
is not None:
796 for r
in inputData[
'defects']:
797 bbox = afwGeom.BoxI(afwGeom.PointI(r.get(
"x0"), r.get(
"y0")),
798 afwGeom.ExtentI(r.get(
"width"), r.get(
"height")))
799 defectList.append(
Defect(bbox))
801 inputData[
'defects'] = defectList
818 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
824 """!Retrieve necessary frames for instrument signature removal. 826 Pre-fetching all required ISR data products limits the IO 827 required by the ISR. Any conflict between the calibration data 828 available and that needed for ISR is also detected prior to 829 doing processing, allowing it to fail quickly. 833 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 834 Butler reference of the detector data to be processed 835 rawExposure : `afw.image.Exposure` 836 The raw exposure that will later be corrected with the 837 retrieved calibration data; should not be modified in this 842 result : `lsst.pipe.base.Struct` 843 Result struct with components (which may be `None`): 844 - ``bias``: bias calibration frame (`afw.image.Exposure`) 845 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 846 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 847 - ``dark``: dark calibration frame (`afw.image.Exposure`) 848 - ``flat``: flat calibration frame (`afw.image.Exposure`) 849 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 850 - ``defects``: list of defects (`list`) 851 - ``fringes``: `lsst.pipe.base.Struct` with components: 852 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 853 - ``seed``: random seed derived from the ccdExposureId for random 854 number generator (`uint32`) 855 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 856 A ``TransmissionCurve`` that represents the throughput of the optics, 857 to be evaluated in focal-plane coordinates. 858 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 859 A ``TransmissionCurve`` that represents the throughput of the filter 860 itself, to be evaluated in focal-plane coordinates. 861 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 862 A ``TransmissionCurve`` that represents the throughput of the sensor 863 itself, to be evaluated in post-assembly trimmed detector coordinates. 864 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 865 A ``TransmissionCurve`` that represents the throughput of the 866 atmosphere, assumed to be spatially constant. 869 ccd = rawExposure.getDetector()
870 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
871 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
872 if self.config.doBias
else None)
874 linearizer = (dataRef.get(
"linearizer", immediate=
True)
876 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
877 if self.config.doCrosstalk
else None)
878 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
879 if self.config.doDark
else None)
880 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
881 if self.config.doFlat
else None)
882 brighterFatterKernel = (dataRef.get(
"bfKernel")
883 if self.config.doBrighterFatter
else None)
884 defectList = (dataRef.get(
"defects")
885 if self.config.doDefect
else None)
886 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
887 if self.config.doAssembleIsrExposures
else None)
888 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
889 else pipeBase.Struct(fringes=
None))
891 if self.config.doAttachTransmissionCurve:
892 opticsTransmission = (dataRef.get(
"transmission_optics")
893 if self.config.doUseOpticsTransmission
else None)
894 filterTransmission = (dataRef.get(
"transmission_filter")
895 if self.config.doUseFilterTransmission
else None)
896 sensorTransmission = (dataRef.get(
"transmission_sensor")
897 if self.config.doUseSensorTransmission
else None)
898 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
899 if self.config.doUseAtmosphereTransmission
else None)
901 opticsTransmission =
None 902 filterTransmission =
None 903 sensorTransmission =
None 904 atmosphereTransmission =
None 907 return pipeBase.Struct(bias=biasExposure,
908 linearizer=linearizer,
909 crosstalkSources=crosstalkSources,
912 bfKernel=brighterFatterKernel,
914 fringes=fringeStruct,
915 opticsTransmission=opticsTransmission,
916 filterTransmission=filterTransmission,
917 sensorTransmission=sensorTransmission,
918 atmosphereTransmission=atmosphereTransmission,
922 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
923 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
924 opticsTransmission=None, filterTransmission=None,
925 sensorTransmission=None, atmosphereTransmission=None,
926 detectorNum=None, isGen3=False
928 """!Perform instrument signature removal on an exposure. 930 Steps included in the ISR processing, in order performed, are: 931 - saturation and suspect pixel masking 932 - overscan subtraction 933 - CCD assembly of individual amplifiers 935 - variance image construction 936 - linearization of non-linear response 938 - brighter-fatter correction 941 - stray light subtraction 943 - masking of known defects and camera specific features 944 - vignette calculation 945 - appending transmission curve and distortion model 949 ccdExposure : `lsst.afw.image.Exposure` 950 The raw exposure that is to be run through ISR. The 951 exposure is modified by this method. 952 camera : `lsst.afw.cameraGeom.Camera`, optional 953 The camera geometry for this exposure. Used to select the 954 distortion model appropriate for this data. 955 bias : `lsst.afw.image.Exposure`, optional 956 Bias calibration frame. 957 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 958 Functor for linearization. 959 crosstalkSources : `list`, optional 960 List of possible crosstalk sources. 961 dark : `lsst.afw.image.Exposure`, optional 962 Dark calibration frame. 963 flat : `lsst.afw.image.Exposure`, optional 964 Flat calibration frame. 965 bfKernel : `numpy.ndarray`, optional 966 Brighter-fatter kernel. 967 defects : `list`, optional 969 fringes : `lsst.pipe.base.Struct`, optional 970 Struct containing the fringe correction data, with 972 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 973 - ``seed``: random seed derived from the ccdExposureId for random 974 number generator (`uint32`) 975 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 976 A ``TransmissionCurve`` that represents the throughput of the optics, 977 to be evaluated in focal-plane coordinates. 978 filterTransmission : `lsst.afw.image.TransmissionCurve` 979 A ``TransmissionCurve`` that represents the throughput of the filter 980 itself, to be evaluated in focal-plane coordinates. 981 sensorTransmission : `lsst.afw.image.TransmissionCurve` 982 A ``TransmissionCurve`` that represents the throughput of the sensor 983 itself, to be evaluated in post-assembly trimmed detector coordinates. 984 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 985 A ``TransmissionCurve`` that represents the throughput of the 986 atmosphere, assumed to be spatially constant. 987 detectorNum : `int`, optional 988 The integer number for the detector to process. 989 isGen3 : bool, optional 990 Flag this call to run() as using the Gen3 butler environment. 994 result : `lsst.pipe.base.Struct` 995 Result struct with component: 996 - ``exposure`` : `afw.image.Exposure` 997 The fully ISR corrected exposure. 998 - ``outputExposure`` : `afw.image.Exposure` 999 An alias for `exposure` 1000 - ``ossThumb`` : `numpy.ndarray` 1001 Thumbnail image of the exposure after overscan subtraction. 1002 - ``flattenedThumb`` : `numpy.ndarray` 1003 Thumbnail image of the exposure after flat-field correction. 1008 Raised if a configuration option is set to True, but the 1009 required calibration data has not been specified. 1013 The current processed exposure can be viewed by setting the 1014 appropriate lsstDebug entries in the `debug.display` 1015 dictionary. The names of these entries correspond to some of 1016 the IsrTaskConfig Boolean options, with the value denoting the 1017 frame to use. The exposure is shown inside the matching 1018 option check and after the processing of that step has 1019 finished. The steps with debug points are: 1030 In addition, setting the "postISRCCD" entry displays the 1031 exposure after all ISR processing has finished. 1039 self.config.doFringe =
False 1042 if detectorNum
is None:
1043 raise RuntimeError(
"Must supply the detectorNum if running as Gen3")
1045 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1050 if isinstance(ccdExposure, ButlerDataRef):
1053 ccd = ccdExposure.getDetector()
1056 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 1057 ccd = [
FakeAmp(ccdExposure, self.config)]
1060 if self.config.doBias
and bias
is None:
1061 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1063 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1064 if self.config.doBrighterFatter
and bfKernel
is None:
1065 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1066 if self.config.doDark
and dark
is None:
1067 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1069 fringes = pipeBase.Struct(fringes=
None)
1070 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
1071 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1072 if self.config.doFlat
and flat
is None:
1073 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1074 if self.config.doDefect
and defects
is None:
1075 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1076 if self.config.doAddDistortionModel
and camera
is None:
1077 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
1080 if self.config.doConvertIntToFloat:
1081 self.log.info(
"Converting exposure to floating point values")
1088 if ccdExposure.getBBox().contains(amp.getBBox()):
1092 if self.config.doOverscan
and not badAmp:
1095 self.log.debug(
"Corrected overscan for amplifier %s" % (amp.getName()))
1096 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1097 if isinstance(overscanResults.overscanFit, float):
1098 qaMedian = overscanResults.overscanFit
1099 qaStdev = float(
"NaN")
1101 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1102 afwMath.MEDIAN | afwMath.STDEVCLIP)
1103 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1104 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1106 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1107 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1108 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f" %
1109 (amp.getName(), qaMedian, qaStdev))
1110 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1112 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
1113 overscanResults =
None 1115 overscans.append(overscanResults
if overscanResults
is not None else None)
1117 self.log.info(
"Skipped OSCAN")
1119 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1120 self.log.info(
"Applying crosstalk correction.")
1121 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1122 self.
debugView(ccdExposure,
"doCrosstalk")
1124 if self.config.doAssembleCcd:
1125 self.log.info(
"Assembling CCD from amplifiers")
1126 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1128 if self.config.expectWcs
and not ccdExposure.getWcs():
1129 self.log.warn(
"No WCS found in input exposure")
1130 self.
debugView(ccdExposure,
"doAssembleCcd")
1133 if self.config.qa.doThumbnailOss:
1134 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1136 if self.config.doBias:
1137 self.log.info(
"Applying bias correction.")
1138 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1139 trimToFit=self.config.doTrimToMatchCalib)
1142 if self.config.doVariance:
1143 for amp, overscanResults
in zip(ccd, overscans):
1144 if ccdExposure.getBBox().contains(amp.getBBox()):
1145 self.log.debug(
"Constructing variance map for amplifer %s" % (amp.getName()))
1146 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1147 if overscanResults
is not None:
1149 overscanImage=overscanResults.overscanImage)
1153 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1154 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1155 afwMath.MEDIAN | afwMath.STDEVCLIP)
1156 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
1157 qaStats.getValue(afwMath.MEDIAN))
1158 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
1159 qaStats.getValue(afwMath.STDEVCLIP))
1160 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f" %
1161 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1162 qaStats.getValue(afwMath.STDEVCLIP)))
1165 self.log.info(
"Applying linearizer.")
1166 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1168 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1169 self.log.info(
"Applying crosstalk correction.")
1170 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1171 self.
debugView(ccdExposure,
"doCrosstalk")
1173 if self.config.doWidenSaturationTrails:
1174 self.log.info(
"Widening saturation trails.")
1175 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1177 interpolationDone =
False 1178 if self.config.doBrighterFatter:
1184 if self.config.doDefect:
1187 if self.config.doSaturationInterpolation:
1191 interpolationDone =
True 1193 if self.config.brighterFatterLevel ==
'DETECTOR':
1194 kernelElement = bfKernel
1197 raise NotImplementedError(
"per-amplifier brighter-fatter correction not yet implemented")
1198 self.log.info(
"Applying brighter fatter correction.")
1199 isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
1200 self.config.brighterFatterMaxIter,
1201 self.config.brighterFatterThreshold,
1202 self.config.brighterFatterApplyGain,
1204 self.
debugView(ccdExposure,
"doBrighterFatter")
1206 if self.config.doDark:
1207 self.log.info(
"Applying dark correction.")
1211 if self.config.doFringe
and not self.config.fringeAfterFlat:
1212 self.log.info(
"Applying fringe correction before flat.")
1213 self.fringe.
run(ccdExposure, **fringes.getDict())
1216 if self.config.doStrayLight:
1217 self.log.info(
"Applying stray light correction.")
1218 self.strayLight.
run(ccdExposure)
1219 self.
debugView(ccdExposure,
"doStrayLight")
1221 if self.config.doFlat:
1222 self.log.info(
"Applying flat correction.")
1226 if self.config.doApplyGains:
1227 self.log.info(
"Applying gain correction instead of flat.")
1228 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1230 if self.config.doDefect
and not interpolationDone:
1231 self.log.info(
"Masking and interpolating defects.")
1234 if self.config.doSaturationInterpolation
and not interpolationDone:
1235 self.log.info(
"Interpolating saturated pixels.")
1238 if self.config.doNanInterpAfterFlat
or not interpolationDone:
1239 self.log.info(
"Masking and interpolating NAN value pixels.")
1242 if self.config.doFringe
and self.config.fringeAfterFlat:
1243 self.log.info(
"Applying fringe correction after flat.")
1244 self.fringe.
run(ccdExposure, **fringes.getDict())
1246 if self.config.doSetBadRegions:
1247 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1248 if badPixelCount > 0:
1249 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1251 flattenedThumb =
None 1252 if self.config.qa.doThumbnailFlattened:
1253 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1255 if self.config.doCameraSpecificMasking:
1256 self.log.info(
"Masking regions for camera specific reasons.")
1257 self.masking.
run(ccdExposure)
1261 if self.config.doVignette:
1262 self.log.info(
"Constructing Vignette polygon.")
1265 if self.config.vignette.doWriteVignettePolygon:
1268 if self.config.doAttachTransmissionCurve:
1269 self.log.info(
"Adding transmission curves.")
1270 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1271 filterTransmission=filterTransmission,
1272 sensorTransmission=sensorTransmission,
1273 atmosphereTransmission=atmosphereTransmission)
1275 if self.config.doAddDistortionModel:
1276 self.log.info(
"Adding a distortion model to the WCS.")
1277 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1279 if self.config.doMeasureBackground:
1280 self.log.info(
"Measuring background level:")
1283 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1285 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1286 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1287 afwMath.MEDIAN | afwMath.STDEVCLIP)
1288 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1289 qaStats.getValue(afwMath.MEDIAN))
1290 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1291 qaStats.getValue(afwMath.STDEVCLIP))
1292 self.log.debug(
" Background stats for amplifer %s: %f +/- %f" %
1293 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1294 qaStats.getValue(afwMath.STDEVCLIP)))
1296 self.
debugView(ccdExposure,
"postISRCCD")
1298 return pipeBase.Struct(
1299 exposure=ccdExposure,
1301 flattenedThumb=flattenedThumb,
1303 outputExposure=ccdExposure,
1304 outputOssThumbnail=ossThumb,
1305 outputFlattenedThumbnail=flattenedThumb,
1308 @pipeBase.timeMethod
1310 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1312 This method contains the `CmdLineTask` interface to the ISR 1313 processing. All IO is handled here, freeing the `run()` method 1314 to manage only pixel-level calculations. The steps performed 1316 - Read in necessary detrending/isr/calibration data. 1317 - Process raw exposure in `run()`. 1318 - Persist the ISR-corrected exposure as "postISRCCD" if 1319 config.doWrite=True. 1323 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1324 DataRef of the detector data to be processed 1328 result : `lsst.pipe.base.Struct` 1329 Result struct with component: 1330 - ``exposure`` : `afw.image.Exposure` 1331 The fully ISR corrected exposure. 1336 Raised if a configuration option is set to True, but the 1337 required calibration data does not exist. 1340 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1342 ccdExposure = sensorRef.get(self.config.datasetType)
1344 camera = sensorRef.get(
"camera")
1345 if camera
is None and self.config.doAddDistortionModel:
1346 raise RuntimeError(
"config.doAddDistortionModel is True " 1347 "but could not get a camera from the butler")
1348 isrData = self.
readIsrData(sensorRef, ccdExposure)
1350 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1352 if self.config.doWrite:
1353 sensorRef.put(result.exposure,
"postISRCCD")
1354 if result.ossThumb
is not None:
1355 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1356 if result.flattenedThumb
is not None:
1357 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1362 """!Retrieve a calibration dataset for removing instrument signature. 1367 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1368 DataRef of the detector data to find calibration datasets 1371 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1373 If True, disable butler proxies to enable error handling 1374 within this routine. 1378 exposure : `lsst.afw.image.Exposure` 1379 Requested calibration frame. 1384 Raised if no matching calibration frame can be found. 1387 exp = dataRef.get(datasetType, immediate=immediate)
1388 except Exception
as exc1:
1389 if not self.config.fallbackFilterName:
1390 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1392 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1393 except Exception
as exc2:
1394 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1395 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1396 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1398 if self.config.doAssembleIsrExposures:
1399 exp = self.assembleCcd.assembleCcd(exp)
1403 """Ensure that the data returned by Butler is a fully constructed exposure. 1405 ISR requires exposure-level image data for historical reasons, so if we did 1406 not recieve that from Butler, construct it from what we have, modifying the 1411 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1412 `lsst.afw.image.ImageF` 1413 The input data structure obtained from Butler. 1414 camera : `lsst.afw.cameraGeom.camera` 1415 The camera associated with the image. Used to find the appropriate 1418 The detector this exposure should match. 1422 inputExp : `lsst.afw.image.Exposure` 1423 The re-constructed exposure, with appropriate detector parameters. 1428 Raised if the input data cannot be used to construct an exposure. 1430 if isinstance(inputExp, afwImage.DecoratedImageU):
1431 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1432 elif isinstance(inputExp, afwImage.ImageF):
1433 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1434 elif isinstance(inputExp, afwImage.MaskedImageF):
1435 inputExp = afwImage.makeExposure(inputExp)
1436 elif isinstance(inputExp, afwImage.Exposure):
1439 raise TypeError(f
"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1441 if inputExp.getDetector()
is None:
1442 inputExp.setDetector(camera[detectorNum])
1447 """Convert exposure image from uint16 to float. 1449 If the exposure does not need to be converted, the input is 1450 immediately returned. For exposures that are converted to use 1451 floating point pixels, the variance is set to unity and the 1456 exposure : `lsst.afw.image.Exposure` 1457 The raw exposure to be converted. 1461 newexposure : `lsst.afw.image.Exposure` 1462 The input ``exposure``, converted to floating point pixels. 1467 Raised if the exposure type cannot be converted to float. 1470 if isinstance(exposure, afwImage.ExposureF):
1473 if not hasattr(exposure,
"convertF"):
1474 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1476 newexposure = exposure.convertF()
1477 newexposure.variance[:] = 1
1478 newexposure.mask[:] = 0x0
1483 """Identify bad amplifiers, saturated and suspect pixels. 1487 ccdExposure : `lsst.afw.image.Exposure` 1488 Input exposure to be masked. 1489 amp : `lsst.afw.table.AmpInfoCatalog` 1490 Catalog of parameters defining the amplifier on this 1493 List of defects. Used to determine if the entire 1499 If this is true, the entire amplifier area is covered by 1500 defects and unusable. 1503 maskedImage = ccdExposure.getMaskedImage()
1509 if defects
is not None:
1510 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1515 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1517 maskView = dataView.getMask()
1518 maskView |= maskView.getPlaneBitMask(
"BAD")
1525 if self.config.doSaturation
and not badAmp:
1526 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1527 if self.config.doSuspect
and not badAmp:
1528 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1530 for maskName, maskThreshold
in limits.items():
1531 if not math.isnan(maskThreshold):
1532 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1533 isrFunctions.makeThresholdMask(
1534 maskedImage=dataView,
1535 threshold=maskThreshold,
1541 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1543 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1544 self.config.suspectMaskName])
1545 if numpy.all(maskView.getArray() & maskVal > 0):
1551 """Apply overscan correction in place. 1553 This method does initial pixel rejection of the overscan 1554 region. The overscan can also be optionally segmented to 1555 allow for discontinuous overscan responses to be fit 1556 separately. The actual overscan subtraction is performed by 1557 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1558 which is called here after the amplifier is preprocessed. 1562 ccdExposure : `lsst.afw.image.Exposure` 1563 Exposure to have overscan correction performed. 1564 amp : `lsst.afw.table.AmpInfoCatalog` 1565 The amplifier to consider while correcting the overscan. 1569 overscanResults : `lsst.pipe.base.Struct` 1570 Result struct with components: 1571 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1572 Value or fit subtracted from the amplifier image data. 1573 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1574 Value or fit subtracted from the overscan image data. 1575 - ``overscanImage`` : `lsst.afw.image.Image` 1576 Image of the overscan region with the overscan 1577 correction applied. This quantity is used to estimate 1578 the amplifier read noise empirically. 1583 Raised if the ``amp`` does not contain raw pixel information. 1587 lsst.ip.isr.isrFunctions.overscanCorrection 1589 if not amp.getHasRawInfo():
1590 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1592 if amp.getRawHorizontalOverscanBBox().isEmpty():
1593 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1596 statControl = afwMath.StatisticsControl()
1597 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1600 dataBBox = amp.getRawDataBBox()
1601 oscanBBox = amp.getRawHorizontalOverscanBBox()
1605 prescanBBox = amp.getRawPrescanBBox()
1606 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1607 dx0 += self.config.overscanNumLeadingColumnsToSkip
1608 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1610 dx0 += self.config.overscanNumTrailingColumnsToSkip
1611 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1617 if ((self.config.overscanBiasJump
and 1618 self.config.overscanBiasJumpLocation)
and 1619 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1620 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1621 self.config.overscanBiasJumpDevices)):
1622 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1623 yLower = self.config.overscanBiasJumpLocation
1624 yUpper = dataBBox.getHeight() - yLower
1626 yUpper = self.config.overscanBiasJumpLocation
1627 yLower = dataBBox.getHeight() - yUpper
1629 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1630 afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1631 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1632 afwGeom.Extent2I(dx0, 0),
1633 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1636 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1637 afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1638 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1639 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1642 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1643 afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1644 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1645 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1646 oscanBBox.getHeight())))
1649 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1650 ampImage = ccdExposure.maskedImage[imageBBox]
1651 overscanImage = ccdExposure.maskedImage[overscanBBox]
1653 overscanArray = overscanImage.image.array
1654 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1655 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1656 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1658 statControl = afwMath.StatisticsControl()
1659 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1661 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1662 overscanImage=overscanImage,
1663 fitType=self.config.overscanFitType,
1664 order=self.config.overscanOrder,
1665 collapseRej=self.config.overscanNumSigmaClip,
1666 statControl=statControl,
1667 overscanIsInt=self.config.overscanIsInt
1671 levelStat = afwMath.MEDIAN
1672 sigmaStat = afwMath.STDEVCLIP
1674 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1675 self.config.qa.flatness.nIter)
1676 metadata = ccdExposure.getMetadata()
1677 ampNum = amp.getName()
1678 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1679 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1680 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1682 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1683 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1684 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1686 return overscanResults
1689 """Set the variance plane using the amplifier gain and read noise 1691 The read noise is calculated from the ``overscanImage`` if the 1692 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1693 the value from the amplifier data is used. 1697 ampExposure : `lsst.afw.image.Exposure` 1698 Exposure to process. 1699 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1700 Amplifier detector data. 1701 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1702 Image of overscan, required only for empirical read noise. 1706 lsst.ip.isr.isrFunctions.updateVariance 1708 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1709 gain = amp.getGain()
1711 if math.isnan(gain):
1713 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1716 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1717 (amp.getName(), gain, patchedGain))
1720 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1721 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1723 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1724 stats = afwMath.StatisticsControl()
1725 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1726 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1727 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1729 readNoise = amp.getReadNoise()
1731 isrFunctions.updateVariance(
1732 maskedImage=ampExposure.getMaskedImage(),
1734 readNoise=readNoise,
1738 """!Apply dark correction in place. 1742 exposure : `lsst.afw.image.Exposure` 1743 Exposure to process. 1744 darkExposure : `lsst.afw.image.Exposure` 1745 Dark exposure of the same size as ``exposure``. 1746 invert : `Bool`, optional 1747 If True, re-add the dark to an already corrected image. 1752 Raised if either ``exposure`` or ``darkExposure`` do not 1753 have their dark time defined. 1757 lsst.ip.isr.isrFunctions.darkCorrection 1759 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1760 if math.isnan(expScale):
1761 raise RuntimeError(
"Exposure darktime is NAN")
1762 if darkExposure.getInfo().getVisitInfo()
is not None:
1763 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1769 if math.isnan(darkScale):
1770 raise RuntimeError(
"Dark calib darktime is NAN")
1771 isrFunctions.darkCorrection(
1772 maskedImage=exposure.getMaskedImage(),
1773 darkMaskedImage=darkExposure.getMaskedImage(),
1775 darkScale=darkScale,
1777 trimToFit=self.config.doTrimToMatchCalib
1781 """!Check if linearization is needed for the detector cameraGeom. 1783 Checks config.doLinearize and the linearity type of the first 1788 detector : `lsst.afw.cameraGeom.Detector` 1789 Detector to get linearity type from. 1793 doLinearize : `Bool` 1794 If True, linearization should be performed. 1796 return self.config.doLinearize
and \
1797 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1800 """!Apply flat correction in place. 1804 exposure : `lsst.afw.image.Exposure` 1805 Exposure to process. 1806 flatExposure : `lsst.afw.image.Exposure` 1807 Flat exposure of the same size as ``exposure``. 1808 invert : `Bool`, optional 1809 If True, unflatten an already flattened image. 1813 lsst.ip.isr.isrFunctions.flatCorrection 1815 isrFunctions.flatCorrection(
1816 maskedImage=exposure.getMaskedImage(),
1817 flatMaskedImage=flatExposure.getMaskedImage(),
1818 scalingType=self.config.flatScalingType,
1819 userScale=self.config.flatUserScale,
1821 trimToFit=self.config.doTrimToMatchCalib
1825 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1829 exposure : `lsst.afw.image.Exposure` 1830 Exposure to process. Only the amplifier DataSec is processed. 1831 amp : `lsst.afw.table.AmpInfoCatalog` 1832 Amplifier detector data. 1836 lsst.ip.isr.isrFunctions.makeThresholdMask 1838 if not math.isnan(amp.getSaturation()):
1839 maskedImage = exposure.getMaskedImage()
1840 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1841 isrFunctions.makeThresholdMask(
1842 maskedImage=dataView,
1843 threshold=amp.getSaturation(),
1845 maskName=self.config.saturatedMaskName,
1849 """!Interpolate over saturated pixels, in place. 1851 This method should be called after `saturationDetection`, to 1852 ensure that the saturated pixels have been identified in the 1853 SAT mask. It should also be called after `assembleCcd`, since 1854 saturated regions may cross amplifier boundaries. 1858 exposure : `lsst.afw.image.Exposure` 1859 Exposure to process. 1863 lsst.ip.isr.isrTask.saturationDetection 1864 lsst.ip.isr.isrFunctions.interpolateFromMask 1866 isrFunctions.interpolateFromMask(
1867 maskedImage=ccdExposure.getMaskedImage(),
1868 fwhm=self.config.fwhm,
1869 growFootprints=self.config.growSaturationFootprintSize,
1870 maskName=self.config.saturatedMaskName,
1874 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1878 exposure : `lsst.afw.image.Exposure` 1879 Exposure to process. Only the amplifier DataSec is processed. 1880 amp : `lsst.afw.table.AmpInfoCatalog` 1881 Amplifier detector data. 1885 lsst.ip.isr.isrFunctions.makeThresholdMask 1889 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1890 This is intended to indicate pixels that may be affected by unknown systematics; 1891 for example if non-linearity corrections above a certain level are unstable 1892 then that would be a useful value for suspectLevel. A value of `nan` indicates 1893 that no such level exists and no pixels are to be masked as suspicious. 1895 suspectLevel = amp.getSuspectLevel()
1896 if math.isnan(suspectLevel):
1899 maskedImage = exposure.getMaskedImage()
1900 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1901 isrFunctions.makeThresholdMask(
1902 maskedImage=dataView,
1903 threshold=suspectLevel,
1905 maskName=self.config.suspectMaskName,
1909 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1913 ccdExposure : `lsst.afw.image.Exposure` 1914 Exposure to process. 1915 defectBaseList : `List` 1916 List of defects to mask and interpolate. 1920 Call this after CCD assembly, since defects may cross amplifier boundaries. 1922 maskedImage = ccdExposure.getMaskedImage()
1924 for d
in defectBaseList:
1926 nd = measAlg.Defect(bbox)
1927 defectList.append(nd)
1928 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
1929 isrFunctions.interpolateDefectList(
1930 maskedImage=maskedImage,
1931 defectList=defectList,
1932 fwhm=self.config.fwhm,
1935 if self.config.numEdgeSuspect > 0:
1936 goodBBox = maskedImage.getBBox()
1938 goodBBox.grow(-self.config.numEdgeSuspect)
1940 SourceDetectionTask.setEdgeBits(
1943 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1947 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1951 exposure : `lsst.afw.image.Exposure` 1952 Exposure to process. 1956 We mask and interpolate over all NaNs, including those 1957 that are masked with other bits (because those may or may 1958 not be interpolated over later, and we want to remove all 1959 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1960 is used to preserve the historical name. 1962 maskedImage = exposure.getMaskedImage()
1965 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
1966 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
1967 numNans =
maskNans(maskedImage, maskVal)
1968 self.metadata.set(
"NUMNANS", numNans)
1972 self.log.warn(
"There were %i unmasked NaNs", numNans)
1973 nanDefectList = isrFunctions.getDefectListFromMask(
1974 maskedImage=maskedImage,
1975 maskName=
'UNMASKEDNAN',
1977 isrFunctions.interpolateDefectList(
1978 maskedImage=exposure.getMaskedImage(),
1979 defectList=nanDefectList,
1980 fwhm=self.config.fwhm,
1984 """Measure the image background in subgrids, for quality control purposes. 1988 exposure : `lsst.afw.image.Exposure` 1989 Exposure to process. 1990 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 1991 Configuration object containing parameters on which background 1992 statistics and subgrids to use. 1994 if IsrQaConfig
is not None:
1995 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1996 IsrQaConfig.flatness.nIter)
1997 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
1998 statsControl.setAndMask(maskVal)
1999 maskedImage = exposure.getMaskedImage()
2000 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2001 skyLevel = stats.getValue(afwMath.MEDIAN)
2002 skySigma = stats.getValue(afwMath.STDEVCLIP)
2003 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2004 metadata = exposure.getMetadata()
2005 metadata.set(
'SKYLEVEL', skyLevel)
2006 metadata.set(
'SKYSIGMA', skySigma)
2009 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2010 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2011 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2012 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2013 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2014 skyLevels = numpy.zeros((nX, nY))
2017 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2019 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2021 xLLC = xc - meshXHalf
2022 yLLC = yc - meshYHalf
2023 xURC = xc + meshXHalf - 1
2024 yURC = yc + meshYHalf - 1
2026 bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2027 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2029 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2031 good = numpy.where(numpy.isfinite(skyLevels))
2032 skyMedian = numpy.median(skyLevels[good])
2033 flatness = (skyLevels[good] - skyMedian) / skyMedian
2034 flatness_rms = numpy.std(flatness)
2035 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2037 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2038 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
2039 (nX, nY, flatness_pp, flatness_rms))
2041 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2042 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2043 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2044 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2045 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2048 """Set an approximate magnitude zero point for the exposure. 2052 exposure : `lsst.afw.image.Exposure` 2053 Exposure to process. 2055 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2056 if filterName
in self.config.fluxMag0T1:
2057 fluxMag0 = self.config.fluxMag0T1[filterName]
2059 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
2060 fluxMag0 = self.config.defaultFluxMag0T1
2062 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2064 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
2067 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2068 exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2071 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2075 ccdExposure : `lsst.afw.image.Exposure` 2076 Exposure to process. 2077 fpPolygon : `lsst.afw.geom.Polygon` 2078 Polygon in focal plane coordinates. 2081 ccd = ccdExposure.getDetector()
2082 fpCorners = ccd.getCorners(FOCAL_PLANE)
2083 ccdPolygon = Polygon(fpCorners)
2086 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2089 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2090 validPolygon = Polygon(ccdPoints)
2091 ccdExposure.getInfo().setValidPolygon(validPolygon)
2095 """Context manager that applies and removes flats and darks, 2096 if the task is configured to apply them. 2100 exp : `lsst.afw.image.Exposure` 2101 Exposure to process. 2102 flat : `lsst.afw.image.Exposure` 2103 Flat exposure the same size as ``exp``. 2104 dark : `lsst.afw.image.Exposure`, optional 2105 Dark exposure the same size as ``exp``. 2109 exp : `lsst.afw.image.Exposure` 2110 The flat and dark corrected exposure. 2112 if self.config.doDark
and dark
is not None:
2114 if self.config.doFlat:
2119 if self.config.doFlat:
2121 if self.config.doDark
and dark
is not None:
2125 """Utility function to examine ISR exposure at different stages. 2129 exposure : `lsst.afw.image.Exposure` 2132 State of processing to view. 2134 frame = getDebugFrame(self._display, stepname)
2136 display = getDisplay(frame)
2137 display.scale(
'asinh',
'zscale')
2138 display.mtv(exposure)
2142 """A Detector-like object that supports returning gain and saturation level 2144 This is used when the input exposure does not have a detector. 2148 exposure : `lsst.afw.image.Exposure` 2149 Exposure to generate a fake amplifier for. 2150 config : `lsst.ip.isr.isrTaskConfig` 2151 Configuration to apply to the fake amplifier. 2155 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2157 self.
_gain = config.gain
2187 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2191 """Task to wrap the default IsrTask to allow it to be retargeted. 2193 The standard IsrTask can be called directly from a command line 2194 program, but doing so removes the ability of the task to be 2195 retargeted. As most cameras override some set of the IsrTask 2196 methods, this would remove those data-specific methods in the 2197 output post-ISR images. This wrapping class fixes the issue, 2198 allowing identical post-ISR images to be generated by both the 2199 processCcd and isrTask code. 2201 ConfigClass = RunIsrConfig
2202 _DefaultName =
"runIsr" 2206 self.makeSubtask(
"isr")
2212 dataRef : `lsst.daf.persistence.ButlerDataRef` 2213 data reference of the detector data to be processed 2217 result : `pipeBase.Struct` 2218 Result struct with component: 2220 - exposure : `lsst.afw.image.Exposure` 2221 Post-ISR processed exposure. def getInputDatasetTypes(cls, config)
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def getPrerequisiteDatasetTypes(cls, config)
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def getOutputDatasetTypes(cls, config)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def makeDatasetType(self, dsConfig)
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def getPerDatasetTypeDimensions(cls, config)
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)