32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
47 from .assembleCcdTask
import AssembleCcdTask
48 from .crosstalk
import CrosstalkTask
49 from .fringe
import FringeTask
50 from .isr
import maskNans
51 from .masking
import MaskingTask
52 from .straylight
import StrayLightTask
53 from .vignette
import VignetteTask
55 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
59 """Configuration parameters for IsrTask. 61 Items are grouped in the order in which they are executed by the task. 66 isrName = pexConfig.Field(
73 ccdExposure = pipeBase.InputDatasetField(
74 doc=
"Input exposure to process",
77 storageClass=
"Exposure",
78 dimensions=[
"instrument",
"exposure",
"detector"],
80 camera = pipeBase.InputDatasetField(
81 doc=
"Input camera to construct complete exposures.",
84 storageClass=
"TablePersistableCamera",
85 dimensions=[
"instrument",
"calibration_label"],
87 bias = pipeBase.InputDatasetField(
88 doc=
"Input bias calibration.",
91 storageClass=
"ImageF",
92 dimensions=[
"instrument",
"calibration_label",
"detector"],
94 dark = pipeBase.InputDatasetField(
95 doc=
"Input dark calibration.",
98 storageClass=
"ImageF",
99 dimensions=[
"instrument",
"calibration_label",
"detector"],
101 flat = pipeBase.InputDatasetField(
102 doc=
"Input flat calibration.",
105 storageClass=
"MaskedImageF",
106 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
108 bfKernel = pipeBase.InputDatasetField(
109 doc=
"Input brighter-fatter kernel.",
112 storageClass=
"NumpyArray",
113 dimensions=[
"instrument",
"calibration_label"],
115 defects = pipeBase.InputDatasetField(
116 doc=
"Input defect tables.",
119 storageClass=
"DefectsList",
120 dimensions=[
"instrument",
"calibration_label",
"detector"],
122 opticsTransmission = pipeBase.InputDatasetField(
123 doc=
"Transmission curve due to the optics.",
124 name=
"transmission_optics",
126 storageClass=
"TablePersistableTransmissionCurve",
127 dimensions=[
"instrument",
"calibration_label"],
129 filterTransmission = pipeBase.InputDatasetField(
130 doc=
"Transmission curve due to the filter.",
131 name=
"transmission_filter",
133 storageClass=
"TablePersistableTransmissionCurve",
134 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
136 sensorTransmission = pipeBase.InputDatasetField(
137 doc=
"Transmission curve due to the sensor.",
138 name=
"transmission_sensor",
140 storageClass=
"TablePersistableTransmissionCurve",
141 dimensions=[
"instrument",
"calibration_label",
"detector"],
143 atmosphereTransmission = pipeBase.InputDatasetField(
144 doc=
"Transmission curve due to the atmosphere.",
145 name=
"transmission_atmosphere",
147 storageClass=
"TablePersistableTransmissionCurve",
148 dimensions=[
"instrument"],
152 outputExposure = pipeBase.OutputDatasetField(
153 doc=
"Output ISR processed exposure.",
156 storageClass=
"ExposureF",
157 dimensions=[
"instrument",
"visit",
"detector"],
159 outputOssThumbnail = pipeBase.OutputDatasetField(
160 doc=
"Output Overscan-subtracted thumbnail image.",
163 storageClass=
"Thumbnail",
164 dimensions=[
"instrument",
"visit",
"detector"],
166 outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167 doc=
"Output flat-corrected thumbnail image.",
168 name=
"FlattenedThumb",
170 storageClass=
"TextStorage",
171 dimensions=[
"instrument",
"visit",
"detector"],
174 quantum = pipeBase.QuantumConfig(
175 dimensions=[
"visit",
"detector",
"instrument"],
179 datasetType = pexConfig.Field(
181 doc=
"Dataset type for input data; users will typically leave this alone, " 182 "but camera-specific ISR tasks will override it",
186 fallbackFilterName = pexConfig.Field(
188 doc=
"Fallback default filter name for calibrations.",
191 expectWcs = pexConfig.Field(
194 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 196 fwhm = pexConfig.Field(
198 doc=
"FWHM of PSF in arcseconds.",
201 qa = pexConfig.ConfigField(
203 doc=
"QA related configuration options.",
207 doConvertIntToFloat = pexConfig.Field(
209 doc=
"Convert integer raw images to floating point values?",
214 doSaturation = pexConfig.Field(
216 doc=
"Mask saturated pixels? NB: this is totally independent of the" 217 " interpolation option - this is ONLY setting the bits in the mask." 218 " To have them interpolated make sure doSaturationInterpolation=True",
221 saturatedMaskName = pexConfig.Field(
223 doc=
"Name of mask plane to use in saturation detection and interpolation",
226 saturation = pexConfig.Field(
228 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229 default=float(
"NaN"),
231 growSaturationFootprintSize = pexConfig.Field(
233 doc=
"Number of pixels by which to grow the saturation footprints",
238 doSuspect = pexConfig.Field(
240 doc=
"Mask suspect pixels?",
243 suspectMaskName = pexConfig.Field(
245 doc=
"Name of mask plane to use for suspect pixels",
248 numEdgeSuspect = pexConfig.Field(
250 doc=
"Number of edge pixels to be flagged as untrustworthy.",
255 doSetBadRegions = pexConfig.Field(
257 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
260 badStatistic = pexConfig.ChoiceField(
262 doc=
"How to estimate the average value for BAD regions.",
265 "MEANCLIP":
"Correct using the (clipped) mean of good data",
266 "MEDIAN":
"Correct using the median of the good data",
271 doOverscan = pexConfig.Field(
273 doc=
"Do overscan subtraction?",
276 overscanFitType = pexConfig.ChoiceField(
278 doc=
"The method for fitting the overscan bias level.",
281 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
282 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
283 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
284 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
285 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
286 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
287 "MEAN":
"Correct using the mean of the overscan region",
288 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
289 "MEDIAN":
"Correct using the median of the overscan region",
292 overscanOrder = pexConfig.Field(
294 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
295 "or number of spline knots if overscan fit type is a spline."),
298 overscanNumSigmaClip = pexConfig.Field(
300 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
303 overscanIsInt = pexConfig.Field(
305 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
308 overscanNumLeadingColumnsToSkip = pexConfig.Field(
310 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
313 overscanNumTrailingColumnsToSkip = pexConfig.Field(
315 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
318 overscanMaxDev = pexConfig.Field(
320 doc=
"Maximum deviation from the median for overscan",
321 default=1000.0, check=
lambda x: x > 0
323 overscanBiasJump = pexConfig.Field(
325 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
328 overscanBiasJumpKeyword = pexConfig.Field(
330 doc=
"Header keyword containing information about devices.",
331 default=
"NO_SUCH_KEY",
333 overscanBiasJumpDevices = pexConfig.ListField(
335 doc=
"List of devices that need piecewise overscan correction.",
338 overscanBiasJumpLocation = pexConfig.Field(
340 doc=
"Location of bias jump along y-axis.",
345 doAssembleCcd = pexConfig.Field(
348 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 350 assembleCcd = pexConfig.ConfigurableField(
351 target=AssembleCcdTask,
352 doc=
"CCD assembly task",
356 doAssembleIsrExposures = pexConfig.Field(
359 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 361 doTrimToMatchCalib = pexConfig.Field(
364 doc=
"Trim raw data to match calibration bounding boxes?" 368 doBias = pexConfig.Field(
370 doc=
"Apply bias frame correction?",
373 biasDataProductName = pexConfig.Field(
375 doc=
"Name of the bias data product",
380 doVariance = pexConfig.Field(
382 doc=
"Calculate variance?",
385 gain = pexConfig.Field(
387 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388 default=float(
"NaN"),
390 readNoise = pexConfig.Field(
392 doc=
"The read noise to use if no Detector is present in the Exposure",
395 doEmpiricalReadNoise = pexConfig.Field(
398 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 402 doLinearize = pexConfig.Field(
404 doc=
"Correct for nonlinearity of the detector's response?",
409 doCrosstalk = pexConfig.Field(
411 doc=
"Apply intra-CCD crosstalk correction?",
414 doCrosstalkBeforeAssemble = pexConfig.Field(
416 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
419 crosstalk = pexConfig.ConfigurableField(
420 target=CrosstalkTask,
421 doc=
"Intra-CCD crosstalk correction",
425 doWidenSaturationTrails = pexConfig.Field(
427 doc=
"Widen bleed trails based on their width?",
432 doBrighterFatter = pexConfig.Field(
435 doc=
"Apply the brighter fatter correction" 437 brighterFatterLevel = pexConfig.ChoiceField(
440 doc=
"The level at which to correct for brighter-fatter.",
442 "AMP":
"Every amplifier treated separately.",
443 "DETECTOR":
"One kernel per detector",
446 brighterFatterKernelFile = pexConfig.Field(
449 doc=
"Kernel file used for the brighter fatter correction" 451 brighterFatterMaxIter = pexConfig.Field(
454 doc=
"Maximum number of iterations for the brighter fatter correction" 456 brighterFatterThreshold = pexConfig.Field(
459 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 460 " absolute value of the difference between the current corrected image and the one" 461 " from the previous iteration summed over all the pixels." 463 brighterFatterApplyGain = pexConfig.Field(
466 doc=
"Should the gain be applied when applying the brighter fatter correction?" 470 doDefect = pexConfig.Field(
472 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
475 doSaturationInterpolation = pexConfig.Field(
477 doc=
"Perform interpolation over pixels masked as saturated?" 478 " NB: This is independent of doSaturation; if that is False this plane" 479 " will likely be blank, resulting in a no-op here.",
482 numEdgeSuspect = pexConfig.Field(
484 doc=
"Number of edge pixels to be flagged as untrustworthy.",
489 doDark = pexConfig.Field(
491 doc=
"Apply dark frame correction?",
494 darkDataProductName = pexConfig.Field(
496 doc=
"Name of the dark data product",
501 doStrayLight = pexConfig.Field(
503 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
506 strayLight = pexConfig.ConfigurableField(
507 target=StrayLightTask,
508 doc=
"y-band stray light correction" 512 doFlat = pexConfig.Field(
514 doc=
"Apply flat field correction?",
517 flatDataProductName = pexConfig.Field(
519 doc=
"Name of the flat data product",
522 flatScalingType = pexConfig.ChoiceField(
524 doc=
"The method for scaling the flat on the fly.",
527 "USER":
"Scale by flatUserScale",
528 "MEAN":
"Scale by the inverse of the mean",
529 "MEDIAN":
"Scale by the inverse of the median",
532 flatUserScale = pexConfig.Field(
534 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
537 doTweakFlat = pexConfig.Field(
539 doc=
"Tweak flats to match observed amplifier ratios?",
544 doApplyGains = pexConfig.Field(
546 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
549 normalizeGains = pexConfig.Field(
551 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
556 doFringe = pexConfig.Field(
558 doc=
"Apply fringe correction?",
561 fringe = pexConfig.ConfigurableField(
563 doc=
"Fringe subtraction task",
565 fringeAfterFlat = pexConfig.Field(
567 doc=
"Do fringe subtraction after flat-fielding?",
572 doNanInterpAfterFlat = pexConfig.Field(
574 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 575 "also have to interpolate them before flat-fielding."),
580 doAddDistortionModel = pexConfig.Field(
582 doc=
"Apply a distortion model based on camera geometry to the WCS?",
587 doMeasureBackground = pexConfig.Field(
589 doc=
"Measure the background level on the reduced image?",
594 doCameraSpecificMasking = pexConfig.Field(
596 doc=
"Mask camera-specific bad regions?",
599 masking = pexConfig.ConfigurableField(
605 fluxMag0T1 = pexConfig.DictField(
608 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
612 defaultFluxMag0T1 = pexConfig.Field(
614 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
615 default=pow(10.0, 0.4*28.0)
619 doVignette = pexConfig.Field(
621 doc=
"Apply vignetting parameters?",
624 vignette = pexConfig.ConfigurableField(
626 doc=
"Vignetting task.",
630 doAttachTransmissionCurve = pexConfig.Field(
633 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 635 doUseOpticsTransmission = pexConfig.Field(
638 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 640 doUseFilterTransmission = pexConfig.Field(
643 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 645 doUseSensorTransmission = pexConfig.Field(
648 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 650 doUseAtmosphereTransmission = pexConfig.Field(
653 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 657 doWrite = pexConfig.Field(
659 doc=
"Persist postISRCCD?",
666 raise ValueError(
"You may not specify both doFlat and doApplyGains")
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670 """Apply common instrument signature correction algorithms to a raw frame. 672 The process for correcting imaging data is very similar from 673 camera to camera. This task provides a vanilla implementation of 674 doing these corrections, including the ability to turn certain 675 corrections off if they are not needed. The inputs to the primary 676 method, `run()`, are a raw exposure to be corrected and the 677 calibration data products. The raw input is a single chip sized 678 mosaic of all amps including overscans and other non-science 679 pixels. The method `runDataRef()` identifies and defines the 680 calibration data products, and is intended for use by a 681 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 682 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 683 subclassed for different camera, although the most camera specific 684 methods have been split into subtasks that can be redirected 687 The __init__ method sets up the subtasks for ISR processing, using 688 the defaults from `lsst.ip.isr`. 693 Positional arguments passed to the Task constructor. None used at this time. 694 kwargs : `dict`, optional 695 Keyword arguments passed on to the Task constructor. None used at this time. 697 ConfigClass = IsrTaskConfig
702 self.makeSubtask(
"assembleCcd")
703 self.makeSubtask(
"crosstalk")
704 self.makeSubtask(
"strayLight")
705 self.makeSubtask(
"fringe")
706 self.makeSubtask(
"masking")
707 self.makeSubtask(
"vignette")
716 if config.doBias
is not True:
717 inputTypeDict.pop(
"bias",
None)
718 if config.doLinearize
is not True:
719 inputTypeDict.pop(
"linearizer",
None)
720 if config.doCrosstalk
is not True:
721 inputTypeDict.pop(
"crosstalkSources",
None)
722 if config.doBrighterFatter
is not True:
723 inputTypeDict.pop(
"bfKernel",
None)
724 if config.doDefect
is not True:
725 inputTypeDict.pop(
"defects",
None)
726 if config.doDark
is not True:
727 inputTypeDict.pop(
"dark",
None)
728 if config.doFlat
is not True:
729 inputTypeDict.pop(
"flat",
None)
730 if config.doAttachTransmissionCurve
is not True:
731 inputTypeDict.pop(
"opticsTransmission",
None)
732 inputTypeDict.pop(
"filterTransmission",
None)
733 inputTypeDict.pop(
"sensorTransmission",
None)
734 inputTypeDict.pop(
"atmosphereTransmission",
None)
735 if config.doUseOpticsTransmission
is not True:
736 inputTypeDict.pop(
"opticsTransmission",
None)
737 if config.doUseFilterTransmission
is not True:
738 inputTypeDict.pop(
"filterTransmission",
None)
739 if config.doUseSensorTransmission
is not True:
740 inputTypeDict.pop(
"sensorTransmission",
None)
741 if config.doUseAtmosphereTransmission
is not True:
742 inputTypeDict.pop(
"atmosphereTransmission",
None)
750 if config.qa.doThumbnailOss
is not True:
751 outputTypeDict.pop(
"outputOssThumbnail",
None)
752 if config.qa.doThumbnailFlattened
is not True:
753 outputTypeDict.pop(
"outputFlattenedThumbnail",
None)
754 if config.doWrite
is not True:
755 outputTypeDict.pop(
"outputExposure",
None)
757 return outputTypeDict
766 names.remove(
"ccdExposure")
775 return frozenset([
"calibration_label"])
779 inputData[
'detectorNum'] = int(inputDataIds[
'ccdExposure'][
'detector'])
780 except Exception
as e:
781 raise ValueError(f
"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
783 inputData[
'isGen3'] =
True 785 if self.config.doLinearize
is True:
786 if 'linearizer' not in inputData.keys():
787 detector = inputData[
'camera'][inputData[
'detectorNum']]
788 linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
789 inputData[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
791 if inputData[
'defects']
is not None:
794 if not isinstance(inputData[
"defects"], Defects):
795 inputData[
"defects"] = Defects.fromTable(inputData[
"defects"])
812 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
818 """!Retrieve necessary frames for instrument signature removal. 820 Pre-fetching all required ISR data products limits the IO 821 required by the ISR. Any conflict between the calibration data 822 available and that needed for ISR is also detected prior to 823 doing processing, allowing it to fail quickly. 827 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 828 Butler reference of the detector data to be processed 829 rawExposure : `afw.image.Exposure` 830 The raw exposure that will later be corrected with the 831 retrieved calibration data; should not be modified in this 836 result : `lsst.pipe.base.Struct` 837 Result struct with components (which may be `None`): 838 - ``bias``: bias calibration frame (`afw.image.Exposure`) 839 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 840 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 841 - ``dark``: dark calibration frame (`afw.image.Exposure`) 842 - ``flat``: flat calibration frame (`afw.image.Exposure`) 843 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 844 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`) 845 - ``fringes``: `lsst.pipe.base.Struct` with components: 846 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 847 - ``seed``: random seed derived from the ccdExposureId for random 848 number generator (`uint32`) 849 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 850 A ``TransmissionCurve`` that represents the throughput of the optics, 851 to be evaluated in focal-plane coordinates. 852 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 853 A ``TransmissionCurve`` that represents the throughput of the filter 854 itself, to be evaluated in focal-plane coordinates. 855 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 856 A ``TransmissionCurve`` that represents the throughput of the sensor 857 itself, to be evaluated in post-assembly trimmed detector coordinates. 858 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 859 A ``TransmissionCurve`` that represents the throughput of the 860 atmosphere, assumed to be spatially constant. 861 - ``strayLightData`` : `object` 862 An opaque object containing calibration information for 863 stray-light correction. If `None`, no correction will be 868 NotImplementedError : 869 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 871 ccd = rawExposure.getDetector()
872 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
873 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
874 if self.config.doBias
else None)
876 linearizer = (dataRef.get(
"linearizer", immediate=
True)
878 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
879 if self.config.doCrosstalk
else None)
880 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
881 if self.config.doDark
else None)
882 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
883 if self.config.doFlat
else None)
885 brighterFatterKernel =
None 886 if self.config.doBrighterFatter
is True:
890 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
894 brighterFatterKernel = dataRef.get(
"bfKernel")
896 brighterFatterKernel =
None 897 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
900 if self.config.brighterFatterLevel ==
'DETECTOR':
901 brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
904 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
906 defectList = (dataRef.get(
"defects")
907 if self.config.doDefect
else None)
908 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
909 if self.config.doAssembleIsrExposures
else None)
910 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
911 else pipeBase.Struct(fringes=
None))
913 if self.config.doAttachTransmissionCurve:
914 opticsTransmission = (dataRef.get(
"transmission_optics")
915 if self.config.doUseOpticsTransmission
else None)
916 filterTransmission = (dataRef.get(
"transmission_filter")
917 if self.config.doUseFilterTransmission
else None)
918 sensorTransmission = (dataRef.get(
"transmission_sensor")
919 if self.config.doUseSensorTransmission
else None)
920 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
921 if self.config.doUseAtmosphereTransmission
else None)
923 opticsTransmission =
None 924 filterTransmission =
None 925 sensorTransmission =
None 926 atmosphereTransmission =
None 928 if self.config.doStrayLight:
929 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
931 strayLightData =
None 934 return pipeBase.Struct(bias=biasExposure,
935 linearizer=linearizer,
936 crosstalkSources=crosstalkSources,
939 bfKernel=brighterFatterKernel,
941 fringes=fringeStruct,
942 opticsTransmission=opticsTransmission,
943 filterTransmission=filterTransmission,
944 sensorTransmission=sensorTransmission,
945 atmosphereTransmission=atmosphereTransmission,
946 strayLightData=strayLightData
950 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
951 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
952 opticsTransmission=None, filterTransmission=None,
953 sensorTransmission=None, atmosphereTransmission=None,
954 detectorNum=None, strayLightData=None, isGen3=False,
956 """!Perform instrument signature removal on an exposure. 958 Steps included in the ISR processing, in order performed, are: 959 - saturation and suspect pixel masking 960 - overscan subtraction 961 - CCD assembly of individual amplifiers 963 - variance image construction 964 - linearization of non-linear response 966 - brighter-fatter correction 969 - stray light subtraction 971 - masking of known defects and camera specific features 972 - vignette calculation 973 - appending transmission curve and distortion model 977 ccdExposure : `lsst.afw.image.Exposure` 978 The raw exposure that is to be run through ISR. The 979 exposure is modified by this method. 980 camera : `lsst.afw.cameraGeom.Camera`, optional 981 The camera geometry for this exposure. Used to select the 982 distortion model appropriate for this data. 983 bias : `lsst.afw.image.Exposure`, optional 984 Bias calibration frame. 985 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 986 Functor for linearization. 987 crosstalkSources : `list`, optional 988 List of possible crosstalk sources. 989 dark : `lsst.afw.image.Exposure`, optional 990 Dark calibration frame. 991 flat : `lsst.afw.image.Exposure`, optional 992 Flat calibration frame. 993 bfKernel : `numpy.ndarray`, optional 994 Brighter-fatter kernel. 995 defects : `lsst.meas.algorithms.Defects`, optional 997 fringes : `lsst.pipe.base.Struct`, optional 998 Struct containing the fringe correction data, with 1000 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 1001 - ``seed``: random seed derived from the ccdExposureId for random 1002 number generator (`uint32`) 1003 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 1004 A ``TransmissionCurve`` that represents the throughput of the optics, 1005 to be evaluated in focal-plane coordinates. 1006 filterTransmission : `lsst.afw.image.TransmissionCurve` 1007 A ``TransmissionCurve`` that represents the throughput of the filter 1008 itself, to be evaluated in focal-plane coordinates. 1009 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1010 A ``TransmissionCurve`` that represents the throughput of the sensor 1011 itself, to be evaluated in post-assembly trimmed detector coordinates. 1012 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1013 A ``TransmissionCurve`` that represents the throughput of the 1014 atmosphere, assumed to be spatially constant. 1015 detectorNum : `int`, optional 1016 The integer number for the detector to process. 1017 isGen3 : bool, optional 1018 Flag this call to run() as using the Gen3 butler environment. 1019 strayLightData : `object`, optional 1020 Opaque object containing calibration information for stray-light 1021 correction. If `None`, no correction will be performed. 1025 result : `lsst.pipe.base.Struct` 1026 Result struct with component: 1027 - ``exposure`` : `afw.image.Exposure` 1028 The fully ISR corrected exposure. 1029 - ``outputExposure`` : `afw.image.Exposure` 1030 An alias for `exposure` 1031 - ``ossThumb`` : `numpy.ndarray` 1032 Thumbnail image of the exposure after overscan subtraction. 1033 - ``flattenedThumb`` : `numpy.ndarray` 1034 Thumbnail image of the exposure after flat-field correction. 1039 Raised if a configuration option is set to True, but the 1040 required calibration data has not been specified. 1044 The current processed exposure can be viewed by setting the 1045 appropriate lsstDebug entries in the `debug.display` 1046 dictionary. The names of these entries correspond to some of 1047 the IsrTaskConfig Boolean options, with the value denoting the 1048 frame to use. The exposure is shown inside the matching 1049 option check and after the processing of that step has 1050 finished. The steps with debug points are: 1061 In addition, setting the "postISRCCD" entry displays the 1062 exposure after all ISR processing has finished. 1070 self.config.doFringe =
False 1073 if detectorNum
is None:
1074 raise RuntimeError(
"Must supply the detectorNum if running as Gen3")
1076 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1081 if isinstance(ccdExposure, ButlerDataRef):
1084 ccd = ccdExposure.getDetector()
1087 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 1088 ccd = [
FakeAmp(ccdExposure, self.config)]
1091 if self.config.doBias
and bias
is None:
1092 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1094 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1095 if self.config.doBrighterFatter
and bfKernel
is None:
1096 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1097 if self.config.doDark
and dark
is None:
1098 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1100 fringes = pipeBase.Struct(fringes=
None)
1101 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
1102 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1103 if self.config.doFlat
and flat
is None:
1104 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1105 if self.config.doDefect
and defects
is None:
1106 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1107 if self.config.doAddDistortionModel
and camera
is None:
1108 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
1111 if self.config.doConvertIntToFloat:
1112 self.log.info(
"Converting exposure to floating point values")
1119 if ccdExposure.getBBox().contains(amp.getBBox()):
1123 if self.config.doOverscan
and not badAmp:
1126 self.log.debug(
"Corrected overscan for amplifier %s" % (amp.getName()))
1127 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1128 if isinstance(overscanResults.overscanFit, float):
1129 qaMedian = overscanResults.overscanFit
1130 qaStdev = float(
"NaN")
1132 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1133 afwMath.MEDIAN | afwMath.STDEVCLIP)
1134 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1135 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1137 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1138 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1139 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f" %
1140 (amp.getName(), qaMedian, qaStdev))
1141 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1144 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
1145 overscanResults =
None 1147 overscans.append(overscanResults
if overscanResults
is not None else None)
1149 self.log.info(
"Skipped OSCAN")
1151 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1152 self.log.info(
"Applying crosstalk correction.")
1153 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1154 self.
debugView(ccdExposure,
"doCrosstalk")
1156 if self.config.doAssembleCcd:
1157 self.log.info(
"Assembling CCD from amplifiers")
1158 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1160 if self.config.expectWcs
and not ccdExposure.getWcs():
1161 self.log.warn(
"No WCS found in input exposure")
1162 self.
debugView(ccdExposure,
"doAssembleCcd")
1165 if self.config.qa.doThumbnailOss:
1166 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1168 if self.config.doBias:
1169 self.log.info(
"Applying bias correction.")
1170 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1171 trimToFit=self.config.doTrimToMatchCalib)
1174 if self.config.doVariance:
1175 for amp, overscanResults
in zip(ccd, overscans):
1176 if ccdExposure.getBBox().contains(amp.getBBox()):
1177 self.log.debug(
"Constructing variance map for amplifer %s" % (amp.getName()))
1178 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1179 if overscanResults
is not None:
1181 overscanImage=overscanResults.overscanImage)
1185 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1186 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1187 afwMath.MEDIAN | afwMath.STDEVCLIP)
1188 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
1189 qaStats.getValue(afwMath.MEDIAN))
1190 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
1191 qaStats.getValue(afwMath.STDEVCLIP))
1192 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f" %
1193 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1194 qaStats.getValue(afwMath.STDEVCLIP)))
1197 self.log.info(
"Applying linearizer.")
1198 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1200 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1201 self.log.info(
"Applying crosstalk correction.")
1202 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=
True)
1203 self.
debugView(ccdExposure,
"doCrosstalk")
1205 if self.config.doWidenSaturationTrails:
1206 self.log.info(
"Widening saturation trails.")
1207 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1209 interpolationDone =
False 1210 if self.config.doBrighterFatter:
1216 if self.config.doDefect:
1219 if self.config.doSaturationInterpolation:
1223 interpolationDone =
True 1225 self.log.info(
"Applying brighter fatter correction.")
1226 isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1227 self.config.brighterFatterMaxIter,
1228 self.config.brighterFatterThreshold,
1229 self.config.brighterFatterApplyGain,
1231 self.
debugView(ccdExposure,
"doBrighterFatter")
1233 if self.config.doDark:
1234 self.log.info(
"Applying dark correction.")
1238 if self.config.doFringe
and not self.config.fringeAfterFlat:
1239 self.log.info(
"Applying fringe correction before flat.")
1240 self.fringe.
run(ccdExposure, **fringes.getDict())
1243 if self.config.doStrayLight:
1244 if strayLightData
is not None:
1245 self.log.info(
"Applying stray light correction.")
1246 self.strayLight.
run(ccdExposure, strayLightData)
1247 self.
debugView(ccdExposure,
"doStrayLight")
1249 self.log.debug(
"Skipping stray light correction: no data found for this image.")
1251 if self.config.doFlat:
1252 self.log.info(
"Applying flat correction.")
1256 if self.config.doApplyGains:
1257 self.log.info(
"Applying gain correction instead of flat.")
1258 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1260 if self.config.doDefect
and not interpolationDone:
1261 self.log.info(
"Masking and interpolating defects.")
1264 if self.config.doSaturationInterpolation
and not interpolationDone:
1265 self.log.info(
"Interpolating saturated pixels.")
1268 if self.config.doNanInterpAfterFlat
or not interpolationDone:
1269 self.log.info(
"Masking and interpolating NAN value pixels.")
1272 if self.config.doFringe
and self.config.fringeAfterFlat:
1273 self.log.info(
"Applying fringe correction after flat.")
1274 self.fringe.
run(ccdExposure, **fringes.getDict())
1276 if self.config.doSetBadRegions:
1277 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1278 if badPixelCount > 0:
1279 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1281 flattenedThumb =
None 1282 if self.config.qa.doThumbnailFlattened:
1283 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1285 if self.config.doCameraSpecificMasking:
1286 self.log.info(
"Masking regions for camera specific reasons.")
1287 self.masking.
run(ccdExposure)
1291 if self.config.doVignette:
1292 self.log.info(
"Constructing Vignette polygon.")
1295 if self.config.vignette.doWriteVignettePolygon:
1298 if self.config.doAttachTransmissionCurve:
1299 self.log.info(
"Adding transmission curves.")
1300 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1301 filterTransmission=filterTransmission,
1302 sensorTransmission=sensorTransmission,
1303 atmosphereTransmission=atmosphereTransmission)
1305 if self.config.doAddDistortionModel:
1306 self.log.info(
"Adding a distortion model to the WCS.")
1307 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1309 if self.config.doMeasureBackground:
1310 self.log.info(
"Measuring background level:")
1313 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1315 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1316 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1317 afwMath.MEDIAN | afwMath.STDEVCLIP)
1318 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1319 qaStats.getValue(afwMath.MEDIAN))
1320 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1321 qaStats.getValue(afwMath.STDEVCLIP))
1322 self.log.debug(
" Background stats for amplifer %s: %f +/- %f" %
1323 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1324 qaStats.getValue(afwMath.STDEVCLIP)))
1326 self.
debugView(ccdExposure,
"postISRCCD")
1328 return pipeBase.Struct(
1329 exposure=ccdExposure,
1331 flattenedThumb=flattenedThumb,
1333 outputExposure=ccdExposure,
1334 outputOssThumbnail=ossThumb,
1335 outputFlattenedThumbnail=flattenedThumb,
1338 @pipeBase.timeMethod
1340 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1342 This method contains the `CmdLineTask` interface to the ISR 1343 processing. All IO is handled here, freeing the `run()` method 1344 to manage only pixel-level calculations. The steps performed 1346 - Read in necessary detrending/isr/calibration data. 1347 - Process raw exposure in `run()`. 1348 - Persist the ISR-corrected exposure as "postISRCCD" if 1349 config.doWrite=True. 1353 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1354 DataRef of the detector data to be processed 1358 result : `lsst.pipe.base.Struct` 1359 Result struct with component: 1360 - ``exposure`` : `afw.image.Exposure` 1361 The fully ISR corrected exposure. 1366 Raised if a configuration option is set to True, but the 1367 required calibration data does not exist. 1370 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1372 ccdExposure = sensorRef.get(self.config.datasetType)
1374 camera = sensorRef.get(
"camera")
1375 if camera
is None and self.config.doAddDistortionModel:
1376 raise RuntimeError(
"config.doAddDistortionModel is True " 1377 "but could not get a camera from the butler")
1378 isrData = self.
readIsrData(sensorRef, ccdExposure)
1380 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1382 if self.config.doWrite:
1383 sensorRef.put(result.exposure,
"postISRCCD")
1384 if result.ossThumb
is not None:
1385 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1386 if result.flattenedThumb
is not None:
1387 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1392 """!Retrieve a calibration dataset for removing instrument signature. 1397 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1398 DataRef of the detector data to find calibration datasets 1401 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1403 If True, disable butler proxies to enable error handling 1404 within this routine. 1408 exposure : `lsst.afw.image.Exposure` 1409 Requested calibration frame. 1414 Raised if no matching calibration frame can be found. 1417 exp = dataRef.get(datasetType, immediate=immediate)
1418 except Exception
as exc1:
1419 if not self.config.fallbackFilterName:
1420 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1422 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1423 except Exception
as exc2:
1424 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1425 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1426 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1428 if self.config.doAssembleIsrExposures:
1429 exp = self.assembleCcd.assembleCcd(exp)
1433 """Ensure that the data returned by Butler is a fully constructed exposure. 1435 ISR requires exposure-level image data for historical reasons, so if we did 1436 not recieve that from Butler, construct it from what we have, modifying the 1441 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1442 `lsst.afw.image.ImageF` 1443 The input data structure obtained from Butler. 1444 camera : `lsst.afw.cameraGeom.camera` 1445 The camera associated with the image. Used to find the appropriate 1448 The detector this exposure should match. 1452 inputExp : `lsst.afw.image.Exposure` 1453 The re-constructed exposure, with appropriate detector parameters. 1458 Raised if the input data cannot be used to construct an exposure. 1460 if isinstance(inputExp, afwImage.DecoratedImageU):
1461 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1462 elif isinstance(inputExp, afwImage.ImageF):
1463 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1464 elif isinstance(inputExp, afwImage.MaskedImageF):
1465 inputExp = afwImage.makeExposure(inputExp)
1466 elif isinstance(inputExp, afwImage.Exposure):
1468 elif inputExp
is None:
1472 raise TypeError(f
"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1474 if inputExp.getDetector()
is None:
1475 inputExp.setDetector(camera[detectorNum])
1480 """Convert exposure image from uint16 to float. 1482 If the exposure does not need to be converted, the input is 1483 immediately returned. For exposures that are converted to use 1484 floating point pixels, the variance is set to unity and the 1489 exposure : `lsst.afw.image.Exposure` 1490 The raw exposure to be converted. 1494 newexposure : `lsst.afw.image.Exposure` 1495 The input ``exposure``, converted to floating point pixels. 1500 Raised if the exposure type cannot be converted to float. 1503 if isinstance(exposure, afwImage.ExposureF):
1506 if not hasattr(exposure,
"convertF"):
1507 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1509 newexposure = exposure.convertF()
1510 newexposure.variance[:] = 1
1511 newexposure.mask[:] = 0x0
1516 """Identify bad amplifiers, saturated and suspect pixels. 1520 ccdExposure : `lsst.afw.image.Exposure` 1521 Input exposure to be masked. 1522 amp : `lsst.afw.table.AmpInfoCatalog` 1523 Catalog of parameters defining the amplifier on this 1525 defects : `lsst.meas.algorithms.Defects` 1526 List of defects. Used to determine if the entire 1532 If this is true, the entire amplifier area is covered by 1533 defects and unusable. 1536 maskedImage = ccdExposure.getMaskedImage()
1542 if defects
is not None:
1543 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1548 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1550 maskView = dataView.getMask()
1551 maskView |= maskView.getPlaneBitMask(
"BAD")
1558 if self.config.doSaturation
and not badAmp:
1559 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1560 if self.config.doSuspect
and not badAmp:
1561 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1562 if math.isfinite(self.config.saturation):
1563 limits.update({self.config.saturatedMaskName: self.config.saturation})
1565 for maskName, maskThreshold
in limits.items():
1566 if not math.isnan(maskThreshold):
1567 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1568 isrFunctions.makeThresholdMask(
1569 maskedImage=dataView,
1570 threshold=maskThreshold,
1576 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1578 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1579 self.config.suspectMaskName])
1580 if numpy.all(maskView.getArray() & maskVal > 0):
1586 """Apply overscan correction in place. 1588 This method does initial pixel rejection of the overscan 1589 region. The overscan can also be optionally segmented to 1590 allow for discontinuous overscan responses to be fit 1591 separately. The actual overscan subtraction is performed by 1592 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1593 which is called here after the amplifier is preprocessed. 1597 ccdExposure : `lsst.afw.image.Exposure` 1598 Exposure to have overscan correction performed. 1599 amp : `lsst.afw.table.AmpInfoCatalog` 1600 The amplifier to consider while correcting the overscan. 1604 overscanResults : `lsst.pipe.base.Struct` 1605 Result struct with components: 1606 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1607 Value or fit subtracted from the amplifier image data. 1608 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1609 Value or fit subtracted from the overscan image data. 1610 - ``overscanImage`` : `lsst.afw.image.Image` 1611 Image of the overscan region with the overscan 1612 correction applied. This quantity is used to estimate 1613 the amplifier read noise empirically. 1618 Raised if the ``amp`` does not contain raw pixel information. 1622 lsst.ip.isr.isrFunctions.overscanCorrection 1624 if not amp.getHasRawInfo():
1625 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1627 if amp.getRawHorizontalOverscanBBox().isEmpty():
1628 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1631 statControl = afwMath.StatisticsControl()
1632 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1635 dataBBox = amp.getRawDataBBox()
1636 oscanBBox = amp.getRawHorizontalOverscanBBox()
1640 prescanBBox = amp.getRawPrescanBBox()
1641 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1642 dx0 += self.config.overscanNumLeadingColumnsToSkip
1643 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1645 dx0 += self.config.overscanNumTrailingColumnsToSkip
1646 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1652 if ((self.config.overscanBiasJump
and 1653 self.config.overscanBiasJumpLocation)
and 1654 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1655 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1656 self.config.overscanBiasJumpDevices)):
1657 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1658 yLower = self.config.overscanBiasJumpLocation
1659 yUpper = dataBBox.getHeight() - yLower
1661 yUpper = self.config.overscanBiasJumpLocation
1662 yLower = dataBBox.getHeight() - yUpper
1681 oscanBBox.getHeight())))
1684 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1685 ampImage = ccdExposure.maskedImage[imageBBox]
1686 overscanImage = ccdExposure.maskedImage[overscanBBox]
1688 overscanArray = overscanImage.image.array
1689 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1690 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1691 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1693 statControl = afwMath.StatisticsControl()
1694 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1696 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1697 overscanImage=overscanImage,
1698 fitType=self.config.overscanFitType,
1699 order=self.config.overscanOrder,
1700 collapseRej=self.config.overscanNumSigmaClip,
1701 statControl=statControl,
1702 overscanIsInt=self.config.overscanIsInt
1706 levelStat = afwMath.MEDIAN
1707 sigmaStat = afwMath.STDEVCLIP
1709 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1710 self.config.qa.flatness.nIter)
1711 metadata = ccdExposure.getMetadata()
1712 ampNum = amp.getName()
1713 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1714 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1715 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1717 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1718 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1719 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1721 return overscanResults
1724 """Set the variance plane using the amplifier gain and read noise 1726 The read noise is calculated from the ``overscanImage`` if the 1727 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1728 the value from the amplifier data is used. 1732 ampExposure : `lsst.afw.image.Exposure` 1733 Exposure to process. 1734 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1735 Amplifier detector data. 1736 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1737 Image of overscan, required only for empirical read noise. 1741 lsst.ip.isr.isrFunctions.updateVariance 1743 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1744 gain = amp.getGain()
1746 if math.isnan(gain):
1748 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1751 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1752 (amp.getName(), gain, patchedGain))
1755 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1756 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1758 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1759 stats = afwMath.StatisticsControl()
1760 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1761 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1762 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1764 readNoise = amp.getReadNoise()
1766 isrFunctions.updateVariance(
1767 maskedImage=ampExposure.getMaskedImage(),
1769 readNoise=readNoise,
1773 """!Apply dark correction in place. 1777 exposure : `lsst.afw.image.Exposure` 1778 Exposure to process. 1779 darkExposure : `lsst.afw.image.Exposure` 1780 Dark exposure of the same size as ``exposure``. 1781 invert : `Bool`, optional 1782 If True, re-add the dark to an already corrected image. 1787 Raised if either ``exposure`` or ``darkExposure`` do not 1788 have their dark time defined. 1792 lsst.ip.isr.isrFunctions.darkCorrection 1794 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1795 if math.isnan(expScale):
1796 raise RuntimeError(
"Exposure darktime is NAN")
1797 if darkExposure.getInfo().getVisitInfo()
is not None:
1798 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1804 if math.isnan(darkScale):
1805 raise RuntimeError(
"Dark calib darktime is NAN")
1806 isrFunctions.darkCorrection(
1807 maskedImage=exposure.getMaskedImage(),
1808 darkMaskedImage=darkExposure.getMaskedImage(),
1810 darkScale=darkScale,
1812 trimToFit=self.config.doTrimToMatchCalib
1816 """!Check if linearization is needed for the detector cameraGeom. 1818 Checks config.doLinearize and the linearity type of the first 1823 detector : `lsst.afw.cameraGeom.Detector` 1824 Detector to get linearity type from. 1828 doLinearize : `Bool` 1829 If True, linearization should be performed. 1831 return self.config.doLinearize
and \
1832 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1835 """!Apply flat correction in place. 1839 exposure : `lsst.afw.image.Exposure` 1840 Exposure to process. 1841 flatExposure : `lsst.afw.image.Exposure` 1842 Flat exposure of the same size as ``exposure``. 1843 invert : `Bool`, optional 1844 If True, unflatten an already flattened image. 1848 lsst.ip.isr.isrFunctions.flatCorrection 1850 isrFunctions.flatCorrection(
1851 maskedImage=exposure.getMaskedImage(),
1852 flatMaskedImage=flatExposure.getMaskedImage(),
1853 scalingType=self.config.flatScalingType,
1854 userScale=self.config.flatUserScale,
1856 trimToFit=self.config.doTrimToMatchCalib
1860 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1864 exposure : `lsst.afw.image.Exposure` 1865 Exposure to process. Only the amplifier DataSec is processed. 1866 amp : `lsst.afw.table.AmpInfoCatalog` 1867 Amplifier detector data. 1871 lsst.ip.isr.isrFunctions.makeThresholdMask 1873 if not math.isnan(amp.getSaturation()):
1874 maskedImage = exposure.getMaskedImage()
1875 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1876 isrFunctions.makeThresholdMask(
1877 maskedImage=dataView,
1878 threshold=amp.getSaturation(),
1880 maskName=self.config.saturatedMaskName,
1884 """!Interpolate over saturated pixels, in place. 1886 This method should be called after `saturationDetection`, to 1887 ensure that the saturated pixels have been identified in the 1888 SAT mask. It should also be called after `assembleCcd`, since 1889 saturated regions may cross amplifier boundaries. 1893 exposure : `lsst.afw.image.Exposure` 1894 Exposure to process. 1898 lsst.ip.isr.isrTask.saturationDetection 1899 lsst.ip.isr.isrFunctions.interpolateFromMask 1901 isrFunctions.interpolateFromMask(
1902 maskedImage=ccdExposure.getMaskedImage(),
1903 fwhm=self.config.fwhm,
1904 growFootprints=self.config.growSaturationFootprintSize,
1905 maskName=self.config.saturatedMaskName,
1909 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1913 exposure : `lsst.afw.image.Exposure` 1914 Exposure to process. Only the amplifier DataSec is processed. 1915 amp : `lsst.afw.table.AmpInfoCatalog` 1916 Amplifier detector data. 1920 lsst.ip.isr.isrFunctions.makeThresholdMask 1924 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1925 This is intended to indicate pixels that may be affected by unknown systematics; 1926 for example if non-linearity corrections above a certain level are unstable 1927 then that would be a useful value for suspectLevel. A value of `nan` indicates 1928 that no such level exists and no pixels are to be masked as suspicious. 1930 suspectLevel = amp.getSuspectLevel()
1931 if math.isnan(suspectLevel):
1934 maskedImage = exposure.getMaskedImage()
1935 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1936 isrFunctions.makeThresholdMask(
1937 maskedImage=dataView,
1938 threshold=suspectLevel,
1940 maskName=self.config.suspectMaskName,
1944 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1948 ccdExposure : `lsst.afw.image.Exposure` 1949 Exposure to process. 1950 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of 1951 `lsst.afw.image.DefectBase`. 1952 List of defects to mask and interpolate. 1956 Call this after CCD assembly, since defects may cross amplifier boundaries. 1958 maskedImage = ccdExposure.getMaskedImage()
1959 if not isinstance(defectBaseList, Defects):
1961 defectList = Defects(defectBaseList)
1963 defectList = defectBaseList
1964 defectList.maskPixels(maskedImage, maskName=
"BAD")
1965 isrFunctions.interpolateDefectList(
1966 maskedImage=maskedImage,
1967 defectList=defectList,
1968 fwhm=self.config.fwhm,
1971 if self.config.numEdgeSuspect > 0:
1972 goodBBox = maskedImage.getBBox()
1974 goodBBox.grow(-self.config.numEdgeSuspect)
1976 SourceDetectionTask.setEdgeBits(
1979 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1983 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1987 exposure : `lsst.afw.image.Exposure` 1988 Exposure to process. 1992 We mask and interpolate over all NaNs, including those 1993 that are masked with other bits (because those may or may 1994 not be interpolated over later, and we want to remove all 1995 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1996 is used to preserve the historical name. 1998 maskedImage = exposure.getMaskedImage()
2001 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2002 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2003 numNans =
maskNans(maskedImage, maskVal)
2004 self.metadata.set(
"NUMNANS", numNans)
2008 self.log.warn(
"There were %i unmasked NaNs", numNans)
2009 nanDefectList = Defects.fromMask(
2010 maskedImage=maskedImage,
2011 maskName=
'UNMASKEDNAN',
2013 isrFunctions.interpolateDefectList(
2014 maskedImage=exposure.getMaskedImage(),
2015 defectList=nanDefectList,
2016 fwhm=self.config.fwhm,
2020 """Measure the image background in subgrids, for quality control purposes. 2024 exposure : `lsst.afw.image.Exposure` 2025 Exposure to process. 2026 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 2027 Configuration object containing parameters on which background 2028 statistics and subgrids to use. 2030 if IsrQaConfig
is not None:
2031 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2032 IsrQaConfig.flatness.nIter)
2033 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2034 statsControl.setAndMask(maskVal)
2035 maskedImage = exposure.getMaskedImage()
2036 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2037 skyLevel = stats.getValue(afwMath.MEDIAN)
2038 skySigma = stats.getValue(afwMath.STDEVCLIP)
2039 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2040 metadata = exposure.getMetadata()
2041 metadata.set(
'SKYLEVEL', skyLevel)
2042 metadata.set(
'SKYSIGMA', skySigma)
2045 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2046 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2047 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2048 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2049 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2050 skyLevels = numpy.zeros((nX, nY))
2053 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2055 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2057 xLLC = xc - meshXHalf
2058 yLLC = yc - meshYHalf
2059 xURC = xc + meshXHalf - 1
2060 yURC = yc + meshYHalf - 1
2063 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2065 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2067 good = numpy.where(numpy.isfinite(skyLevels))
2068 skyMedian = numpy.median(skyLevels[good])
2069 flatness = (skyLevels[good] - skyMedian) / skyMedian
2070 flatness_rms = numpy.std(flatness)
2071 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2073 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2074 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
2075 (nX, nY, flatness_pp, flatness_rms))
2077 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2078 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2079 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2080 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2081 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2084 """Set an approximate magnitude zero point for the exposure. 2088 exposure : `lsst.afw.image.Exposure` 2089 Exposure to process. 2091 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2092 if filterName
in self.config.fluxMag0T1:
2093 fluxMag0 = self.config.fluxMag0T1[filterName]
2095 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
2096 fluxMag0 = self.config.defaultFluxMag0T1
2098 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2100 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
2103 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2104 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2107 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2111 ccdExposure : `lsst.afw.image.Exposure` 2112 Exposure to process. 2113 fpPolygon : `lsst.afw.geom.Polygon` 2114 Polygon in focal plane coordinates. 2117 ccd = ccdExposure.getDetector()
2118 fpCorners = ccd.getCorners(FOCAL_PLANE)
2119 ccdPolygon = Polygon(fpCorners)
2122 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2125 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2126 validPolygon = Polygon(ccdPoints)
2127 ccdExposure.getInfo().setValidPolygon(validPolygon)
2131 """Context manager that applies and removes flats and darks, 2132 if the task is configured to apply them. 2136 exp : `lsst.afw.image.Exposure` 2137 Exposure to process. 2138 flat : `lsst.afw.image.Exposure` 2139 Flat exposure the same size as ``exp``. 2140 dark : `lsst.afw.image.Exposure`, optional 2141 Dark exposure the same size as ``exp``. 2145 exp : `lsst.afw.image.Exposure` 2146 The flat and dark corrected exposure. 2148 if self.config.doDark
and dark
is not None:
2150 if self.config.doFlat:
2155 if self.config.doFlat:
2157 if self.config.doDark
and dark
is not None:
2161 """Utility function to examine ISR exposure at different stages. 2165 exposure : `lsst.afw.image.Exposure` 2168 State of processing to view. 2170 frame = getDebugFrame(self._display, stepname)
2172 display = getDisplay(frame)
2173 display.scale(
'asinh',
'zscale')
2174 display.mtv(exposure)
2178 """A Detector-like object that supports returning gain and saturation level 2180 This is used when the input exposure does not have a detector. 2184 exposure : `lsst.afw.image.Exposure` 2185 Exposure to generate a fake amplifier for. 2186 config : `lsst.ip.isr.isrTaskConfig` 2187 Configuration to apply to the fake amplifier. 2191 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2193 self.
_gain = config.gain
2223 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2227 """Task to wrap the default IsrTask to allow it to be retargeted. 2229 The standard IsrTask can be called directly from a command line 2230 program, but doing so removes the ability of the task to be 2231 retargeted. As most cameras override some set of the IsrTask 2232 methods, this would remove those data-specific methods in the 2233 output post-ISR images. This wrapping class fixes the issue, 2234 allowing identical post-ISR images to be generated by both the 2235 processCcd and isrTask code. 2237 ConfigClass = RunIsrConfig
2238 _DefaultName =
"runIsr" 2242 self.makeSubtask(
"isr")
2248 dataRef : `lsst.daf.persistence.ButlerDataRef` 2249 data reference of the detector data to be processed 2253 result : `pipeBase.Struct` 2254 Result struct with component: 2256 - exposure : `lsst.afw.image.Exposure` 2257 Post-ISR processed exposure. def getInputDatasetTypes(cls, config)
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def getPrerequisiteDatasetTypes(cls, config)
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def getOutputDatasetTypes(cls, config)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def makeDatasetType(self, dsConfig)
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def getPerDatasetTypeDimensions(cls, config)
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, isGen3=False)
Perform instrument signature removal on an exposure.
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)