33 from contextlib
import contextmanager
34 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
47 from .assembleCcdTask
import AssembleCcdTask
48 from .crosstalk
import CrosstalkTask
49 from .fringe
import FringeTask
50 from .isr
import maskNans
51 from .masking
import MaskingTask
52 from .straylight
import StrayLightTask
53 from .vignette
import VignetteTask
55 __all__ = [
"IsrTask",
"RunIsrTask"]
59 """Configuration parameters for IsrTask. 61 Items are grouped in the order in which they are executed by the task. 66 isrName = pexConfig.Field(
73 ccdExposure = pipeBase.InputDatasetField(
74 doc=
"Input exposure to process",
77 storageClass=
"ExposureU",
78 dimensions=[
"Instrument",
"Exposure",
"Detector"],
80 camera = pipeBase.InputDatasetField(
81 doc=
"Input camera to construct complete exposures.",
84 storageClass=
"TablePersistableCamera",
85 dimensions=[
"Instrument",
"ExposureRange"],
87 bias = pipeBase.InputDatasetField(
88 doc=
"Input bias calibration.",
91 storageClass=
"ImageF",
92 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
94 dark = pipeBase.InputDatasetField(
95 doc=
"Input dark calibration.",
98 storageClass=
"ImageF",
99 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
101 flat = pipeBase.InputDatasetField(
102 doc=
"Input flat calibration.",
105 storageClass=
"MaskedImageF",
106 dimensions=[
"Instrument",
"PhysicalFilter",
"ExposureRange",
"Detector"],
108 bfKernel = pipeBase.InputDatasetField(
109 doc=
"Input brighter-fatter kernel.",
112 storageClass=
"NumpyArray",
113 dimensions=[
"Instrument",
"ExposureRange"],
115 defects = pipeBase.InputDatasetField(
116 doc=
"Input defect tables.",
119 storageClass=
"Catalog",
120 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
122 opticsTransmission = pipeBase.InputDatasetField(
123 doc=
"Transmission curve due to the optics.",
124 name=
"transmission_optics",
126 storageClass=
"TablePersistableTransmissionCurve",
127 dimensions=[
"Instrument",
"ExposureRange"],
129 filterTransmission = pipeBase.InputDatasetField(
130 doc=
"Transmission curve due to the filter.",
131 name=
"transmission_filter",
133 storageClass=
"TablePersistableTransmissionCurve",
134 dimensions=[
"Instrument",
"PhysicalFilter",
"ExposureRange"],
136 sensorTransmission = pipeBase.InputDatasetField(
137 doc=
"Transmission curve due to the sensor.",
138 name=
"transmission_sensor",
140 storageClass=
"TablePersistableTransmissionCurve",
141 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
143 atmosphereTransmission = pipeBase.InputDatasetField(
144 doc=
"Transmission curve due to the atmosphere.",
145 name=
"transmission_atmosphere",
147 storageClass=
"TablePersistableTransmissionCurve",
148 dimensions=[
"Instrument"],
152 outputExposure = pipeBase.OutputDatasetField(
153 doc=
"Output ISR processed exposure.",
156 storageClass=
"ExposureF",
157 dimensions=[
"Instrument",
"Visit",
"Detector"],
159 outputOssThumbnail = pipeBase.OutputDatasetField(
160 doc=
"Output Overscan-subtracted thumbnail image.",
163 storageClass=
"Thumbnail",
164 dimensions=[
"Instrument",
"Visit",
"Detector"],
166 outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167 doc=
"Output flat-corrected thumbnail image.",
168 name=
"FlattenedThumb",
170 storageClass=
"TextStorage",
171 dimensions=[
"Instrument",
"Visit",
"Detector"],
174 quantum = pipeBase.QuantumConfig(
175 dimensions=[
"Visit",
"Detector",
"Instrument"],
179 datasetType = pexConfig.Field(
181 doc=
"Dataset type for input data; users will typically leave this alone, " 182 "but camera-specific ISR tasks will override it",
186 fallbackFilterName = pexConfig.Field(
188 doc=
"Fallback default filter name for calibrations.",
191 expectWcs = pexConfig.Field(
194 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 196 fwhm = pexConfig.Field(
198 doc=
"FWHM of PSF in arcseconds.",
201 qa = pexConfig.ConfigField(
203 doc=
"QA related configuration options.",
207 doConvertIntToFloat = pexConfig.Field(
209 doc=
"Convert integer raw images to floating point values?",
214 doSaturation = pexConfig.Field(
216 doc=
"Mask saturated pixels? NB: this is totally independent of the" 217 " interpolation option - this is ONLY setting the bits in the mask." 218 " To have them interpolated make sure doSaturationInterpolation=True",
221 saturatedMaskName = pexConfig.Field(
223 doc=
"Name of mask plane to use in saturation detection and interpolation",
226 saturation = pexConfig.Field(
228 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229 default=float(
"NaN"),
231 growSaturationFootprintSize = pexConfig.Field(
233 doc=
"Number of pixels by which to grow the saturation footprints",
238 doSuspect = pexConfig.Field(
240 doc=
"Mask suspect pixels?",
243 suspectMaskName = pexConfig.Field(
245 doc=
"Name of mask plane to use for suspect pixels",
248 numEdgeSuspect = pexConfig.Field(
250 doc=
"Number of edge pixels to be flagged as untrustworthy.",
255 doSetBadRegions = pexConfig.Field(
257 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
260 badStatistic = pexConfig.ChoiceField(
262 doc=
"How to estimate the average value for BAD regions.",
265 "MEANCLIP":
"Correct using the (clipped) mean of good data",
266 "MEDIAN":
"Correct using the median of the good data",
271 doOverscan = pexConfig.Field(
273 doc=
"Do overscan subtraction?",
276 overscanFitType = pexConfig.ChoiceField(
278 doc=
"The method for fitting the overscan bias level.",
281 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
282 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
283 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
284 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
285 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
286 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
287 "MEAN":
"Correct using the mean of the overscan region",
288 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
289 "MEDIAN":
"Correct using the median of the overscan region",
292 overscanOrder = pexConfig.Field(
294 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
295 "or number of spline knots if overscan fit type is a spline."),
298 overscanNumSigmaClip = pexConfig.Field(
300 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
303 overscanIsInt = pexConfig.Field(
305 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
308 overscanNumLeadingColumnsToSkip = pexConfig.Field(
310 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
313 overscanNumTrailingColumnsToSkip = pexConfig.Field(
315 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
318 overscanMaxDev = pexConfig.Field(
320 doc=
"Maximum deviation from the median for overscan",
321 default=1000.0, check=
lambda x: x > 0
323 overscanBiasJump = pexConfig.Field(
325 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
328 overscanBiasJumpKeyword = pexConfig.Field(
330 doc=
"Header keyword containing information about devices.",
331 default=
"NO_SUCH_KEY",
333 overscanBiasJumpDevices = pexConfig.ListField(
335 doc=
"List of devices that need piecewise overscan correction.",
338 overscanBiasJumpLocation = pexConfig.Field(
340 doc=
"Location of bias jump along y-axis.",
345 doAssembleCcd = pexConfig.Field(
348 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 350 assembleCcd = pexConfig.ConfigurableField(
351 target=AssembleCcdTask,
352 doc=
"CCD assembly task",
356 doAssembleIsrExposures = pexConfig.Field(
359 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 361 doTrimToMatchCalib = pexConfig.Field(
364 doc=
"Trim raw data to match calibration bounding boxes?" 368 doBias = pexConfig.Field(
370 doc=
"Apply bias frame correction?",
373 biasDataProductName = pexConfig.Field(
375 doc=
"Name of the bias data product",
380 doVariance = pexConfig.Field(
382 doc=
"Calculate variance?",
385 gain = pexConfig.Field(
387 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388 default=float(
"NaN"),
390 readNoise = pexConfig.Field(
392 doc=
"The read noise to use if no Detector is present in the Exposure",
395 doEmpiricalReadNoise = pexConfig.Field(
398 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 402 doLinearize = pexConfig.Field(
404 doc=
"Correct for nonlinearity of the detector's response?",
409 doCrosstalk = pexConfig.Field(
411 doc=
"Apply intra-CCD crosstalk correction?",
414 doCrosstalkBeforeAssemble = pexConfig.Field(
416 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
419 crosstalk = pexConfig.ConfigurableField(
420 target=CrosstalkTask,
421 doc=
"Intra-CCD crosstalk correction",
425 doWidenSaturationTrails = pexConfig.Field(
427 doc=
"Widen bleed trails based on their width?",
432 doBrighterFatter = pexConfig.Field(
435 doc=
"Apply the brighter fatter correction" 437 brighterFatterLevel = pexConfig.ChoiceField(
440 doc=
"The level at which to correct for brighter-fatter.",
442 "AMP":
"Every amplifier treated separately.",
443 "DETECTOR":
"One kernel per detector",
446 brighterFatterKernelFile = pexConfig.Field(
449 doc=
"Kernel file used for the brighter fatter correction" 451 brighterFatterMaxIter = pexConfig.Field(
454 doc=
"Maximum number of iterations for the brighter fatter correction" 456 brighterFatterThreshold = pexConfig.Field(
459 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 460 " absolute value of the difference between the current corrected image and the one" 461 " from the previous iteration summed over all the pixels." 463 brighterFatterApplyGain = pexConfig.Field(
466 doc=
"Should the gain be applied when applying the brighter fatter correction?" 470 doDefect = pexConfig.Field(
472 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
475 doSaturationInterpolation = pexConfig.Field(
477 doc=
"Perform interpolation over pixels masked as saturated?" 478 " NB: This is independent of doSaturation; if that is False this plane" 479 " will likely be blank, resulting in a no-op here.",
482 numEdgeSuspect = pexConfig.Field(
484 doc=
"Number of edge pixels to be flagged as untrustworthy.",
489 doDark = pexConfig.Field(
491 doc=
"Apply dark frame correction?",
494 darkDataProductName = pexConfig.Field(
496 doc=
"Name of the dark data product",
501 doStrayLight = pexConfig.Field(
503 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
506 strayLight = pexConfig.ConfigurableField(
507 target=StrayLightTask,
508 doc=
"y-band stray light correction" 512 doFlat = pexConfig.Field(
514 doc=
"Apply flat field correction?",
517 flatDataProductName = pexConfig.Field(
519 doc=
"Name of the flat data product",
522 flatScalingType = pexConfig.ChoiceField(
524 doc=
"The method for scaling the flat on the fly.",
527 "USER":
"Scale by flatUserScale",
528 "MEAN":
"Scale by the inverse of the mean",
529 "MEDIAN":
"Scale by the inverse of the median",
532 flatUserScale = pexConfig.Field(
534 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
537 doTweakFlat = pexConfig.Field(
539 doc=
"Tweak flats to match observed amplifier ratios?",
544 doApplyGains = pexConfig.Field(
546 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
549 normalizeGains = pexConfig.Field(
551 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
556 doFringe = pexConfig.Field(
558 doc=
"Apply fringe correction?",
561 fringe = pexConfig.ConfigurableField(
563 doc=
"Fringe subtraction task",
565 fringeAfterFlat = pexConfig.Field(
567 doc=
"Do fringe subtraction after flat-fielding?",
572 doNanInterpAfterFlat = pexConfig.Field(
574 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 575 "also have to interpolate them before flat-fielding."),
580 doAddDistortionModel = pexConfig.Field(
582 doc=
"Apply a distortion model based on camera geometry to the WCS?",
587 doMeasureBackground = pexConfig.Field(
589 doc=
"Measure the background level on the reduced image?",
594 doCameraSpecificMasking = pexConfig.Field(
596 doc=
"Mask camera-specific bad regions?",
599 masking = pexConfig.ConfigurableField(
605 fluxMag0T1 = pexConfig.DictField(
608 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
612 defaultFluxMag0T1 = pexConfig.Field(
614 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
615 default=pow(10.0, 0.4*28.0)
619 doVignette = pexConfig.Field(
621 doc=
"Apply vignetting parameters?",
624 vignette = pexConfig.ConfigurableField(
626 doc=
"Vignetting task.",
630 doAttachTransmissionCurve = pexConfig.Field(
633 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 635 doUseOpticsTransmission = pexConfig.Field(
638 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 640 doUseFilterTransmission = pexConfig.Field(
643 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 645 doUseSensorTransmission = pexConfig.Field(
648 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 650 doUseAtmosphereTransmission = pexConfig.Field(
653 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 657 doWrite = pexConfig.Field(
659 doc=
"Persist postISRCCD?",
666 raise ValueError(
"You may not specify both doFlat and doApplyGains")
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670 r"""Apply common instrument signature correction algorithms to a raw frame. 672 The process for correcting imaging data is very similar from 673 camera to camera. This task provides a vanilla implementation of 674 doing these corrections, including the ability to turn certain 675 corrections off if they are not needed. The inputs to the primary 676 method, `run()`, are a raw exposure to be corrected and the 677 calibration data products. The raw input is a single chip sized 678 mosaic of all amps including overscans and other non-science 679 pixels. The method `runDataRef()` identifies and defines the 680 calibration data products, and is intended for use by a 681 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 682 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 683 subclassed for different camera, although the most camera specific 684 methods have been split into subtasks that can be redirected 687 The __init__ method sets up the subtasks for ISR processing, using 688 the defaults from `lsst.ip.isr`. 693 Positional arguments passed to the Task constructor. None used at this time. 694 kwargs : `dict`, optional 695 Keyword arguments passed on to the Task constructor. None used at this time. 697 ConfigClass = IsrTaskConfig
702 self.makeSubtask(
"assembleCcd")
703 self.makeSubtask(
"crosstalk")
704 self.makeSubtask(
"strayLight")
705 self.makeSubtask(
"fringe")
706 self.makeSubtask(
"masking")
707 self.makeSubtask(
"vignette")
716 if config.doBias
is not True:
717 inputTypeDict.pop(
"bias",
None)
718 if config.doLinearize
is not True:
719 inputTypeDict.pop(
"linearizer",
None)
720 if config.doCrosstalk
is not True:
721 inputTypeDict.pop(
"crosstalkSources",
None)
722 if config.doBrighterFatter
is not True:
723 inputTypeDict.pop(
"bfKernel",
None)
724 if config.doDefect
is not True:
725 inputTypeDict.pop(
"defects",
None)
726 if config.doDark
is not True:
727 inputTypeDict.pop(
"dark",
None)
728 if config.doFlat
is not True:
729 inputTypeDict.pop(
"flat",
None)
730 if config.doAttachTransmissionCurve
is not True:
731 inputTypeDict.pop(
"opticsTransmission",
None)
732 inputTypeDict.pop(
"filterTransmission",
None)
733 inputTypeDict.pop(
"sensorTransmission",
None)
734 inputTypeDict.pop(
"atmosphereTransmission",
None)
735 if config.doUseOpticsTransmission
is not True:
736 inputTypeDict.pop(
"opticsTransmission",
None)
737 if config.doUseFilterTransmission
is not True:
738 inputTypeDict.pop(
"filterTransmission",
None)
739 if config.doUseSensorTransmission
is not True:
740 inputTypeDict.pop(
"sensorTransmission",
None)
741 if config.doUseAtmosphereTransmission
is not True:
742 inputTypeDict.pop(
"atmosphereTransmission",
None)
750 if config.qa.doThumbnailOss
is not True:
751 outputTypeDict.pop(
"outputOssThumbnail",
None)
752 if config.qa.doThumbnailFlattened
is not True:
753 outputTypeDict.pop(
"outputFlattenedThumbnail",
None)
754 if config.doWrite
is not True:
755 outputTypeDict.pop(
"outputExposure",
None)
757 return outputTypeDict
761 inputData[
'detectorNum'] = int(inputDataIds[
'ccdExposure'][
'detector'])
762 except Exception
as e:
763 raise ValueError(f
"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
765 inputData[
'isGen3'] =
True 767 if self.config.doLinearize
is True:
768 if 'linearizer' not in inputData.keys():
769 detector = inputData[
'camera'][inputData[
'detectorNum']]
770 linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
771 inputData[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
773 if inputData[
'defects']
is not None:
778 for r
in inputData[
'defects']:
779 bbox = afwGeom.BoxI(afwGeom.PointI(r.get(
"x0"), r.get(
"y0")),
780 afwGeom.ExtentI(r.get(
"width"), r.get(
"height")))
781 defectList.append(
Defect(bbox))
783 inputData[
'defects'] = defectList
800 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
806 """!Retrieve necessary frames for instrument signature removal. 808 Pre-fetching all required ISR data products limits the IO 809 required by the ISR. Any conflict between the calibration data 810 available and that needed for ISR is also detected prior to 811 doing processing, allowing it to fail quickly. 815 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 816 Butler reference of the detector data to be processed 817 rawExposure : `afw.image.Exposure` 818 The raw exposure that will later be corrected with the 819 retrieved calibration data; should not be modified in this 824 result : `lsst.pipe.base.Struct` 825 Result struct with components (which may be `None`): 826 - ``bias``: bias calibration frame (`afw.image.Exposure`) 827 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 828 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 829 - ``dark``: dark calibration frame (`afw.image.Exposure`) 830 - ``flat``: flat calibration frame (`afw.image.Exposure`) 831 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 832 - ``defects``: list of defects (`list`) 833 - ``fringes``: `lsst.pipe.base.Struct` with components: 834 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 835 - ``seed``: random seed derived from the ccdExposureId for random 836 number generator (`uint32`) 837 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 838 A ``TransmissionCurve`` that represents the throughput of the optics, 839 to be evaluated in focal-plane coordinates. 840 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 841 A ``TransmissionCurve`` that represents the throughput of the filter 842 itself, to be evaluated in focal-plane coordinates. 843 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 844 A ``TransmissionCurve`` that represents the throughput of the sensor 845 itself, to be evaluated in post-assembly trimmed detector coordinates. 846 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 847 A ``TransmissionCurve`` that represents the throughput of the 848 atmosphere, assumed to be spatially constant. 851 ccd = rawExposure.getDetector()
852 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
853 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
854 if self.config.doBias
else None)
856 linearizer = (dataRef.get(
"linearizer", immediate=
True)
858 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
859 if self.config.doCrosstalk
else None)
860 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
861 if self.config.doDark
else None)
862 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
863 if self.config.doFlat
else None)
864 brighterFatterKernel = (dataRef.get(
"bfKernel")
865 if self.config.doBrighterFatter
else None)
866 defectList = (dataRef.get(
"defects")
867 if self.config.doDefect
else None)
868 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
869 if self.config.doAssembleIsrExposures
else None)
870 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
871 else pipeBase.Struct(fringes=
None))
873 if self.config.doAttachTransmissionCurve:
874 opticsTransmission = (dataRef.get(
"transmission_optics")
875 if self.config.doUseOpticsTransmission
else None)
876 filterTransmission = (dataRef.get(
"transmission_filter")
877 if self.config.doUseFilterTransmission
else None)
878 sensorTransmission = (dataRef.get(
"transmission_sensor")
879 if self.config.doUseSensorTransmission
else None)
880 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
881 if self.config.doUseAtmosphereTransmission
else None)
883 opticsTransmission =
None 884 filterTransmission =
None 885 sensorTransmission =
None 886 atmosphereTransmission =
None 889 return pipeBase.Struct(bias=biasExposure,
890 linearizer=linearizer,
891 crosstalkSources=crosstalkSources,
894 bfKernel=brighterFatterKernel,
896 fringes=fringeStruct,
897 opticsTransmission=opticsTransmission,
898 filterTransmission=filterTransmission,
899 sensorTransmission=sensorTransmission,
900 atmosphereTransmission=atmosphereTransmission,
904 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
905 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
906 opticsTransmission=None, filterTransmission=None,
907 sensorTransmission=None, atmosphereTransmission=None,
908 detectorNum=None, isGen3=False
910 """!Perform instrument signature removal on an exposure. 912 Steps included in the ISR processing, in order performed, are: 913 - saturation and suspect pixel masking 914 - overscan subtraction 915 - CCD assembly of individual amplifiers 917 - variance image construction 918 - linearization of non-linear response 920 - brighter-fatter correction 923 - stray light subtraction 925 - masking of known defects and camera specific features 926 - vignette calculation 927 - appending transmission curve and distortion model 931 ccdExposure : `lsst.afw.image.Exposure` 932 The raw exposure that is to be run through ISR. The 933 exposure is modified by this method. 934 camera : `lsst.afw.cameraGeom.Camera`, optional 935 The camera geometry for this exposure. Used to select the 936 distortion model appropriate for this data. 937 bias : `lsst.afw.image.Exposure`, optional 938 Bias calibration frame. 939 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 940 Functor for linearization. 941 crosstalkSources : `list`, optional 942 List of possible crosstalk sources. 943 dark : `lsst.afw.image.Exposure`, optional 944 Dark calibration frame. 945 flat : `lsst.afw.image.Exposure`, optional 946 Flat calibration frame. 947 bfKernel : `numpy.ndarray`, optional 948 Brighter-fatter kernel. 949 defects : `list`, optional 951 fringes : `lsst.pipe.base.Struct`, optional 952 Struct containing the fringe correction data, with 954 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 955 - ``seed``: random seed derived from the ccdExposureId for random 956 number generator (`uint32`) 957 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 958 A ``TransmissionCurve`` that represents the throughput of the optics, 959 to be evaluated in focal-plane coordinates. 960 filterTransmission : `lsst.afw.image.TransmissionCurve` 961 A ``TransmissionCurve`` that represents the throughput of the filter 962 itself, to be evaluated in focal-plane coordinates. 963 sensorTransmission : `lsst.afw.image.TransmissionCurve` 964 A ``TransmissionCurve`` that represents the throughput of the sensor 965 itself, to be evaluated in post-assembly trimmed detector coordinates. 966 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 967 A ``TransmissionCurve`` that represents the throughput of the 968 atmosphere, assumed to be spatially constant. 969 detectorNum : `int`, optional 970 The integer number for the detector to process. 971 isGen3 : bool, optional 972 Flag this call to run() as using the Gen3 butler environment. 976 result : `lsst.pipe.base.Struct` 977 Result struct with component: 978 - ``exposure`` : `afw.image.Exposure` 979 The fully ISR corrected exposure. 980 - ``outputExposure`` : `afw.image.Exposure` 981 An alias for `exposure` 982 - ``ossThumb`` : `numpy.ndarray` 983 Thumbnail image of the exposure after overscan subtraction. 984 - ``flattenedThumb`` : `numpy.ndarray` 985 Thumbnail image of the exposure after flat-field correction. 990 Raised if a configuration option is set to True, but the 991 required calibration data has not been specified. 995 The current processed exposure can be viewed by setting the 996 appropriate lsstDebug entries in the `debug.display` 997 dictionary. The names of these entries correspond to some of 998 the IsrTaskConfig Boolean options, with the value denoting the 999 frame to use. The exposure is shown inside the matching 1000 option check and after the processing of that step has 1001 finished. The steps with debug points are: 1012 In addition, setting the "postISRCCD" entry displays the 1013 exposure after all ISR processing has finished. 1021 self.config.doFringe =
False 1024 if detectorNum
is None:
1025 raise RuntimeError(
"Must supply the detectorNum if running as Gen3")
1027 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1032 if isinstance(ccdExposure, ButlerDataRef):
1035 ccd = ccdExposure.getDetector()
1038 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 1039 ccd = [
FakeAmp(ccdExposure, self.config)]
1042 if self.config.doBias
and bias
is None:
1043 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1045 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1046 if self.config.doBrighterFatter
and bfKernel
is None:
1047 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1048 if self.config.doDark
and dark
is None:
1049 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1051 fringes = pipeBase.Struct(fringes=
None)
1052 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
1053 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1054 if self.config.doFlat
and flat
is None:
1055 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1056 if self.config.doDefect
and defects
is None:
1057 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1058 if self.config.doAddDistortionModel
and camera
is None:
1059 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
1062 if self.config.doConvertIntToFloat:
1063 self.log.info(
"Converting exposure to floating point values")
1070 if ccdExposure.getBBox().contains(amp.getBBox()):
1074 if self.config.doOverscan
and not badAmp:
1077 self.log.debug(
"Corrected overscan for amplifier %s" % (amp.getName()))
1078 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1079 if isinstance(overscanResults.overscanFit, float):
1080 qaMedian = overscanResults.overscanFit
1081 qaStdev = float(
"NaN")
1083 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1084 afwMath.MEDIAN | afwMath.STDEVCLIP)
1085 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1086 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1088 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1089 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1090 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f" %
1091 (amp.getName(), qaMedian, qaStdev))
1092 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1094 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
1095 overscanResults =
None 1097 overscans.append(overscanResults
if overscanResults
is not None else None)
1099 self.log.info(
"Skipped OSCAN")
1101 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1102 self.log.info(
"Applying crosstalk correction.")
1103 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1104 self.
debugView(ccdExposure,
"doCrosstalk")
1106 if self.config.doAssembleCcd:
1107 self.log.info(
"Assembling CCD from amplifiers")
1108 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1110 if self.config.expectWcs
and not ccdExposure.getWcs():
1111 self.log.warn(
"No WCS found in input exposure")
1112 self.
debugView(ccdExposure,
"doAssembleCcd")
1115 if self.config.qa.doThumbnailOss:
1116 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1118 if self.config.doBias:
1119 self.log.info(
"Applying bias correction.")
1120 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1121 trimToFit=self.config.doTrimToMatchCalib)
1124 if self.config.doVariance:
1125 for amp, overscanResults
in zip(ccd, overscans):
1126 if ccdExposure.getBBox().contains(amp.getBBox()):
1127 self.log.debug(
"Constructing variance map for amplifer %s" % (amp.getName()))
1128 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1129 if overscanResults
is not None:
1131 overscanImage=overscanResults.overscanImage)
1135 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1136 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1137 afwMath.MEDIAN | afwMath.STDEVCLIP)
1138 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
1139 qaStats.getValue(afwMath.MEDIAN))
1140 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
1141 qaStats.getValue(afwMath.STDEVCLIP))
1142 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f" %
1143 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1144 qaStats.getValue(afwMath.STDEVCLIP)))
1147 self.log.info(
"Applying linearizer.")
1148 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1150 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1151 self.log.info(
"Applying crosstalk correction.")
1152 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1153 self.
debugView(ccdExposure,
"doCrosstalk")
1155 if self.config.doWidenSaturationTrails:
1156 self.log.info(
"Widening saturation trails.")
1157 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1159 interpolationDone =
False 1160 if self.config.doBrighterFatter:
1166 if self.config.doDefect:
1169 if self.config.doSaturationInterpolation:
1173 interpolationDone =
True 1175 if self.config.brighterFatterLevel ==
'DETECTOR':
1176 kernelElement = bfKernel
1179 raise NotImplementedError(
"per-amplifier brighter-fatter correction not yet implemented")
1180 self.log.info(
"Applying brighter fatter correction.")
1181 isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
1182 self.config.brighterFatterMaxIter,
1183 self.config.brighterFatterThreshold,
1184 self.config.brighterFatterApplyGain,
1186 self.
debugView(ccdExposure,
"doBrighterFatter")
1188 if self.config.doDark:
1189 self.log.info(
"Applying dark correction.")
1193 if self.config.doFringe
and not self.config.fringeAfterFlat:
1194 self.log.info(
"Applying fringe correction before flat.")
1195 self.fringe.
run(ccdExposure, **fringes.getDict())
1198 if self.config.doStrayLight:
1199 self.log.info(
"Applying stray light correction.")
1200 self.strayLight.
run(ccdExposure)
1201 self.
debugView(ccdExposure,
"doStrayLight")
1203 if self.config.doFlat:
1204 self.log.info(
"Applying flat correction.")
1208 if self.config.doApplyGains:
1209 self.log.info(
"Applying gain correction instead of flat.")
1210 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1212 if self.config.doDefect
and not interpolationDone:
1213 self.log.info(
"Masking and interpolating defects.")
1216 if self.config.doSaturationInterpolation
and not interpolationDone:
1217 self.log.info(
"Interpolating saturated pixels.")
1220 if self.config.doNanInterpAfterFlat
or not interpolationDone:
1221 self.log.info(
"Masking and interpolating NAN value pixels.")
1224 if self.config.doFringe
and self.config.fringeAfterFlat:
1225 self.log.info(
"Applying fringe correction after flat.")
1226 self.fringe.
run(ccdExposure, **fringes.getDict())
1228 if self.config.doSetBadRegions:
1229 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1230 if badPixelCount > 0:
1231 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1233 flattenedThumb =
None 1234 if self.config.qa.doThumbnailFlattened:
1235 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1237 if self.config.doCameraSpecificMasking:
1238 self.log.info(
"Masking regions for camera specific reasons.")
1239 self.masking.
run(ccdExposure)
1243 if self.config.doVignette:
1244 self.log.info(
"Constructing Vignette polygon.")
1247 if self.config.vignette.doWriteVignettePolygon:
1250 if self.config.doAttachTransmissionCurve:
1251 self.log.info(
"Adding transmission curves.")
1252 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1253 filterTransmission=filterTransmission,
1254 sensorTransmission=sensorTransmission,
1255 atmosphereTransmission=atmosphereTransmission)
1257 if self.config.doAddDistortionModel:
1258 self.log.info(
"Adding a distortion model to the WCS.")
1259 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1261 if self.config.doMeasureBackground:
1262 self.log.info(
"Measuring background level:")
1265 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1267 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1268 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1269 afwMath.MEDIAN | afwMath.STDEVCLIP)
1270 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1271 qaStats.getValue(afwMath.MEDIAN))
1272 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1273 qaStats.getValue(afwMath.STDEVCLIP))
1274 self.log.debug(
" Background stats for amplifer %s: %f +/- %f" %
1275 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1276 qaStats.getValue(afwMath.STDEVCLIP)))
1278 self.
debugView(ccdExposure,
"postISRCCD")
1280 return pipeBase.Struct(
1281 exposure=ccdExposure,
1283 flattenedThumb=flattenedThumb,
1285 outputExposure=ccdExposure,
1286 outputOssThumbnail=ossThumb,
1287 outputFlattenedThumbnail=flattenedThumb,
1290 @pipeBase.timeMethod
1292 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1294 This method contains the `CmdLineTask` interface to the ISR 1295 processing. All IO is handled here, freeing the `run()` method 1296 to manage only pixel-level calculations. The steps performed 1298 - Read in necessary detrending/isr/calibration data. 1299 - Process raw exposure in `run()`. 1300 - Persist the ISR-corrected exposure as "postISRCCD" if 1301 config.doWrite=True. 1305 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1306 DataRef of the detector data to be processed 1310 result : `lsst.pipe.base.Struct` 1311 Result struct with component: 1312 - ``exposure`` : `afw.image.Exposure` 1313 The fully ISR corrected exposure. 1318 Raised if a configuration option is set to True, but the 1319 required calibration data does not exist. 1322 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1324 ccdExposure = sensorRef.get(self.config.datasetType)
1326 camera = sensorRef.get(
"camera")
1327 if camera
is None and self.config.doAddDistortionModel:
1328 raise RuntimeError(
"config.doAddDistortionModel is True " 1329 "but could not get a camera from the butler")
1330 isrData = self.
readIsrData(sensorRef, ccdExposure)
1332 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1334 if self.config.doWrite:
1335 sensorRef.put(result.exposure,
"postISRCCD")
1336 if result.ossThumb
is not None:
1337 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1338 if result.flattenedThumb
is not None:
1339 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1344 """!Retrieve a calibration dataset for removing instrument signature. 1349 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1350 DataRef of the detector data to find calibration datasets 1353 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1355 If True, disable butler proxies to enable error handling 1356 within this routine. 1360 exposure : `lsst.afw.image.Exposure` 1361 Requested calibration frame. 1366 Raised if no matching calibration frame can be found. 1369 exp = dataRef.get(datasetType, immediate=immediate)
1370 except Exception
as exc1:
1371 if not self.config.fallbackFilterName:
1372 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1374 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1375 except Exception
as exc2:
1376 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1377 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1378 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1380 if self.config.doAssembleIsrExposures:
1381 exp = self.assembleCcd.assembleCcd(exp)
1385 """Ensure that the data returned by Butler is a fully constructed exposure. 1387 ISR requires exposure-level image data for historical reasons, so if we did 1388 not recieve that from Butler, construct it from what we have, modifying the 1393 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1394 `lsst.afw.image.ImageF` 1395 The input data structure obtained from Butler. 1396 camera : `lsst.afw.cameraGeom.camera` 1397 The camera associated with the image. Used to find the appropriate 1400 The detector this exposure should match. 1404 inputExp : `lsst.afw.image.Exposure` 1405 The re-constructed exposure, with appropriate detector parameters. 1410 Raised if the input data cannot be used to construct an exposure. 1412 if isinstance(inputExp, afwImage.DecoratedImageU):
1413 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1414 elif isinstance(inputExp, afwImage.ImageF):
1415 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1416 elif isinstance(inputExp, afwImage.MaskedImageF):
1417 inputExp = afwImage.makeExposure(inputExp)
1418 elif isinstance(inputExp, afwImage.Exposure):
1421 raise TypeError(f
"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1423 if inputExp.getDetector()
is None:
1424 inputExp.setDetector(camera[detectorNum])
1429 """Convert exposure image from uint16 to float. 1431 If the exposure does not need to be converted, the input is 1432 immediately returned. For exposures that are converted to use 1433 floating point pixels, the variance is set to unity and the 1438 exposure : `lsst.afw.image.Exposure` 1439 The raw exposure to be converted. 1443 newexposure : `lsst.afw.image.Exposure` 1444 The input ``exposure``, converted to floating point pixels. 1449 Raised if the exposure type cannot be converted to float. 1452 if isinstance(exposure, afwImage.ExposureF):
1455 if not hasattr(exposure,
"convertF"):
1456 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1458 newexposure = exposure.convertF()
1459 newexposure.variance[:] = 1
1460 newexposure.mask[:] = 0x0
1465 """Identify bad amplifiers, saturated and suspect pixels. 1469 ccdExposure : `lsst.afw.image.Exposure` 1470 Input exposure to be masked. 1471 amp : `lsst.afw.table.AmpInfoCatalog` 1472 Catalog of parameters defining the amplifier on this 1475 List of defects. Used to determine if the entire 1481 If this is true, the entire amplifier area is covered by 1482 defects and unusable. 1485 maskedImage = ccdExposure.getMaskedImage()
1491 if defects
is not None:
1492 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1497 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1499 maskView = dataView.getMask()
1500 maskView |= maskView.getPlaneBitMask(
"BAD")
1507 if self.config.doSaturation
and not badAmp:
1508 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1509 if self.config.doSuspect
and not badAmp:
1510 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1512 for maskName, maskThreshold
in limits.items():
1513 if not math.isnan(maskThreshold):
1514 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1515 isrFunctions.makeThresholdMask(
1516 maskedImage=dataView,
1517 threshold=maskThreshold,
1523 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1525 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1526 self.config.suspectMaskName])
1527 if numpy.all(maskView.getArray() & maskVal > 0):
1533 """Apply overscan correction in place. 1535 This method does initial pixel rejection of the overscan 1536 region. The overscan can also be optionally segmented to 1537 allow for discontinuous overscan responses to be fit 1538 separately. The actual overscan subtraction is performed by 1539 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1540 which is called here after the amplifier is preprocessed. 1544 ccdExposure : `lsst.afw.image.Exposure` 1545 Exposure to have overscan correction performed. 1546 amp : `lsst.afw.table.AmpInfoCatalog` 1547 The amplifier to consider while correcting the overscan. 1551 overscanResults : `lsst.pipe.base.Struct` 1552 Result struct with components: 1553 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1554 Value or fit subtracted from the amplifier image data. 1555 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1556 Value or fit subtracted from the overscan image data. 1557 - ``overscanImage`` : `lsst.afw.image.Image` 1558 Image of the overscan region with the overscan 1559 correction applied. This quantity is used to estimate 1560 the amplifier read noise empirically. 1565 Raised if the ``amp`` does not contain raw pixel information. 1569 lsst.ip.isr.isrFunctions.overscanCorrection 1571 if not amp.getHasRawInfo():
1572 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1574 if amp.getRawHorizontalOverscanBBox().isEmpty():
1575 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1578 statControl = afwMath.StatisticsControl()
1579 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1582 dataBBox = amp.getRawDataBBox()
1583 oscanBBox = amp.getRawHorizontalOverscanBBox()
1587 prescanBBox = amp.getRawPrescanBBox()
1588 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1589 dx0 += self.config.overscanNumLeadingColumnsToSkip
1590 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1592 dx0 += self.config.overscanNumTrailingColumnsToSkip
1593 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1599 if ((self.config.overscanBiasJump
and 1600 self.config.overscanBiasJumpLocation)
and 1601 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1602 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1603 self.config.overscanBiasJumpDevices)):
1604 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1605 yLower = self.config.overscanBiasJumpLocation
1606 yUpper = dataBBox.getHeight() - yLower
1608 yUpper = self.config.overscanBiasJumpLocation
1609 yLower = dataBBox.getHeight() - yUpper
1611 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1612 afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1613 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1614 afwGeom.Extent2I(dx0, 0),
1615 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1618 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1619 afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1620 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1621 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1624 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1625 afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1626 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1627 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1628 oscanBBox.getHeight())))
1631 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1632 ampImage = ccdExposure.maskedImage[imageBBox]
1633 overscanImage = ccdExposure.maskedImage[overscanBBox]
1635 overscanArray = overscanImage.image.array
1636 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1637 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1638 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1640 statControl = afwMath.StatisticsControl()
1641 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1643 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1644 overscanImage=overscanImage,
1645 fitType=self.config.overscanFitType,
1646 order=self.config.overscanOrder,
1647 collapseRej=self.config.overscanNumSigmaClip,
1648 statControl=statControl,
1649 overscanIsInt=self.config.overscanIsInt
1653 levelStat = afwMath.MEDIAN
1654 sigmaStat = afwMath.STDEVCLIP
1656 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1657 self.config.qa.flatness.nIter)
1658 metadata = ccdExposure.getMetadata()
1659 ampNum = amp.getName()
1660 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1661 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1662 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1664 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1665 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1666 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1668 return overscanResults
1671 """Set the variance plane using the amplifier gain and read noise 1673 The read noise is calculated from the ``overscanImage`` if the 1674 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1675 the value from the amplifier data is used. 1679 ampExposure : `lsst.afw.image.Exposure` 1680 Exposure to process. 1681 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1682 Amplifier detector data. 1683 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1684 Image of overscan, required only for empirical read noise. 1688 lsst.ip.isr.isrFunctions.updateVariance 1690 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1691 gain = amp.getGain()
1693 if math.isnan(gain):
1695 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1698 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1699 (amp.getName(), gain, patchedGain))
1702 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1703 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1705 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1706 stats = afwMath.StatisticsControl()
1707 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1708 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1709 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1711 readNoise = amp.getReadNoise()
1713 isrFunctions.updateVariance(
1714 maskedImage=ampExposure.getMaskedImage(),
1716 readNoise=readNoise,
1720 """!Apply dark correction in place. 1724 exposure : `lsst.afw.image.Exposure` 1725 Exposure to process. 1726 darkExposure : `lsst.afw.image.Exposure` 1727 Dark exposure of the same size as ``exposure``. 1728 invert : `Bool`, optional 1729 If True, re-add the dark to an already corrected image. 1734 Raised if either ``exposure`` or ``darkExposure`` do not 1735 have their dark time defined. 1739 lsst.ip.isr.isrFunctions.darkCorrection 1741 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1742 if math.isnan(expScale):
1743 raise RuntimeError(
"Exposure darktime is NAN")
1744 if darkExposure.getInfo().getVisitInfo()
is not None:
1745 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1751 if math.isnan(darkScale):
1752 raise RuntimeError(
"Dark calib darktime is NAN")
1753 isrFunctions.darkCorrection(
1754 maskedImage=exposure.getMaskedImage(),
1755 darkMaskedImage=darkExposure.getMaskedImage(),
1757 darkScale=darkScale,
1759 trimToFit=self.config.doTrimToMatchCalib
1763 """!Check if linearization is needed for the detector cameraGeom. 1765 Checks config.doLinearize and the linearity type of the first 1770 detector : `lsst.afw.cameraGeom.Detector` 1771 Detector to get linearity type from. 1775 doLinearize : `Bool` 1776 If True, linearization should be performed. 1778 return self.config.doLinearize
and \
1779 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1782 """!Apply flat correction in place. 1786 exposure : `lsst.afw.image.Exposure` 1787 Exposure to process. 1788 flatExposure : `lsst.afw.image.Exposure` 1789 Flat exposure of the same size as ``exposure``. 1790 invert : `Bool`, optional 1791 If True, unflatten an already flattened image. 1795 lsst.ip.isr.isrFunctions.flatCorrection 1797 isrFunctions.flatCorrection(
1798 maskedImage=exposure.getMaskedImage(),
1799 flatMaskedImage=flatExposure.getMaskedImage(),
1800 scalingType=self.config.flatScalingType,
1801 userScale=self.config.flatUserScale,
1803 trimToFit=self.config.doTrimToMatchCalib
1807 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1811 exposure : `lsst.afw.image.Exposure` 1812 Exposure to process. Only the amplifier DataSec is processed. 1813 amp : `lsst.afw.table.AmpInfoCatalog` 1814 Amplifier detector data. 1818 lsst.ip.isr.isrFunctions.makeThresholdMask 1820 if not math.isnan(amp.getSaturation()):
1821 maskedImage = exposure.getMaskedImage()
1822 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1823 isrFunctions.makeThresholdMask(
1824 maskedImage=dataView,
1825 threshold=amp.getSaturation(),
1827 maskName=self.config.saturatedMaskName,
1831 """!Interpolate over saturated pixels, in place. 1833 This method should be called after `saturationDetection`, to 1834 ensure that the saturated pixels have been identified in the 1835 SAT mask. It should also be called after `assembleCcd`, since 1836 saturated regions may cross amplifier boundaries. 1840 exposure : `lsst.afw.image.Exposure` 1841 Exposure to process. 1845 lsst.ip.isr.isrTask.saturationDetection 1846 lsst.ip.isr.isrFunctions.interpolateFromMask 1848 isrFunctions.interpolateFromMask(
1849 maskedImage=ccdExposure.getMaskedImage(),
1850 fwhm=self.config.fwhm,
1851 growFootprints=self.config.growSaturationFootprintSize,
1852 maskName=self.config.saturatedMaskName,
1856 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1860 exposure : `lsst.afw.image.Exposure` 1861 Exposure to process. Only the amplifier DataSec is processed. 1862 amp : `lsst.afw.table.AmpInfoCatalog` 1863 Amplifier detector data. 1867 lsst.ip.isr.isrFunctions.makeThresholdMask 1871 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1872 This is intended to indicate pixels that may be affected by unknown systematics; 1873 for example if non-linearity corrections above a certain level are unstable 1874 then that would be a useful value for suspectLevel. A value of `nan` indicates 1875 that no such level exists and no pixels are to be masked as suspicious. 1877 suspectLevel = amp.getSuspectLevel()
1878 if math.isnan(suspectLevel):
1881 maskedImage = exposure.getMaskedImage()
1882 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1883 isrFunctions.makeThresholdMask(
1884 maskedImage=dataView,
1885 threshold=suspectLevel,
1887 maskName=self.config.suspectMaskName,
1891 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1895 ccdExposure : `lsst.afw.image.Exposure` 1896 Exposure to process. 1897 defectBaseList : `List` 1898 List of defects to mask and interpolate. 1902 Call this after CCD assembly, since defects may cross amplifier boundaries. 1904 maskedImage = ccdExposure.getMaskedImage()
1906 for d
in defectBaseList:
1908 nd = measAlg.Defect(bbox)
1909 defectList.append(nd)
1910 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
1911 isrFunctions.interpolateDefectList(
1912 maskedImage=maskedImage,
1913 defectList=defectList,
1914 fwhm=self.config.fwhm,
1917 if self.config.numEdgeSuspect > 0:
1918 goodBBox = maskedImage.getBBox()
1920 goodBBox.grow(-self.config.numEdgeSuspect)
1922 SourceDetectionTask.setEdgeBits(
1925 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1929 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1933 exposure : `lsst.afw.image.Exposure` 1934 Exposure to process. 1938 We mask and interpolate over all NaNs, including those 1939 that are masked with other bits (because those may or may 1940 not be interpolated over later, and we want to remove all 1941 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1942 is used to preserve the historical name. 1944 maskedImage = exposure.getMaskedImage()
1947 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
1948 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
1949 numNans =
maskNans(maskedImage, maskVal)
1950 self.metadata.set(
"NUMNANS", numNans)
1954 self.log.warn(
"There were %i unmasked NaNs", numNans)
1955 nanDefectList = isrFunctions.getDefectListFromMask(
1956 maskedImage=maskedImage,
1957 maskName=
'UNMASKEDNAN',
1959 isrFunctions.interpolateDefectList(
1960 maskedImage=exposure.getMaskedImage(),
1961 defectList=nanDefectList,
1962 fwhm=self.config.fwhm,
1966 """Measure the image background in subgrids, for quality control purposes. 1970 exposure : `lsst.afw.image.Exposure` 1971 Exposure to process. 1972 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 1973 Configuration object containing parameters on which background 1974 statistics and subgrids to use. 1976 if IsrQaConfig
is not None:
1977 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1978 IsrQaConfig.flatness.nIter)
1979 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
1980 statsControl.setAndMask(maskVal)
1981 maskedImage = exposure.getMaskedImage()
1982 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
1983 skyLevel = stats.getValue(afwMath.MEDIAN)
1984 skySigma = stats.getValue(afwMath.STDEVCLIP)
1985 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
1986 metadata = exposure.getMetadata()
1987 metadata.set(
'SKYLEVEL', skyLevel)
1988 metadata.set(
'SKYSIGMA', skySigma)
1991 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
1992 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
1993 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
1994 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
1995 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
1996 skyLevels = numpy.zeros((nX, nY))
1999 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2001 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2003 xLLC = xc - meshXHalf
2004 yLLC = yc - meshYHalf
2005 xURC = xc + meshXHalf - 1
2006 yURC = yc + meshYHalf - 1
2008 bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2009 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2011 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2013 good = numpy.where(numpy.isfinite(skyLevels))
2014 skyMedian = numpy.median(skyLevels[good])
2015 flatness = (skyLevels[good] - skyMedian) / skyMedian
2016 flatness_rms = numpy.std(flatness)
2017 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2019 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2020 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
2021 (nX, nY, flatness_pp, flatness_rms))
2023 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2024 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2025 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2026 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2027 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2030 """Set an approximate magnitude zero point for the exposure. 2034 exposure : `lsst.afw.image.Exposure` 2035 Exposure to process. 2037 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2038 if filterName
in self.config.fluxMag0T1:
2039 fluxMag0 = self.config.fluxMag0T1[filterName]
2041 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
2042 fluxMag0 = self.config.defaultFluxMag0T1
2044 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2046 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
2049 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2050 exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2053 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2057 ccdExposure : `lsst.afw.image.Exposure` 2058 Exposure to process. 2059 fpPolygon : `lsst.afw.geom.Polygon` 2060 Polygon in focal plane coordinates. 2063 ccd = ccdExposure.getDetector()
2064 fpCorners = ccd.getCorners(FOCAL_PLANE)
2065 ccdPolygon = Polygon(fpCorners)
2068 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2071 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2072 validPolygon = Polygon(ccdPoints)
2073 ccdExposure.getInfo().setValidPolygon(validPolygon)
2077 """Context manager that applies and removes flats and darks, 2078 if the task is configured to apply them. 2082 exp : `lsst.afw.image.Exposure` 2083 Exposure to process. 2084 flat : `lsst.afw.image.Exposure` 2085 Flat exposure the same size as ``exp``. 2086 dark : `lsst.afw.image.Exposure`, optional 2087 Dark exposure the same size as ``exp``. 2091 exp : `lsst.afw.image.Exposure` 2092 The flat and dark corrected exposure. 2094 if self.config.doDark
and dark
is not None:
2096 if self.config.doFlat:
2101 if self.config.doFlat:
2103 if self.config.doDark
and dark
is not None:
2107 """Utility function to examine ISR exposure at different stages. 2111 exposure : `lsst.afw.image.Exposure` 2114 State of processing to view. 2116 frame = getDebugFrame(self._display, stepname)
2118 display = getDisplay(frame)
2119 display.scale(
'asinh',
'zscale')
2120 display.mtv(exposure)
2124 """A Detector-like object that supports returning gain and saturation level 2126 This is used when the input exposure does not have a detector. 2130 exposure : `lsst.afw.image.Exposure` 2131 Exposure to generate a fake amplifier for. 2132 config : `lsst.ip.isr.isrTaskConfig` 2133 Configuration to apply to the fake amplifier. 2137 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2139 self.
_gain = config.gain
2169 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2173 """Task to wrap the default IsrTask to allow it to be retargeted. 2175 The standard IsrTask can be called directly from a command line 2176 program, but doing so removes the ability of the task to be 2177 retargeted. As most cameras override some set of the IsrTask 2178 methods, this would remove those data-specific methods in the 2179 output post-ISR images. This wrapping class fixes the issue, 2180 allowing identical post-ISR images to be generated by both the 2181 processCcd and isrTask code. 2183 ConfigClass = RunIsrConfig
2184 _DefaultName =
"runIsr" 2188 self.makeSubtask(
"isr")
2194 dataRef : `lsst.daf.persistence.ButlerDataRef` 2195 data reference of the detector data to be processed 2199 result : `pipeBase.Struct` 2200 Result struct with component: 2202 - exposure : `lsst.afw.image.Exposure` 2203 Post-ISR processed exposure. def getInputDatasetTypes(cls, config)
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def getOutputDatasetTypes(cls, config)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def makeDatasetType(self, dsConfig)
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)