33 from contextlib
import contextmanager
34 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
47 from .assembleCcdTask
import AssembleCcdTask
48 from .crosstalk
import CrosstalkTask
49 from .fringe
import FringeTask
50 from .isr
import maskNans
51 from .masking
import MaskingTask
52 from .straylight
import StrayLightTask
53 from .vignette
import VignetteTask
55 __all__ = [
"IsrTask",
"RunIsrTask"]
59 """Configuration parameters for IsrTask. 61 Items are grouped in the order in which they are executed by the task. 66 isrName = pexConfig.Field(
73 ccdExposure = pipeBase.InputDatasetField(
74 doc=
"Input exposure to process",
77 storageClass=
"ExposureU",
78 dimensions=[
"Instrument",
"Exposure",
"Detector"],
80 camera = pipeBase.InputDatasetField(
81 doc=
"Input camera to construct complete exposures.",
84 storageClass=
"TablePersistableCamera",
85 dimensions=[
"Instrument",
"ExposureRange"],
87 bias = pipeBase.InputDatasetField(
88 doc=
"Input bias calibration.",
91 storageClass=
"ImageF",
92 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
94 dark = pipeBase.InputDatasetField(
95 doc=
"Input dark calibration.",
98 storageClass=
"ImageF",
99 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
101 flat = pipeBase.InputDatasetField(
102 doc=
"Input flat calibration.",
105 storageClass=
"MaskedImageF",
106 dimensions=[
"Instrument",
"PhysicalFilter",
"ExposureRange",
"Detector"],
108 bfKernel = pipeBase.InputDatasetField(
109 doc=
"Input brighter-fatter kernel.",
112 storageClass=
"NumpyArray",
113 dimensions=[
"Instrument",
"ExposureRange"],
115 defects = pipeBase.InputDatasetField(
116 doc=
"Input defect tables.",
119 storageClass=
"Catalog",
120 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
122 opticsTransmission = pipeBase.InputDatasetField(
123 doc=
"Transmission curve due to the optics.",
124 name=
"transmission_optics",
126 storageClass=
"TablePersistableTransmissionCurve",
127 dimensions=[
"Instrument",
"ExposureRange"],
129 filterTransmission = pipeBase.InputDatasetField(
130 doc=
"Transmission curve due to the filter.",
131 name=
"transmission_filter",
133 storageClass=
"TablePersistableTransmissionCurve",
134 dimensions=[
"Instrument",
"PhysicalFilter",
"ExposureRange"],
136 sensorTransmission = pipeBase.InputDatasetField(
137 doc=
"Transmission curve due to the sensor.",
138 name=
"transmission_sensor",
140 storageClass=
"TablePersistableTransmissionCurve",
141 dimensions=[
"Instrument",
"ExposureRange",
"Detector"],
143 atmosphereTransmission = pipeBase.InputDatasetField(
144 doc=
"Transmission curve due to the atmosphere.",
145 name=
"transmission_atmosphere",
147 storageClass=
"TablePersistableTransmissionCurve",
148 dimensions=[
"Instrument"],
152 outputExposure = pipeBase.OutputDatasetField(
153 doc=
"Output ISR processed exposure.",
156 storageClass=
"ExposureF",
157 dimensions=[
"Instrument",
"Visit",
"Detector"],
159 outputOssThumbnail = pipeBase.OutputDatasetField(
160 doc=
"Output Overscan-subtracted thumbnail image.",
163 storageClass=
"Thumbnail",
164 dimensions=[
"Instrument",
"Visit",
"Detector"],
166 outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167 doc=
"Output flat-corrected thumbnail image.",
168 name=
"FlattenedThumb",
170 storageClass=
"TextStorage",
171 dimensions=[
"Instrument",
"Visit",
"Detector"],
174 quantum = pipeBase.QuantumConfig(
175 dimensions=[
"Visit",
"Detector",
"Instrument"],
179 datasetType = pexConfig.Field(
181 doc=
"Dataset type for input data; users will typically leave this alone, " 182 "but camera-specific ISR tasks will override it",
186 fallbackFilterName = pexConfig.Field(
188 doc=
"Fallback default filter name for calibrations.",
191 expectWcs = pexConfig.Field(
194 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 196 fwhm = pexConfig.Field(
198 doc=
"FWHM of PSF in arcseconds.",
201 qa = pexConfig.ConfigField(
203 doc=
"QA related configuration options.",
207 doConvertIntToFloat = pexConfig.Field(
209 doc=
"Convert integer raw images to floating point values?",
214 doSaturation = pexConfig.Field(
216 doc=
"Mask saturated pixels?",
219 saturatedMaskName = pexConfig.Field(
221 doc=
"Name of mask plane to use in saturation detection and interpolation",
224 saturation = pexConfig.Field(
226 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
227 default=float(
"NaN"),
229 growSaturationFootprintSize = pexConfig.Field(
231 doc=
"Number of pixels by which to grow the saturation footprints",
236 doSuspect = pexConfig.Field(
238 doc=
"Mask suspect pixels?",
241 suspectMaskName = pexConfig.Field(
243 doc=
"Name of mask plane to use for suspect pixels",
246 numEdgeSuspect = pexConfig.Field(
248 doc=
"Number of edge pixels to be flagged as untrustworthy.",
253 doSetBadRegions = pexConfig.Field(
255 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
258 badStatistic = pexConfig.ChoiceField(
260 doc=
"How to estimate the average value for BAD regions.",
263 "MEANCLIP":
"Correct using the (clipped) mean of good data",
264 "MEDIAN":
"Correct using the median of the good data",
269 doOverscan = pexConfig.Field(
271 doc=
"Do overscan subtraction?",
274 overscanFitType = pexConfig.ChoiceField(
276 doc=
"The method for fitting the overscan bias level.",
279 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
280 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
281 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
282 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
283 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
284 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
285 "MEAN":
"Correct using the mean of the overscan region",
286 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
287 "MEDIAN":
"Correct using the median of the overscan region",
290 overscanOrder = pexConfig.Field(
292 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
293 "or number of spline knots if overscan fit type is a spline."),
296 overscanNumSigmaClip = pexConfig.Field(
298 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
301 overscanIsInt = pexConfig.Field(
303 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
306 overscanNumLeadingColumnsToSkip = pexConfig.Field(
308 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
311 overscanNumTrailingColumnsToSkip = pexConfig.Field(
313 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
316 overscanMaxDev = pexConfig.Field(
318 doc=
"Maximum deviation from the median for overscan",
319 default=1000.0, check=
lambda x: x > 0
321 overscanBiasJump = pexConfig.Field(
323 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
326 overscanBiasJumpKeyword = pexConfig.Field(
328 doc=
"Header keyword containing information about devices.",
329 default=
"NO_SUCH_KEY",
331 overscanBiasJumpDevices = pexConfig.ListField(
333 doc=
"List of devices that need piecewise overscan correction.",
336 overscanBiasJumpLocation = pexConfig.Field(
338 doc=
"Location of bias jump along y-axis.",
343 doAssembleCcd = pexConfig.Field(
346 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 348 assembleCcd = pexConfig.ConfigurableField(
349 target=AssembleCcdTask,
350 doc=
"CCD assembly task",
354 doAssembleIsrExposures = pexConfig.Field(
357 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 359 doTrimToMatchCalib = pexConfig.Field(
362 doc=
"Trim raw data to match calibration bounding boxes?" 366 doBias = pexConfig.Field(
368 doc=
"Apply bias frame correction?",
371 biasDataProductName = pexConfig.Field(
373 doc=
"Name of the bias data product",
378 doVariance = pexConfig.Field(
380 doc=
"Calculate variance?",
383 gain = pexConfig.Field(
385 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
386 default=float(
"NaN"),
388 readNoise = pexConfig.Field(
390 doc=
"The read noise to use if no Detector is present in the Exposure",
393 doEmpiricalReadNoise = pexConfig.Field(
396 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 400 doLinearize = pexConfig.Field(
402 doc=
"Correct for nonlinearity of the detector's response?",
407 doCrosstalk = pexConfig.Field(
409 doc=
"Apply intra-CCD crosstalk correction?",
412 doCrosstalkBeforeAssemble = pexConfig.Field(
414 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
417 crosstalk = pexConfig.ConfigurableField(
418 target=CrosstalkTask,
419 doc=
"Intra-CCD crosstalk correction",
423 doWidenSaturationTrails = pexConfig.Field(
425 doc=
"Widen bleed trails based on their width?",
430 doBrighterFatter = pexConfig.Field(
433 doc=
"Apply the brighter fatter correction" 435 brighterFatterLevel = pexConfig.ChoiceField(
438 doc=
"The level at which to correct for brighter-fatter.",
440 "AMP":
"Every amplifier treated separately.",
441 "DETECTOR":
"One kernel per detector",
444 brighterFatterKernelFile = pexConfig.Field(
447 doc=
"Kernel file used for the brighter fatter correction" 449 brighterFatterMaxIter = pexConfig.Field(
452 doc=
"Maximum number of iterations for the brighter fatter correction" 454 brighterFatterThreshold = pexConfig.Field(
457 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 458 " absolute value of the difference between the current corrected image and the one" 459 " from the previous iteration summed over all the pixels." 461 brighterFatterApplyGain = pexConfig.Field(
464 doc=
"Should the gain be applied when applying the brighter fatter correction?" 468 doDefect = pexConfig.Field(
470 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
473 doSaturationInterpolation = pexConfig.Field(
475 doc=
"Perform interpolation over pixels masked as saturated?",
478 numEdgeSuspect = pexConfig.Field(
480 doc=
"Number of edge pixels to be flagged as untrustworthy.",
485 doDark = pexConfig.Field(
487 doc=
"Apply dark frame correction?",
490 darkDataProductName = pexConfig.Field(
492 doc=
"Name of the dark data product",
497 doStrayLight = pexConfig.Field(
499 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
502 strayLight = pexConfig.ConfigurableField(
503 target=StrayLightTask,
504 doc=
"y-band stray light correction" 508 doFlat = pexConfig.Field(
510 doc=
"Apply flat field correction?",
513 flatDataProductName = pexConfig.Field(
515 doc=
"Name of the flat data product",
518 flatScalingType = pexConfig.ChoiceField(
520 doc=
"The method for scaling the flat on the fly.",
523 "USER":
"Scale by flatUserScale",
524 "MEAN":
"Scale by the inverse of the mean",
525 "MEDIAN":
"Scale by the inverse of the median",
528 flatUserScale = pexConfig.Field(
530 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
533 doTweakFlat = pexConfig.Field(
535 doc=
"Tweak flats to match observed amplifier ratios?",
540 doApplyGains = pexConfig.Field(
542 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
545 normalizeGains = pexConfig.Field(
547 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
552 doFringe = pexConfig.Field(
554 doc=
"Apply fringe correction?",
557 fringe = pexConfig.ConfigurableField(
559 doc=
"Fringe subtraction task",
561 fringeAfterFlat = pexConfig.Field(
563 doc=
"Do fringe subtraction after flat-fielding?",
568 doNanInterpAfterFlat = pexConfig.Field(
570 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 571 "also have to interpolate them before flat-fielding."),
576 doAddDistortionModel = pexConfig.Field(
578 doc=
"Apply a distortion model based on camera geometry to the WCS?",
583 doMeasureBackground = pexConfig.Field(
585 doc=
"Measure the background level on the reduced image?",
590 doCameraSpecificMasking = pexConfig.Field(
592 doc=
"Mask camera-specific bad regions?",
595 masking = pexConfig.ConfigurableField(
601 fluxMag0T1 = pexConfig.DictField(
604 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
605 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
608 defaultFluxMag0T1 = pexConfig.Field(
610 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
611 default=pow(10.0, 0.4*28.0)
615 doVignette = pexConfig.Field(
617 doc=
"Apply vignetting parameters?",
620 vignette = pexConfig.ConfigurableField(
622 doc=
"Vignetting task.",
626 doAttachTransmissionCurve = pexConfig.Field(
629 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 631 doUseOpticsTransmission = pexConfig.Field(
634 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 636 doUseFilterTransmission = pexConfig.Field(
639 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 641 doUseSensorTransmission = pexConfig.Field(
644 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 646 doUseAtmosphereTransmission = pexConfig.Field(
649 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 653 doWrite = pexConfig.Field(
655 doc=
"Persist postISRCCD?",
662 raise ValueError(
"You may not specify both doFlat and doApplyGains")
665 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
666 r"""Apply common instrument signature correction algorithms to a raw frame. 668 The process for correcting imaging data is very similar from 669 camera to camera. This task provides a vanilla implementation of 670 doing these corrections, including the ability to turn certain 671 corrections off if they are not needed. The inputs to the primary 672 method, `run()`, are a raw exposure to be corrected and the 673 calibration data products. The raw input is a single chip sized 674 mosaic of all amps including overscans and other non-science 675 pixels. The method `runDataRef()` identifies and defines the 676 calibration data products, and is intended for use by a 677 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 678 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 679 subclassed for different camera, although the most camera specific 680 methods have been split into subtasks that can be redirected 683 The __init__ method sets up the subtasks for ISR processing, using 684 the defaults from `lsst.ip.isr`. 689 Positional arguments passed to the Task constructor. None used at this time. 690 kwargs : `dict`, optional 691 Keyword arguments passed on to the Task constructor. None used at this time. 693 ConfigClass = IsrTaskConfig
698 self.makeSubtask(
"assembleCcd")
699 self.makeSubtask(
"crosstalk")
700 self.makeSubtask(
"strayLight")
701 self.makeSubtask(
"fringe")
702 self.makeSubtask(
"masking")
703 self.makeSubtask(
"vignette")
712 if config.doBias
is not True:
713 inputTypeDict.pop(
"bias",
None)
714 if config.doLinearize
is not True:
715 inputTypeDict.pop(
"linearizer",
None)
716 if config.doCrosstalk
is not True:
717 inputTypeDict.pop(
"crosstalkSources",
None)
718 if config.doBrighterFatter
is not True:
719 inputTypeDict.pop(
"bfKernel",
None)
720 if config.doDefect
is not True:
721 inputTypeDict.pop(
"defects",
None)
722 if config.doDark
is not True:
723 inputTypeDict.pop(
"dark",
None)
724 if config.doFlat
is not True:
725 inputTypeDict.pop(
"flat",
None)
726 if config.doAttachTransmissionCurve
is not True:
727 inputTypeDict.pop(
"opticsTransmission",
None)
728 inputTypeDict.pop(
"filterTransmission",
None)
729 inputTypeDict.pop(
"sensorTransmission",
None)
730 inputTypeDict.pop(
"atmosphereTransmission",
None)
731 if config.doUseOpticsTransmission
is not True:
732 inputTypeDict.pop(
"opticsTransmission",
None)
733 if config.doUseFilterTransmission
is not True:
734 inputTypeDict.pop(
"filterTransmission",
None)
735 if config.doUseSensorTransmission
is not True:
736 inputTypeDict.pop(
"sensorTransmission",
None)
737 if config.doUseAtmosphereTransmission
is not True:
738 inputTypeDict.pop(
"atmosphereTransmission",
None)
746 if config.qa.doThumbnailOss
is not True:
747 outputTypeDict.pop(
"outputOssThumbnail",
None)
748 if config.qa.doThumbnailFlattened
is not True:
749 outputTypeDict.pop(
"outputFlattenedThumbnail",
None)
750 if config.doWrite
is not True:
751 outputTypeDict.pop(
"outputExposure",
None)
753 return outputTypeDict
757 inputData[
'detectorNum'] = int(inputDataIds[
'ccdExposure'][
'detector'])
758 except Exception
as e:
759 raise ValueError(f
"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
761 inputData[
'isGen3'] =
True 763 if self.config.doLinearize
is True:
764 if 'linearizer' not in inputData.keys():
765 detector = inputData[
'camera'][inputData[
'detectorNum']]
766 linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
767 inputData[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
769 if inputData[
'defects']
is not None:
774 for r
in inputData[
'defects']:
775 bbox = afwGeom.BoxI(afwGeom.PointI(r.get(
"x0"), r.get(
"y0")),
776 afwGeom.ExtentI(r.get(
"width"), r.get(
"height")))
777 defectList.append(
Defect(bbox))
779 inputData[
'defects'] = defectList
796 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
802 """!Retrieve necessary frames for instrument signature removal. 804 Pre-fetching all required ISR data products limits the IO 805 required by the ISR. Any conflict between the calibration data 806 available and that needed for ISR is also detected prior to 807 doing processing, allowing it to fail quickly. 811 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 812 Butler reference of the detector data to be processed 813 rawExposure : `afw.image.Exposure` 814 The raw exposure that will later be corrected with the 815 retrieved calibration data; should not be modified in this 820 result : `lsst.pipe.base.Struct` 821 Result struct with components (which may be `None`): 822 - ``bias``: bias calibration frame (`afw.image.Exposure`) 823 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 824 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 825 - ``dark``: dark calibration frame (`afw.image.Exposure`) 826 - ``flat``: flat calibration frame (`afw.image.Exposure`) 827 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 828 - ``defects``: list of defects (`list`) 829 - ``fringes``: `lsst.pipe.base.Struct` with components: 830 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 831 - ``seed``: random seed derived from the ccdExposureId for random 832 number generator (`uint32`) 833 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 834 A ``TransmissionCurve`` that represents the throughput of the optics, 835 to be evaluated in focal-plane coordinates. 836 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 837 A ``TransmissionCurve`` that represents the throughput of the filter 838 itself, to be evaluated in focal-plane coordinates. 839 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 840 A ``TransmissionCurve`` that represents the throughput of the sensor 841 itself, to be evaluated in post-assembly trimmed detector coordinates. 842 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 843 A ``TransmissionCurve`` that represents the throughput of the 844 atmosphere, assumed to be spatially constant. 847 ccd = rawExposure.getDetector()
848 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
849 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
850 if self.config.doBias
else None)
852 linearizer = (dataRef.get(
"linearizer", immediate=
True)
854 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
855 if self.config.doCrosstalk
else None)
856 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
857 if self.config.doDark
else None)
858 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
859 if self.config.doFlat
else None)
860 brighterFatterKernel = (dataRef.get(
"bfKernel")
861 if self.config.doBrighterFatter
else None)
862 defectList = (dataRef.get(
"defects")
863 if self.config.doDefect
else None)
864 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
865 if self.config.doAssembleIsrExposures
else None)
866 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
867 else pipeBase.Struct(fringes=
None))
869 if self.config.doAttachTransmissionCurve:
870 opticsTransmission = (dataRef.get(
"transmission_optics")
871 if self.config.doUseOpticsTransmission
else None)
872 filterTransmission = (dataRef.get(
"transmission_filter")
873 if self.config.doUseFilterTransmission
else None)
874 sensorTransmission = (dataRef.get(
"transmission_sensor")
875 if self.config.doUseSensorTransmission
else None)
876 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
877 if self.config.doUseAtmosphereTransmission
else None)
879 opticsTransmission =
None 880 filterTransmission =
None 881 sensorTransmission =
None 882 atmosphereTransmission =
None 885 return pipeBase.Struct(bias=biasExposure,
886 linearizer=linearizer,
887 crosstalkSources=crosstalkSources,
890 bfKernel=brighterFatterKernel,
892 fringes=fringeStruct,
893 opticsTransmission=opticsTransmission,
894 filterTransmission=filterTransmission,
895 sensorTransmission=sensorTransmission,
896 atmosphereTransmission=atmosphereTransmission,
900 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
901 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
902 opticsTransmission=None, filterTransmission=None,
903 sensorTransmission=None, atmosphereTransmission=None,
904 detectorNum=None, isGen3=False
906 """!Perform instrument signature removal on an exposure. 908 Steps included in the ISR processing, in order performed, are: 909 - saturation and suspect pixel masking 910 - overscan subtraction 911 - CCD assembly of individual amplifiers 913 - variance image construction 914 - linearization of non-linear response 916 - brighter-fatter correction 919 - stray light subtraction 921 - masking of known defects and camera specific features 922 - vignette calculation 923 - appending transmission curve and distortion model 927 ccdExposure : `lsst.afw.image.Exposure` 928 The raw exposure that is to be run through ISR. The 929 exposure is modified by this method. 930 camera : `lsst.afw.cameraGeom.Camera`, optional 931 The camera geometry for this exposure. Used to select the 932 distortion model appropriate for this data. 933 bias : `lsst.afw.image.Exposure`, optional 934 Bias calibration frame. 935 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 936 Functor for linearization. 937 crosstalkSources : `list`, optional 938 List of possible crosstalk sources. 939 dark : `lsst.afw.image.Exposure`, optional 940 Dark calibration frame. 941 flat : `lsst.afw.image.Exposure`, optional 942 Flat calibration frame. 943 bfKernel : `numpy.ndarray`, optional 944 Brighter-fatter kernel. 945 defects : `list`, optional 947 fringes : `lsst.pipe.base.Struct`, optional 948 Struct containing the fringe correction data, with 950 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 951 - ``seed``: random seed derived from the ccdExposureId for random 952 number generator (`uint32`) 953 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 954 A ``TransmissionCurve`` that represents the throughput of the optics, 955 to be evaluated in focal-plane coordinates. 956 filterTransmission : `lsst.afw.image.TransmissionCurve` 957 A ``TransmissionCurve`` that represents the throughput of the filter 958 itself, to be evaluated in focal-plane coordinates. 959 sensorTransmission : `lsst.afw.image.TransmissionCurve` 960 A ``TransmissionCurve`` that represents the throughput of the sensor 961 itself, to be evaluated in post-assembly trimmed detector coordinates. 962 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 963 A ``TransmissionCurve`` that represents the throughput of the 964 atmosphere, assumed to be spatially constant. 965 detectorNum : `int`, optional 966 The integer number for the detector to process. 967 isGen3 : bool, optional 968 Flag this call to run() as using the Gen3 butler environment. 972 result : `lsst.pipe.base.Struct` 973 Result struct with component: 974 - ``exposure`` : `afw.image.Exposure` 975 The fully ISR corrected exposure. 976 - ``outputExposure`` : `afw.image.Exposure` 977 An alias for `exposure` 978 - ``ossThumb`` : `numpy.ndarray` 979 Thumbnail image of the exposure after overscan subtraction. 980 - ``flattenedThumb`` : `numpy.ndarray` 981 Thumbnail image of the exposure after flat-field correction. 986 Raised if a configuration option is set to True, but the 987 required calibration data has not been specified. 991 The current processed exposure can be viewed by setting the 992 appropriate lsstDebug entries in the `debug.display` 993 dictionary. The names of these entries correspond to some of 994 the IsrTaskConfig Boolean options, with the value denoting the 995 frame to use. The exposure is shown inside the matching 996 option check and after the processing of that step has 997 finished. The steps with debug points are: 1008 In addition, setting the "postISRCCD" entry displays the 1009 exposure after all ISR processing has finished. 1017 self.config.doFringe =
False 1020 if detectorNum
is None:
1021 raise RuntimeError(
"Must supply the detectorNum if running as Gen3")
1023 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1028 if isinstance(ccdExposure, ButlerDataRef):
1031 ccd = ccdExposure.getDetector()
1034 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 1035 ccd = [
FakeAmp(ccdExposure, self.config)]
1038 if self.config.doBias
and bias
is None:
1039 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1041 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1042 if self.config.doBrighterFatter
and bfKernel
is None:
1043 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1044 if self.config.doDark
and dark
is None:
1045 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1047 fringes = pipeBase.Struct(fringes=
None)
1048 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
1049 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1050 if self.config.doFlat
and flat
is None:
1051 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1052 if self.config.doDefect
and defects
is None:
1053 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1054 if self.config.doAddDistortionModel
and camera
is None:
1055 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
1058 if self.config.doConvertIntToFloat:
1059 self.log.info(
"Converting exposure to floating point values")
1066 if ccdExposure.getBBox().contains(amp.getBBox()):
1070 if self.config.doOverscan
and not badAmp:
1073 self.log.debug(
"Corrected overscan for amplifier %s" % (amp.getName()))
1074 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1075 if isinstance(overscanResults.overscanFit, float):
1076 qaMedian = overscanResults.overscanFit
1077 qaStdev = float(
"NaN")
1079 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1080 afwMath.MEDIAN | afwMath.STDEVCLIP)
1081 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1082 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1084 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1085 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1086 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f" %
1087 (amp.getName(), qaMedian, qaStdev))
1088 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1090 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
1091 overscanResults =
None 1093 overscans.append(overscanResults
if overscanResults
is not None else None)
1095 self.log.info(
"Skipped OSCAN")
1097 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1098 self.log.info(
"Applying crosstalk correction.")
1099 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1100 self.
debugView(ccdExposure,
"doCrosstalk")
1102 if self.config.doAssembleCcd:
1103 self.log.info(
"Assembling CCD from amplifiers")
1104 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1106 if self.config.expectWcs
and not ccdExposure.getWcs():
1107 self.log.warn(
"No WCS found in input exposure")
1108 self.
debugView(ccdExposure,
"doAssembleCcd")
1111 if self.config.qa.doThumbnailOss:
1112 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1114 if self.config.doBias:
1115 self.log.info(
"Applying bias correction.")
1116 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1117 trimToFit=self.config.doTrimToMatchCalib)
1120 if self.config.doVariance:
1121 for amp, overscanResults
in zip(ccd, overscans):
1122 if ccdExposure.getBBox().contains(amp.getBBox()):
1123 self.log.debug(
"Constructing variance map for amplifer %s" % (amp.getName()))
1124 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1125 if overscanResults
is not None:
1127 overscanImage=overscanResults.overscanImage)
1131 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1132 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1133 afwMath.MEDIAN | afwMath.STDEVCLIP)
1134 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
1135 qaStats.getValue(afwMath.MEDIAN))
1136 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
1137 qaStats.getValue(afwMath.STDEVCLIP))
1138 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f" %
1139 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1140 qaStats.getValue(afwMath.STDEVCLIP)))
1143 self.log.info(
"Applying linearizer.")
1144 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1146 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1147 self.log.info(
"Applying crosstalk correction.")
1148 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1149 self.
debugView(ccdExposure,
"doCrosstalk")
1151 if self.config.doWidenSaturationTrails:
1152 self.log.info(
"Widening saturation trails.")
1153 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1155 interpolationDone =
False 1156 if self.config.doBrighterFatter:
1162 if self.config.doDefect:
1165 if self.config.doSaturationInterpolation:
1169 interpolationDone =
True 1171 if self.config.brighterFatterLevel ==
'DETECTOR':
1172 kernelElement = bfKernel
1175 raise NotImplementedError(
"per-amplifier brighter-fatter correction not yet implemented")
1176 self.log.info(
"Applying brighter fatter correction.")
1177 isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
1178 self.config.brighterFatterMaxIter,
1179 self.config.brighterFatterThreshold,
1180 self.config.brighterFatterApplyGain,
1182 self.
debugView(ccdExposure,
"doBrighterFatter")
1184 if self.config.doDark:
1185 self.log.info(
"Applying dark correction.")
1189 if self.config.doFringe
and not self.config.fringeAfterFlat:
1190 self.log.info(
"Applying fringe correction before flat.")
1191 self.fringe.
run(ccdExposure, **fringes.getDict())
1194 if self.config.doStrayLight:
1195 self.log.info(
"Applying stray light correction.")
1196 self.strayLight.
run(ccdExposure)
1197 self.
debugView(ccdExposure,
"doStrayLight")
1199 if self.config.doFlat:
1200 self.log.info(
"Applying flat correction.")
1204 if self.config.doApplyGains:
1205 self.log.info(
"Applying gain correction instead of flat.")
1206 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1208 if self.config.doDefect
and not interpolationDone:
1209 self.log.info(
"Masking and interpolating defects.")
1212 if self.config.doSaturation
and not interpolationDone:
1213 self.log.info(
"Interpolating saturated pixels.")
1216 if self.config.doNanInterpAfterFlat
or not interpolationDone:
1217 self.log.info(
"Masking and interpolating NAN value pixels.")
1220 if self.config.doFringe
and self.config.fringeAfterFlat:
1221 self.log.info(
"Applying fringe correction after flat.")
1222 self.fringe.
run(ccdExposure, **fringes.getDict())
1224 if self.config.doSetBadRegions:
1225 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1226 if badPixelCount > 0:
1227 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1229 flattenedThumb =
None 1230 if self.config.qa.doThumbnailFlattened:
1231 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1233 if self.config.doCameraSpecificMasking:
1234 self.log.info(
"Masking regions for camera specific reasons.")
1235 self.masking.
run(ccdExposure)
1239 if self.config.doVignette:
1240 self.log.info(
"Constructing Vignette polygon.")
1243 if self.config.vignette.doWriteVignettePolygon:
1246 if self.config.doAttachTransmissionCurve:
1247 self.log.info(
"Adding transmission curves.")
1248 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1249 filterTransmission=filterTransmission,
1250 sensorTransmission=sensorTransmission,
1251 atmosphereTransmission=atmosphereTransmission)
1253 if self.config.doAddDistortionModel:
1254 self.log.info(
"Adding a distortion model to the WCS.")
1255 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1257 if self.config.doMeasureBackground:
1258 self.log.info(
"Measuring background level:")
1261 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1263 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1264 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1265 afwMath.MEDIAN | afwMath.STDEVCLIP)
1266 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1267 qaStats.getValue(afwMath.MEDIAN))
1268 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1269 qaStats.getValue(afwMath.STDEVCLIP))
1270 self.log.debug(
" Background stats for amplifer %s: %f +/- %f" %
1271 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1272 qaStats.getValue(afwMath.STDEVCLIP)))
1274 self.
debugView(ccdExposure,
"postISRCCD")
1276 return pipeBase.Struct(
1277 exposure=ccdExposure,
1279 flattenedThumb=flattenedThumb,
1281 outputExposure=ccdExposure,
1282 outputOssThumbnail=ossThumb,
1283 outputFlattenedThumbnail=flattenedThumb,
1286 @pipeBase.timeMethod
1288 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1290 This method contains the `CmdLineTask` interface to the ISR 1291 processing. All IO is handled here, freeing the `run()` method 1292 to manage only pixel-level calculations. The steps performed 1294 - Read in necessary detrending/isr/calibration data. 1295 - Process raw exposure in `run()`. 1296 - Persist the ISR-corrected exposure as "postISRCCD" if 1297 config.doWrite=True. 1301 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1302 DataRef of the detector data to be processed 1306 result : `lsst.pipe.base.Struct` 1307 Result struct with component: 1308 - ``exposure`` : `afw.image.Exposure` 1309 The fully ISR corrected exposure. 1314 Raised if a configuration option is set to True, but the 1315 required calibration data does not exist. 1318 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1320 ccdExposure = sensorRef.get(self.config.datasetType)
1322 camera = sensorRef.get(
"camera")
1323 if camera
is None and self.config.doAddDistortionModel:
1324 raise RuntimeError(
"config.doAddDistortionModel is True " 1325 "but could not get a camera from the butler")
1326 isrData = self.
readIsrData(sensorRef, ccdExposure)
1328 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1330 if self.config.doWrite:
1331 sensorRef.put(result.exposure,
"postISRCCD")
1332 if result.ossThumb
is not None:
1333 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1334 if result.flattenedThumb
is not None:
1335 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1340 """!Retrieve a calibration dataset for removing instrument signature. 1345 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1346 DataRef of the detector data to find calibration datasets 1349 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1351 If True, disable butler proxies to enable error handling 1352 within this routine. 1356 exposure : `lsst.afw.image.Exposure` 1357 Requested calibration frame. 1362 Raised if no matching calibration frame can be found. 1365 exp = dataRef.get(datasetType, immediate=immediate)
1366 except Exception
as exc1:
1367 if not self.config.fallbackFilterName:
1368 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1370 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1371 except Exception
as exc2:
1372 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1373 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1374 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1376 if self.config.doAssembleIsrExposures:
1377 exp = self.assembleCcd.assembleCcd(exp)
1381 """Ensure that the data returned by Butler is a fully constructed exposure. 1383 ISR requires exposure-level image data for historical reasons, so if we did 1384 not recieve that from Butler, construct it from what we have, modifying the 1389 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1390 `lsst.afw.image.ImageF` 1391 The input data structure obtained from Butler. 1392 camera : `lsst.afw.cameraGeom.camera` 1393 The camera associated with the image. Used to find the appropriate 1396 The detector this exposure should match. 1400 inputExp : `lsst.afw.image.Exposure` 1401 The re-constructed exposure, with appropriate detector parameters. 1406 Raised if the input data cannot be used to construct an exposure. 1408 if isinstance(inputExp, afwImage.DecoratedImageU):
1409 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1410 elif isinstance(inputExp, afwImage.ImageF):
1411 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1412 elif isinstance(inputExp, afwImage.MaskedImageF):
1413 inputExp = afwImage.makeExposure(inputExp)
1414 elif isinstance(inputExp, afwImage.Exposure):
1417 raise TypeError(f
"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1419 if inputExp.getDetector()
is None:
1420 inputExp.setDetector(camera[detectorNum])
1425 """Convert exposure image from uint16 to float. 1427 If the exposure does not need to be converted, the input is 1428 immediately returned. For exposures that are converted to use 1429 floating point pixels, the variance is set to unity and the 1434 exposure : `lsst.afw.image.Exposure` 1435 The raw exposure to be converted. 1439 newexposure : `lsst.afw.image.Exposure` 1440 The input ``exposure``, converted to floating point pixels. 1445 Raised if the exposure type cannot be converted to float. 1448 if isinstance(exposure, afwImage.ExposureF):
1451 if not hasattr(exposure,
"convertF"):
1452 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1454 newexposure = exposure.convertF()
1455 newexposure.variance[:] = 1
1456 newexposure.mask[:] = 0x0
1461 """Identify bad amplifiers, saturated and suspect pixels. 1465 ccdExposure : `lsst.afw.image.Exposure` 1466 Input exposure to be masked. 1467 amp : `lsst.afw.table.AmpInfoCatalog` 1468 Catalog of parameters defining the amplifier on this 1471 List of defects. Used to determine if the entire 1477 If this is true, the entire amplifier area is covered by 1478 defects and unusable. 1481 maskedImage = ccdExposure.getMaskedImage()
1487 if defects
is not None:
1488 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1493 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1495 maskView = dataView.getMask()
1496 maskView |= maskView.getPlaneBitMask(
"BAD")
1503 if self.config.doSaturation
and not badAmp:
1504 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1505 if self.config.doSuspect
and not badAmp:
1506 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1508 for maskName, maskThreshold
in limits.items():
1509 if not math.isnan(maskThreshold):
1510 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1511 isrFunctions.makeThresholdMask(
1512 maskedImage=dataView,
1513 threshold=maskThreshold,
1519 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1521 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1522 self.config.suspectMaskName])
1523 if numpy.all(maskView.getArray() & maskVal > 0):
1529 """Apply overscan correction in place. 1531 This method does initial pixel rejection of the overscan 1532 region. The overscan can also be optionally segmented to 1533 allow for discontinuous overscan responses to be fit 1534 separately. The actual overscan subtraction is performed by 1535 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1536 which is called here after the amplifier is preprocessed. 1540 ccdExposure : `lsst.afw.image.Exposure` 1541 Exposure to have overscan correction performed. 1542 amp : `lsst.afw.table.AmpInfoCatalog` 1543 The amplifier to consider while correcting the overscan. 1547 overscanResults : `lsst.pipe.base.Struct` 1548 Result struct with components: 1549 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1550 Value or fit subtracted from the amplifier image data. 1551 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1552 Value or fit subtracted from the overscan image data. 1553 - ``overscanImage`` : `lsst.afw.image.Image` 1554 Image of the overscan region with the overscan 1555 correction applied. This quantity is used to estimate 1556 the amplifier read noise empirically. 1561 Raised if the ``amp`` does not contain raw pixel information. 1565 lsst.ip.isr.isrFunctions.overscanCorrection 1567 if not amp.getHasRawInfo():
1568 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1570 if amp.getRawHorizontalOverscanBBox().isEmpty():
1571 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1574 statControl = afwMath.StatisticsControl()
1575 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1578 dataBBox = amp.getRawDataBBox()
1579 oscanBBox = amp.getRawHorizontalOverscanBBox()
1583 prescanBBox = amp.getRawPrescanBBox()
1584 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1585 dx0 += self.config.overscanNumLeadingColumnsToSkip
1586 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1588 dx0 += self.config.overscanNumTrailingColumnsToSkip
1589 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1595 if ((self.config.overscanBiasJump
and 1596 self.config.overscanBiasJumpLocation)
and 1597 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1598 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1599 self.config.overscanBiasJumpDevices)):
1600 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1601 yLower = self.config.overscanBiasJumpLocation
1602 yUpper = dataBBox.getHeight() - yLower
1604 yUpper = self.config.overscanBiasJumpLocation
1605 yLower = dataBBox.getHeight() - yUpper
1607 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1608 afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1609 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1610 afwGeom.Extent2I(dx0, 0),
1611 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1614 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1615 afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1616 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1617 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1620 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1621 afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1622 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1623 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1624 oscanBBox.getHeight())))
1627 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1628 ampImage = ccdExposure.maskedImage[imageBBox]
1629 overscanImage = ccdExposure.maskedImage[overscanBBox]
1631 overscanArray = overscanImage.image.array
1632 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1633 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1634 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1636 statControl = afwMath.StatisticsControl()
1637 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1639 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1640 overscanImage=overscanImage,
1641 fitType=self.config.overscanFitType,
1642 order=self.config.overscanOrder,
1643 collapseRej=self.config.overscanNumSigmaClip,
1644 statControl=statControl,
1645 overscanIsInt=self.config.overscanIsInt
1649 levelStat = afwMath.MEDIAN
1650 sigmaStat = afwMath.STDEVCLIP
1652 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1653 self.config.qa.flatness.nIter)
1654 metadata = ccdExposure.getMetadata()
1655 ampNum = amp.getName()
1656 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1657 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1658 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1660 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1661 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1662 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1664 return overscanResults
1667 """Set the variance plane using the amplifier gain and read noise 1669 The read noise is calculated from the ``overscanImage`` if the 1670 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1671 the value from the amplifier data is used. 1675 ampExposure : `lsst.afw.image.Exposure` 1676 Exposure to process. 1677 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1678 Amplifier detector data. 1679 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1680 Image of overscan, required only for empirical read noise. 1684 lsst.ip.isr.isrFunctions.updateVariance 1686 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1687 gain = amp.getGain()
1689 if math.isnan(gain):
1691 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1694 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1695 (amp.getName(), gain, patchedGain))
1698 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1699 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1701 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1702 stats = afwMath.StatisticsControl()
1703 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1704 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1705 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1707 readNoise = amp.getReadNoise()
1709 isrFunctions.updateVariance(
1710 maskedImage=ampExposure.getMaskedImage(),
1712 readNoise=readNoise,
1716 """!Apply dark correction in place. 1720 exposure : `lsst.afw.image.Exposure` 1721 Exposure to process. 1722 darkExposure : `lsst.afw.image.Exposure` 1723 Dark exposure of the same size as ``exposure``. 1724 invert : `Bool`, optional 1725 If True, re-add the dark to an already corrected image. 1730 Raised if either ``exposure`` or ``darkExposure`` do not 1731 have their dark time defined. 1735 lsst.ip.isr.isrFunctions.darkCorrection 1737 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1738 if math.isnan(expScale):
1739 raise RuntimeError(
"Exposure darktime is NAN")
1740 if darkExposure.getInfo().getVisitInfo()
is not None:
1741 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1747 if math.isnan(darkScale):
1748 raise RuntimeError(
"Dark calib darktime is NAN")
1749 isrFunctions.darkCorrection(
1750 maskedImage=exposure.getMaskedImage(),
1751 darkMaskedImage=darkExposure.getMaskedImage(),
1753 darkScale=darkScale,
1755 trimToFit=self.config.doTrimToMatchCalib
1759 """!Check if linearization is needed for the detector cameraGeom. 1761 Checks config.doLinearize and the linearity type of the first 1766 detector : `lsst.afw.cameraGeom.Detector` 1767 Detector to get linearity type from. 1771 doLinearize : `Bool` 1772 If True, linearization should be performed. 1774 return self.config.doLinearize
and \
1775 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1778 """!Apply flat correction in place. 1782 exposure : `lsst.afw.image.Exposure` 1783 Exposure to process. 1784 flatExposure : `lsst.afw.image.Exposure` 1785 Flat exposure of the same size as ``exposure``. 1786 invert : `Bool`, optional 1787 If True, unflatten an already flattened image. 1791 lsst.ip.isr.isrFunctions.flatCorrection 1793 isrFunctions.flatCorrection(
1794 maskedImage=exposure.getMaskedImage(),
1795 flatMaskedImage=flatExposure.getMaskedImage(),
1796 scalingType=self.config.flatScalingType,
1797 userScale=self.config.flatUserScale,
1799 trimToFit=self.config.doTrimToMatchCalib
1803 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1807 exposure : `lsst.afw.image.Exposure` 1808 Exposure to process. Only the amplifier DataSec is processed. 1809 amp : `lsst.afw.table.AmpInfoCatalog` 1810 Amplifier detector data. 1814 lsst.ip.isr.isrFunctions.makeThresholdMask 1816 if not math.isnan(amp.getSaturation()):
1817 maskedImage = exposure.getMaskedImage()
1818 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1819 isrFunctions.makeThresholdMask(
1820 maskedImage=dataView,
1821 threshold=amp.getSaturation(),
1823 maskName=self.config.saturatedMaskName,
1827 """!Interpolate over saturated pixels, in place. 1829 This method should be called after `saturationDetection`, to 1830 ensure that the saturated pixels have been identified in the 1831 SAT mask. It should also be called after `assembleCcd`, since 1832 saturated regions may cross amplifier boundaries. 1836 exposure : `lsst.afw.image.Exposure` 1837 Exposure to process. 1841 lsst.ip.isr.isrTask.saturationDetection 1842 lsst.ip.isr.isrFunctions.interpolateFromMask 1844 isrFunctions.interpolateFromMask(
1845 maskedImage=ccdExposure.getMaskedImage(),
1846 fwhm=self.config.fwhm,
1847 growFootprints=self.config.growSaturationFootprintSize,
1848 maskName=self.config.saturatedMaskName,
1852 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1856 exposure : `lsst.afw.image.Exposure` 1857 Exposure to process. Only the amplifier DataSec is processed. 1858 amp : `lsst.afw.table.AmpInfoCatalog` 1859 Amplifier detector data. 1863 lsst.ip.isr.isrFunctions.makeThresholdMask 1867 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1868 This is intended to indicate pixels that may be affected by unknown systematics; 1869 for example if non-linearity corrections above a certain level are unstable 1870 then that would be a useful value for suspectLevel. A value of `nan` indicates 1871 that no such level exists and no pixels are to be masked as suspicious. 1873 suspectLevel = amp.getSuspectLevel()
1874 if math.isnan(suspectLevel):
1877 maskedImage = exposure.getMaskedImage()
1878 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1879 isrFunctions.makeThresholdMask(
1880 maskedImage=dataView,
1881 threshold=suspectLevel,
1883 maskName=self.config.suspectMaskName,
1887 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1891 ccdExposure : `lsst.afw.image.Exposure` 1892 Exposure to process. 1893 defectBaseList : `List` 1894 List of defects to mask and interpolate. 1898 Call this after CCD assembly, since defects may cross amplifier boundaries. 1900 maskedImage = ccdExposure.getMaskedImage()
1902 for d
in defectBaseList:
1904 nd = measAlg.Defect(bbox)
1905 defectList.append(nd)
1906 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
1907 isrFunctions.interpolateDefectList(
1908 maskedImage=maskedImage,
1909 defectList=defectList,
1910 fwhm=self.config.fwhm,
1913 if self.config.numEdgeSuspect > 0:
1914 goodBBox = maskedImage.getBBox()
1916 goodBBox.grow(-self.config.numEdgeSuspect)
1918 SourceDetectionTask.setEdgeBits(
1921 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1925 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1929 exposure : `lsst.afw.image.Exposure` 1930 Exposure to process. 1934 We mask and interpolate over all NaNs, including those 1935 that are masked with other bits (because those may or may 1936 not be interpolated over later, and we want to remove all 1937 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1938 is used to preserve the historical name. 1940 maskedImage = exposure.getMaskedImage()
1943 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
1944 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
1945 numNans =
maskNans(maskedImage, maskVal)
1946 self.metadata.set(
"NUMNANS", numNans)
1950 self.log.warn(
"There were %i unmasked NaNs", numNans)
1951 nanDefectList = isrFunctions.getDefectListFromMask(
1952 maskedImage=maskedImage,
1953 maskName=
'UNMASKEDNAN',
1955 isrFunctions.interpolateDefectList(
1956 maskedImage=exposure.getMaskedImage(),
1957 defectList=nanDefectList,
1958 fwhm=self.config.fwhm,
1962 """Measure the image background in subgrids, for quality control purposes. 1966 exposure : `lsst.afw.image.Exposure` 1967 Exposure to process. 1968 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 1969 Configuration object containing parameters on which background 1970 statistics and subgrids to use. 1972 if IsrQaConfig
is not None:
1973 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1974 IsrQaConfig.flatness.nIter)
1975 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
1976 statsControl.setAndMask(maskVal)
1977 maskedImage = exposure.getMaskedImage()
1978 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
1979 skyLevel = stats.getValue(afwMath.MEDIAN)
1980 skySigma = stats.getValue(afwMath.STDEVCLIP)
1981 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
1982 metadata = exposure.getMetadata()
1983 metadata.set(
'SKYLEVEL', skyLevel)
1984 metadata.set(
'SKYSIGMA', skySigma)
1987 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
1988 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
1989 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
1990 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
1991 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
1992 skyLevels = numpy.zeros((nX, nY))
1995 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
1997 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
1999 xLLC = xc - meshXHalf
2000 yLLC = yc - meshYHalf
2001 xURC = xc + meshXHalf - 1
2002 yURC = yc + meshYHalf - 1
2004 bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2005 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2007 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2009 good = numpy.where(numpy.isfinite(skyLevels))
2010 skyMedian = numpy.median(skyLevels[good])
2011 flatness = (skyLevels[good] - skyMedian) / skyMedian
2012 flatness_rms = numpy.std(flatness)
2013 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2015 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2016 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
2017 (nX, nY, flatness_pp, flatness_rms))
2019 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2020 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2021 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2022 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2023 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2026 """Set an approximate magnitude zero point for the exposure. 2030 exposure : `lsst.afw.image.Exposure` 2031 Exposure to process. 2033 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2034 if filterName
in self.config.fluxMag0T1:
2035 fluxMag0 = self.config.fluxMag0T1[filterName]
2037 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
2038 fluxMag0 = self.config.defaultFluxMag0T1
2040 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2042 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
2045 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2046 exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2049 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2053 ccdExposure : `lsst.afw.image.Exposure` 2054 Exposure to process. 2055 fpPolygon : `lsst.afw.geom.Polygon` 2056 Polygon in focal plane coordinates. 2059 ccd = ccdExposure.getDetector()
2060 fpCorners = ccd.getCorners(FOCAL_PLANE)
2061 ccdPolygon = Polygon(fpCorners)
2064 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2067 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2068 validPolygon = Polygon(ccdPoints)
2069 ccdExposure.getInfo().setValidPolygon(validPolygon)
2073 """Context manager that applies and removes flats and darks, 2074 if the task is configured to apply them. 2078 exp : `lsst.afw.image.Exposure` 2079 Exposure to process. 2080 flat : `lsst.afw.image.Exposure` 2081 Flat exposure the same size as ``exp``. 2082 dark : `lsst.afw.image.Exposure`, optional 2083 Dark exposure the same size as ``exp``. 2087 exp : `lsst.afw.image.Exposure` 2088 The flat and dark corrected exposure. 2090 if self.config.doDark
and dark
is not None:
2092 if self.config.doFlat:
2097 if self.config.doFlat:
2099 if self.config.doDark
and dark
is not None:
2103 """Utility function to examine ISR exposure at different stages. 2107 exposure : `lsst.afw.image.Exposure` 2110 State of processing to view. 2112 frame = getDebugFrame(self._display, stepname)
2114 display = getDisplay(frame)
2115 display.scale(
'asinh',
'zscale')
2116 display.mtv(exposure)
2120 """A Detector-like object that supports returning gain and saturation level 2122 This is used when the input exposure does not have a detector. 2126 exposure : `lsst.afw.image.Exposure` 2127 Exposure to generate a fake amplifier for. 2128 config : `lsst.ip.isr.isrTaskConfig` 2129 Configuration to apply to the fake amplifier. 2133 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2135 self.
_gain = config.gain
2165 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2169 """Task to wrap the default IsrTask to allow it to be retargeted. 2171 The standard IsrTask can be called directly from a command line 2172 program, but doing so removes the ability of the task to be 2173 retargeted. As most cameras override some set of the IsrTask 2174 methods, this would remove those data-specific methods in the 2175 output post-ISR images. This wrapping class fixes the issue, 2176 allowing identical post-ISR images to be generated by both the 2177 processCcd and isrTask code. 2179 ConfigClass = RunIsrConfig
2180 _DefaultName =
"runIsr" 2184 self.makeSubtask(
"isr")
2190 dataRef : `lsst.daf.persistence.ButlerDataRef` 2191 data reference of the detector data to be processed 2195 result : `pipeBase.Struct` 2196 Result struct with component: 2198 - exposure : `lsst.afw.image.Exposure` 2199 Post-ISR processed exposure. def getInputDatasetTypes(cls, config)
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def getOutputDatasetTypes(cls, config)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def makeDatasetType(self, dsConfig)
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)