33 from contextlib
import contextmanager
34 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .straylight
import StrayLightTask
54 from .vignette
import VignetteTask
56 __all__ = [
"IsrTask",
"RunIsrTask"]
60 """Configuration parameters for IsrTask. 62 Items are grouped in the order in which they are executed by the task. 67 isrName = pexConfig.Field(
74 ccdExposure = pipeBase.InputDatasetField(
75 doc=
"Input exposure to process",
78 storageClass=
"ExposureU",
79 dimensions=[
"Instrument",
"Exposure",
"Detector"],
81 camera = pipeBase.InputDatasetField(
82 doc=
"Input camera to construct complete exposures.",
85 storageClass=
"TablePersistableCamera",
86 dimensions=[
"Instrument",
"CalibrationLabel"],
88 bias = pipeBase.InputDatasetField(
89 doc=
"Input bias calibration.",
92 storageClass=
"ImageF",
93 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
95 dark = pipeBase.InputDatasetField(
96 doc=
"Input dark calibration.",
99 storageClass=
"ImageF",
100 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
102 flat = pipeBase.InputDatasetField(
103 doc=
"Input flat calibration.",
106 storageClass=
"MaskedImageF",
107 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel",
"Detector"],
109 bfKernel = pipeBase.InputDatasetField(
110 doc=
"Input brighter-fatter kernel.",
113 storageClass=
"NumpyArray",
114 dimensions=[
"Instrument",
"CalibrationLabel"],
116 defects = pipeBase.InputDatasetField(
117 doc=
"Input defect tables.",
120 storageClass=
"Catalog",
121 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
123 opticsTransmission = pipeBase.InputDatasetField(
124 doc=
"Transmission curve due to the optics.",
125 name=
"transmission_optics",
127 storageClass=
"TablePersistableTransmissionCurve",
128 dimensions=[
"Instrument",
"CalibrationLabel"],
130 filterTransmission = pipeBase.InputDatasetField(
131 doc=
"Transmission curve due to the filter.",
132 name=
"transmission_filter",
134 storageClass=
"TablePersistableTransmissionCurve",
135 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel"],
137 sensorTransmission = pipeBase.InputDatasetField(
138 doc=
"Transmission curve due to the sensor.",
139 name=
"transmission_sensor",
141 storageClass=
"TablePersistableTransmissionCurve",
142 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
144 atmosphereTransmission = pipeBase.InputDatasetField(
145 doc=
"Transmission curve due to the atmosphere.",
146 name=
"transmission_atmosphere",
148 storageClass=
"TablePersistableTransmissionCurve",
149 dimensions=[
"Instrument"],
153 outputExposure = pipeBase.OutputDatasetField(
154 doc=
"Output ISR processed exposure.",
157 storageClass=
"ExposureF",
158 dimensions=[
"Instrument",
"Visit",
"Detector"],
160 outputOssThumbnail = pipeBase.OutputDatasetField(
161 doc=
"Output Overscan-subtracted thumbnail image.",
164 storageClass=
"Thumbnail",
165 dimensions=[
"Instrument",
"Visit",
"Detector"],
167 outputFlattenedThumbnail = pipeBase.OutputDatasetField(
168 doc=
"Output flat-corrected thumbnail image.",
169 name=
"FlattenedThumb",
171 storageClass=
"TextStorage",
172 dimensions=[
"Instrument",
"Visit",
"Detector"],
175 quantum = pipeBase.QuantumConfig(
176 dimensions=[
"Visit",
"Detector",
"Instrument"],
180 datasetType = pexConfig.Field(
182 doc=
"Dataset type for input data; users will typically leave this alone, " 183 "but camera-specific ISR tasks will override it",
187 fallbackFilterName = pexConfig.Field(
189 doc=
"Fallback default filter name for calibrations.",
192 expectWcs = pexConfig.Field(
195 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 197 fwhm = pexConfig.Field(
199 doc=
"FWHM of PSF in arcseconds.",
202 qa = pexConfig.ConfigField(
204 doc=
"QA related configuration options.",
208 doConvertIntToFloat = pexConfig.Field(
210 doc=
"Convert integer raw images to floating point values?",
215 doSaturation = pexConfig.Field(
217 doc=
"Mask saturated pixels? NB: this is totally independent of the" 218 " interpolation option - this is ONLY setting the bits in the mask." 219 " To have them interpolated make sure doSaturationInterpolation=True",
222 saturatedMaskName = pexConfig.Field(
224 doc=
"Name of mask plane to use in saturation detection and interpolation",
227 saturation = pexConfig.Field(
229 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
230 default=float(
"NaN"),
232 growSaturationFootprintSize = pexConfig.Field(
234 doc=
"Number of pixels by which to grow the saturation footprints",
239 doSuspect = pexConfig.Field(
241 doc=
"Mask suspect pixels?",
244 suspectMaskName = pexConfig.Field(
246 doc=
"Name of mask plane to use for suspect pixels",
249 numEdgeSuspect = pexConfig.Field(
251 doc=
"Number of edge pixels to be flagged as untrustworthy.",
256 doSetBadRegions = pexConfig.Field(
258 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
261 badStatistic = pexConfig.ChoiceField(
263 doc=
"How to estimate the average value for BAD regions.",
266 "MEANCLIP":
"Correct using the (clipped) mean of good data",
267 "MEDIAN":
"Correct using the median of the good data",
272 doOverscan = pexConfig.Field(
274 doc=
"Do overscan subtraction?",
277 overscanFitType = pexConfig.ChoiceField(
279 doc=
"The method for fitting the overscan bias level.",
282 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
283 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
284 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
285 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
286 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
287 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
288 "MEAN":
"Correct using the mean of the overscan region",
289 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
290 "MEDIAN":
"Correct using the median of the overscan region",
293 overscanOrder = pexConfig.Field(
295 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
296 "or number of spline knots if overscan fit type is a spline."),
299 overscanNumSigmaClip = pexConfig.Field(
301 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
304 overscanIsInt = pexConfig.Field(
306 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
309 overscanNumLeadingColumnsToSkip = pexConfig.Field(
311 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
314 overscanNumTrailingColumnsToSkip = pexConfig.Field(
316 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
319 overscanMaxDev = pexConfig.Field(
321 doc=
"Maximum deviation from the median for overscan",
322 default=1000.0, check=
lambda x: x > 0
324 overscanBiasJump = pexConfig.Field(
326 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
329 overscanBiasJumpKeyword = pexConfig.Field(
331 doc=
"Header keyword containing information about devices.",
332 default=
"NO_SUCH_KEY",
334 overscanBiasJumpDevices = pexConfig.ListField(
336 doc=
"List of devices that need piecewise overscan correction.",
339 overscanBiasJumpLocation = pexConfig.Field(
341 doc=
"Location of bias jump along y-axis.",
346 doAssembleCcd = pexConfig.Field(
349 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 351 assembleCcd = pexConfig.ConfigurableField(
352 target=AssembleCcdTask,
353 doc=
"CCD assembly task",
357 doAssembleIsrExposures = pexConfig.Field(
360 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 362 doTrimToMatchCalib = pexConfig.Field(
365 doc=
"Trim raw data to match calibration bounding boxes?" 369 doBias = pexConfig.Field(
371 doc=
"Apply bias frame correction?",
374 biasDataProductName = pexConfig.Field(
376 doc=
"Name of the bias data product",
381 doVariance = pexConfig.Field(
383 doc=
"Calculate variance?",
386 gain = pexConfig.Field(
388 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
389 default=float(
"NaN"),
391 readNoise = pexConfig.Field(
393 doc=
"The read noise to use if no Detector is present in the Exposure",
396 doEmpiricalReadNoise = pexConfig.Field(
399 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 403 doLinearize = pexConfig.Field(
405 doc=
"Correct for nonlinearity of the detector's response?",
410 doCrosstalk = pexConfig.Field(
412 doc=
"Apply intra-CCD crosstalk correction?",
415 doCrosstalkBeforeAssemble = pexConfig.Field(
417 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
420 crosstalk = pexConfig.ConfigurableField(
421 target=CrosstalkTask,
422 doc=
"Intra-CCD crosstalk correction",
426 doWidenSaturationTrails = pexConfig.Field(
428 doc=
"Widen bleed trails based on their width?",
433 doBrighterFatter = pexConfig.Field(
436 doc=
"Apply the brighter fatter correction" 438 brighterFatterLevel = pexConfig.ChoiceField(
441 doc=
"The level at which to correct for brighter-fatter.",
443 "AMP":
"Every amplifier treated separately.",
444 "DETECTOR":
"One kernel per detector",
447 brighterFatterKernelFile = pexConfig.Field(
450 doc=
"Kernel file used for the brighter fatter correction" 452 brighterFatterMaxIter = pexConfig.Field(
455 doc=
"Maximum number of iterations for the brighter fatter correction" 457 brighterFatterThreshold = pexConfig.Field(
460 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 461 " absolute value of the difference between the current corrected image and the one" 462 " from the previous iteration summed over all the pixels." 464 brighterFatterApplyGain = pexConfig.Field(
467 doc=
"Should the gain be applied when applying the brighter fatter correction?" 471 doDefect = pexConfig.Field(
473 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
476 doSaturationInterpolation = pexConfig.Field(
478 doc=
"Perform interpolation over pixels masked as saturated?" 479 " NB: This is independent of doSaturation; if that is False this plane" 480 " will likely be blank, resulting in a no-op here.",
483 numEdgeSuspect = pexConfig.Field(
485 doc=
"Number of edge pixels to be flagged as untrustworthy.",
490 doDark = pexConfig.Field(
492 doc=
"Apply dark frame correction?",
495 darkDataProductName = pexConfig.Field(
497 doc=
"Name of the dark data product",
502 doStrayLight = pexConfig.Field(
504 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
507 strayLight = pexConfig.ConfigurableField(
508 target=StrayLightTask,
509 doc=
"y-band stray light correction" 513 doFlat = pexConfig.Field(
515 doc=
"Apply flat field correction?",
518 flatDataProductName = pexConfig.Field(
520 doc=
"Name of the flat data product",
523 flatScalingType = pexConfig.ChoiceField(
525 doc=
"The method for scaling the flat on the fly.",
528 "USER":
"Scale by flatUserScale",
529 "MEAN":
"Scale by the inverse of the mean",
530 "MEDIAN":
"Scale by the inverse of the median",
533 flatUserScale = pexConfig.Field(
535 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
538 doTweakFlat = pexConfig.Field(
540 doc=
"Tweak flats to match observed amplifier ratios?",
545 doApplyGains = pexConfig.Field(
547 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
550 normalizeGains = pexConfig.Field(
552 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
557 doFringe = pexConfig.Field(
559 doc=
"Apply fringe correction?",
562 fringe = pexConfig.ConfigurableField(
564 doc=
"Fringe subtraction task",
566 fringeAfterFlat = pexConfig.Field(
568 doc=
"Do fringe subtraction after flat-fielding?",
573 doNanInterpAfterFlat = pexConfig.Field(
575 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 576 "also have to interpolate them before flat-fielding."),
581 doAddDistortionModel = pexConfig.Field(
583 doc=
"Apply a distortion model based on camera geometry to the WCS?",
588 doMeasureBackground = pexConfig.Field(
590 doc=
"Measure the background level on the reduced image?",
595 doCameraSpecificMasking = pexConfig.Field(
597 doc=
"Mask camera-specific bad regions?",
600 masking = pexConfig.ConfigurableField(
606 fluxMag0T1 = pexConfig.DictField(
609 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
610 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
613 defaultFluxMag0T1 = pexConfig.Field(
615 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
616 default=pow(10.0, 0.4*28.0)
620 doVignette = pexConfig.Field(
622 doc=
"Apply vignetting parameters?",
625 vignette = pexConfig.ConfigurableField(
627 doc=
"Vignetting task.",
631 doAttachTransmissionCurve = pexConfig.Field(
634 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 636 doUseOpticsTransmission = pexConfig.Field(
639 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 641 doUseFilterTransmission = pexConfig.Field(
644 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 646 doUseSensorTransmission = pexConfig.Field(
649 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 651 doUseAtmosphereTransmission = pexConfig.Field(
654 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 658 doWrite = pexConfig.Field(
660 doc=
"Persist postISRCCD?",
667 raise ValueError(
"You may not specify both doFlat and doApplyGains")
670 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
671 r"""Apply common instrument signature correction algorithms to a raw frame. 673 The process for correcting imaging data is very similar from 674 camera to camera. This task provides a vanilla implementation of 675 doing these corrections, including the ability to turn certain 676 corrections off if they are not needed. The inputs to the primary 677 method, `run()`, are a raw exposure to be corrected and the 678 calibration data products. The raw input is a single chip sized 679 mosaic of all amps including overscans and other non-science 680 pixels. The method `runDataRef()` identifies and defines the 681 calibration data products, and is intended for use by a 682 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 683 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 684 subclassed for different camera, although the most camera specific 685 methods have been split into subtasks that can be redirected 688 The __init__ method sets up the subtasks for ISR processing, using 689 the defaults from `lsst.ip.isr`. 694 Positional arguments passed to the Task constructor. None used at this time. 695 kwargs : `dict`, optional 696 Keyword arguments passed on to the Task constructor. None used at this time. 698 ConfigClass = IsrTaskConfig
703 self.makeSubtask(
"assembleCcd")
704 self.makeSubtask(
"crosstalk")
705 self.makeSubtask(
"strayLight")
706 self.makeSubtask(
"fringe")
707 self.makeSubtask(
"masking")
708 self.makeSubtask(
"vignette")
717 if config.doBias
is not True:
718 inputTypeDict.pop(
"bias",
None)
719 if config.doLinearize
is not True:
720 inputTypeDict.pop(
"linearizer",
None)
721 if config.doCrosstalk
is not True:
722 inputTypeDict.pop(
"crosstalkSources",
None)
723 if config.doBrighterFatter
is not True:
724 inputTypeDict.pop(
"bfKernel",
None)
725 if config.doDefect
is not True:
726 inputTypeDict.pop(
"defects",
None)
727 if config.doDark
is not True:
728 inputTypeDict.pop(
"dark",
None)
729 if config.doFlat
is not True:
730 inputTypeDict.pop(
"flat",
None)
731 if config.doAttachTransmissionCurve
is not True:
732 inputTypeDict.pop(
"opticsTransmission",
None)
733 inputTypeDict.pop(
"filterTransmission",
None)
734 inputTypeDict.pop(
"sensorTransmission",
None)
735 inputTypeDict.pop(
"atmosphereTransmission",
None)
736 if config.doUseOpticsTransmission
is not True:
737 inputTypeDict.pop(
"opticsTransmission",
None)
738 if config.doUseFilterTransmission
is not True:
739 inputTypeDict.pop(
"filterTransmission",
None)
740 if config.doUseSensorTransmission
is not True:
741 inputTypeDict.pop(
"sensorTransmission",
None)
742 if config.doUseAtmosphereTransmission
is not True:
743 inputTypeDict.pop(
"atmosphereTransmission",
None)
751 if config.qa.doThumbnailOss
is not True:
752 outputTypeDict.pop(
"outputOssThumbnail",
None)
753 if config.qa.doThumbnailFlattened
is not True:
754 outputTypeDict.pop(
"outputFlattenedThumbnail",
None)
755 if config.doWrite
is not True:
756 outputTypeDict.pop(
"outputExposure",
None)
758 return outputTypeDict
767 names.remove(
"ccdExposure")
776 return frozenset([
"CalibrationLabel"])
780 inputData[
'detectorNum'] = int(inputDataIds[
'ccdExposure'][
'detector'])
781 except Exception
as e:
782 raise ValueError(f
"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
784 inputData[
'isGen3'] =
True 786 if self.config.doLinearize
is True:
787 if 'linearizer' not in inputData.keys():
788 detector = inputData[
'camera'][inputData[
'detectorNum']]
789 linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
790 inputData[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
792 if inputData[
'defects']
is not None:
797 for r
in inputData[
'defects']:
798 bbox = afwGeom.BoxI(afwGeom.PointI(r.get(
"x0"), r.get(
"y0")),
799 afwGeom.ExtentI(r.get(
"width"), r.get(
"height")))
800 defectList.append(
Defect(bbox))
802 inputData[
'defects'] = defectList
819 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
825 """!Retrieve necessary frames for instrument signature removal. 827 Pre-fetching all required ISR data products limits the IO 828 required by the ISR. Any conflict between the calibration data 829 available and that needed for ISR is also detected prior to 830 doing processing, allowing it to fail quickly. 834 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 835 Butler reference of the detector data to be processed 836 rawExposure : `afw.image.Exposure` 837 The raw exposure that will later be corrected with the 838 retrieved calibration data; should not be modified in this 843 result : `lsst.pipe.base.Struct` 844 Result struct with components (which may be `None`): 845 - ``bias``: bias calibration frame (`afw.image.Exposure`) 846 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 847 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 848 - ``dark``: dark calibration frame (`afw.image.Exposure`) 849 - ``flat``: flat calibration frame (`afw.image.Exposure`) 850 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 851 - ``defects``: list of defects (`list`) 852 - ``fringes``: `lsst.pipe.base.Struct` with components: 853 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 854 - ``seed``: random seed derived from the ccdExposureId for random 855 number generator (`uint32`) 856 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 857 A ``TransmissionCurve`` that represents the throughput of the optics, 858 to be evaluated in focal-plane coordinates. 859 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 860 A ``TransmissionCurve`` that represents the throughput of the filter 861 itself, to be evaluated in focal-plane coordinates. 862 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 863 A ``TransmissionCurve`` that represents the throughput of the sensor 864 itself, to be evaluated in post-assembly trimmed detector coordinates. 865 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 866 A ``TransmissionCurve`` that represents the throughput of the 867 atmosphere, assumed to be spatially constant. 871 NotImplementedError : 872 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 874 ccd = rawExposure.getDetector()
875 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
876 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
877 if self.config.doBias
else None)
879 linearizer = (dataRef.get(
"linearizer", immediate=
True)
881 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
882 if self.config.doCrosstalk
else None)
883 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
884 if self.config.doDark
else None)
885 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
886 if self.config.doFlat
else None)
888 brighterFatterKernel =
None 889 if self.config.doBrighterFatter
is True:
893 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
897 brighterFatterKernel = dataRef.get(
"bfKernel")
899 brighterFatterKernel =
None 900 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
903 if self.config.brighterFatterLevel ==
'DETECTOR':
904 brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
907 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
909 defectList = (dataRef.get(
"defects")
910 if self.config.doDefect
else None)
911 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
912 if self.config.doAssembleIsrExposures
else None)
913 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
914 else pipeBase.Struct(fringes=
None))
916 if self.config.doAttachTransmissionCurve:
917 opticsTransmission = (dataRef.get(
"transmission_optics")
918 if self.config.doUseOpticsTransmission
else None)
919 filterTransmission = (dataRef.get(
"transmission_filter")
920 if self.config.doUseFilterTransmission
else None)
921 sensorTransmission = (dataRef.get(
"transmission_sensor")
922 if self.config.doUseSensorTransmission
else None)
923 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
924 if self.config.doUseAtmosphereTransmission
else None)
926 opticsTransmission =
None 927 filterTransmission =
None 928 sensorTransmission =
None 929 atmosphereTransmission =
None 932 return pipeBase.Struct(bias=biasExposure,
933 linearizer=linearizer,
934 crosstalkSources=crosstalkSources,
937 bfKernel=brighterFatterKernel,
939 fringes=fringeStruct,
940 opticsTransmission=opticsTransmission,
941 filterTransmission=filterTransmission,
942 sensorTransmission=sensorTransmission,
943 atmosphereTransmission=atmosphereTransmission,
947 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
948 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
949 opticsTransmission=None, filterTransmission=None,
950 sensorTransmission=None, atmosphereTransmission=None,
951 detectorNum=None, isGen3=False
953 """!Perform instrument signature removal on an exposure. 955 Steps included in the ISR processing, in order performed, are: 956 - saturation and suspect pixel masking 957 - overscan subtraction 958 - CCD assembly of individual amplifiers 960 - variance image construction 961 - linearization of non-linear response 963 - brighter-fatter correction 966 - stray light subtraction 968 - masking of known defects and camera specific features 969 - vignette calculation 970 - appending transmission curve and distortion model 974 ccdExposure : `lsst.afw.image.Exposure` 975 The raw exposure that is to be run through ISR. The 976 exposure is modified by this method. 977 camera : `lsst.afw.cameraGeom.Camera`, optional 978 The camera geometry for this exposure. Used to select the 979 distortion model appropriate for this data. 980 bias : `lsst.afw.image.Exposure`, optional 981 Bias calibration frame. 982 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 983 Functor for linearization. 984 crosstalkSources : `list`, optional 985 List of possible crosstalk sources. 986 dark : `lsst.afw.image.Exposure`, optional 987 Dark calibration frame. 988 flat : `lsst.afw.image.Exposure`, optional 989 Flat calibration frame. 990 bfKernel : `numpy.ndarray`, optional 991 Brighter-fatter kernel. 992 defects : `list`, optional 994 fringes : `lsst.pipe.base.Struct`, optional 995 Struct containing the fringe correction data, with 997 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 998 - ``seed``: random seed derived from the ccdExposureId for random 999 number generator (`uint32`) 1000 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 1001 A ``TransmissionCurve`` that represents the throughput of the optics, 1002 to be evaluated in focal-plane coordinates. 1003 filterTransmission : `lsst.afw.image.TransmissionCurve` 1004 A ``TransmissionCurve`` that represents the throughput of the filter 1005 itself, to be evaluated in focal-plane coordinates. 1006 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1007 A ``TransmissionCurve`` that represents the throughput of the sensor 1008 itself, to be evaluated in post-assembly trimmed detector coordinates. 1009 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1010 A ``TransmissionCurve`` that represents the throughput of the 1011 atmosphere, assumed to be spatially constant. 1012 detectorNum : `int`, optional 1013 The integer number for the detector to process. 1014 isGen3 : bool, optional 1015 Flag this call to run() as using the Gen3 butler environment. 1019 result : `lsst.pipe.base.Struct` 1020 Result struct with component: 1021 - ``exposure`` : `afw.image.Exposure` 1022 The fully ISR corrected exposure. 1023 - ``outputExposure`` : `afw.image.Exposure` 1024 An alias for `exposure` 1025 - ``ossThumb`` : `numpy.ndarray` 1026 Thumbnail image of the exposure after overscan subtraction. 1027 - ``flattenedThumb`` : `numpy.ndarray` 1028 Thumbnail image of the exposure after flat-field correction. 1033 Raised if a configuration option is set to True, but the 1034 required calibration data has not been specified. 1038 The current processed exposure can be viewed by setting the 1039 appropriate lsstDebug entries in the `debug.display` 1040 dictionary. The names of these entries correspond to some of 1041 the IsrTaskConfig Boolean options, with the value denoting the 1042 frame to use. The exposure is shown inside the matching 1043 option check and after the processing of that step has 1044 finished. The steps with debug points are: 1055 In addition, setting the "postISRCCD" entry displays the 1056 exposure after all ISR processing has finished. 1064 self.config.doFringe =
False 1067 if detectorNum
is None:
1068 raise RuntimeError(
"Must supply the detectorNum if running as Gen3")
1070 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1075 if isinstance(ccdExposure, ButlerDataRef):
1078 ccd = ccdExposure.getDetector()
1081 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 1082 ccd = [
FakeAmp(ccdExposure, self.config)]
1085 if self.config.doBias
and bias
is None:
1086 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1088 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1089 if self.config.doBrighterFatter
and bfKernel
is None:
1090 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1091 if self.config.doDark
and dark
is None:
1092 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1094 fringes = pipeBase.Struct(fringes=
None)
1095 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
1096 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1097 if self.config.doFlat
and flat
is None:
1098 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1099 if self.config.doDefect
and defects
is None:
1100 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1101 if self.config.doAddDistortionModel
and camera
is None:
1102 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
1105 if self.config.doConvertIntToFloat:
1106 self.log.info(
"Converting exposure to floating point values")
1113 if ccdExposure.getBBox().contains(amp.getBBox()):
1117 if self.config.doOverscan
and not badAmp:
1120 self.log.debug(
"Corrected overscan for amplifier %s" % (amp.getName()))
1121 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1122 if isinstance(overscanResults.overscanFit, float):
1123 qaMedian = overscanResults.overscanFit
1124 qaStdev = float(
"NaN")
1126 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1127 afwMath.MEDIAN | afwMath.STDEVCLIP)
1128 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1129 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1131 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1132 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1133 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f" %
1134 (amp.getName(), qaMedian, qaStdev))
1135 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1137 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
1138 overscanResults =
None 1140 overscans.append(overscanResults
if overscanResults
is not None else None)
1142 self.log.info(
"Skipped OSCAN")
1144 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1145 self.log.info(
"Applying crosstalk correction.")
1146 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1147 self.
debugView(ccdExposure,
"doCrosstalk")
1149 if self.config.doAssembleCcd:
1150 self.log.info(
"Assembling CCD from amplifiers")
1151 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1153 if self.config.expectWcs
and not ccdExposure.getWcs():
1154 self.log.warn(
"No WCS found in input exposure")
1155 self.
debugView(ccdExposure,
"doAssembleCcd")
1158 if self.config.qa.doThumbnailOss:
1159 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1161 if self.config.doBias:
1162 self.log.info(
"Applying bias correction.")
1163 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1164 trimToFit=self.config.doTrimToMatchCalib)
1167 if self.config.doVariance:
1168 for amp, overscanResults
in zip(ccd, overscans):
1169 if ccdExposure.getBBox().contains(amp.getBBox()):
1170 self.log.debug(
"Constructing variance map for amplifer %s" % (amp.getName()))
1171 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1172 if overscanResults
is not None:
1174 overscanImage=overscanResults.overscanImage)
1178 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1179 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1180 afwMath.MEDIAN | afwMath.STDEVCLIP)
1181 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
1182 qaStats.getValue(afwMath.MEDIAN))
1183 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
1184 qaStats.getValue(afwMath.STDEVCLIP))
1185 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f" %
1186 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1187 qaStats.getValue(afwMath.STDEVCLIP)))
1190 self.log.info(
"Applying linearizer.")
1191 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1193 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1194 self.log.info(
"Applying crosstalk correction.")
1195 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1196 self.
debugView(ccdExposure,
"doCrosstalk")
1198 if self.config.doWidenSaturationTrails:
1199 self.log.info(
"Widening saturation trails.")
1200 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1202 interpolationDone =
False 1203 if self.config.doBrighterFatter:
1209 if self.config.doDefect:
1212 if self.config.doSaturationInterpolation:
1216 interpolationDone =
True 1218 self.log.info(
"Applying brighter fatter correction.")
1219 isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1220 self.config.brighterFatterMaxIter,
1221 self.config.brighterFatterThreshold,
1222 self.config.brighterFatterApplyGain,
1224 self.
debugView(ccdExposure,
"doBrighterFatter")
1226 if self.config.doDark:
1227 self.log.info(
"Applying dark correction.")
1231 if self.config.doFringe
and not self.config.fringeAfterFlat:
1232 self.log.info(
"Applying fringe correction before flat.")
1233 self.fringe.
run(ccdExposure, **fringes.getDict())
1236 if self.config.doStrayLight:
1237 self.log.info(
"Applying stray light correction.")
1238 self.strayLight.
run(ccdExposure)
1239 self.
debugView(ccdExposure,
"doStrayLight")
1241 if self.config.doFlat:
1242 self.log.info(
"Applying flat correction.")
1246 if self.config.doApplyGains:
1247 self.log.info(
"Applying gain correction instead of flat.")
1248 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1250 if self.config.doDefect
and not interpolationDone:
1251 self.log.info(
"Masking and interpolating defects.")
1254 if self.config.doSaturationInterpolation
and not interpolationDone:
1255 self.log.info(
"Interpolating saturated pixels.")
1258 if self.config.doNanInterpAfterFlat
or not interpolationDone:
1259 self.log.info(
"Masking and interpolating NAN value pixels.")
1262 if self.config.doFringe
and self.config.fringeAfterFlat:
1263 self.log.info(
"Applying fringe correction after flat.")
1264 self.fringe.
run(ccdExposure, **fringes.getDict())
1266 if self.config.doSetBadRegions:
1267 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1268 if badPixelCount > 0:
1269 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1271 flattenedThumb =
None 1272 if self.config.qa.doThumbnailFlattened:
1273 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1275 if self.config.doCameraSpecificMasking:
1276 self.log.info(
"Masking regions for camera specific reasons.")
1277 self.masking.
run(ccdExposure)
1281 if self.config.doVignette:
1282 self.log.info(
"Constructing Vignette polygon.")
1285 if self.config.vignette.doWriteVignettePolygon:
1288 if self.config.doAttachTransmissionCurve:
1289 self.log.info(
"Adding transmission curves.")
1290 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1291 filterTransmission=filterTransmission,
1292 sensorTransmission=sensorTransmission,
1293 atmosphereTransmission=atmosphereTransmission)
1295 if self.config.doAddDistortionModel:
1296 self.log.info(
"Adding a distortion model to the WCS.")
1297 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1299 if self.config.doMeasureBackground:
1300 self.log.info(
"Measuring background level:")
1303 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1305 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1306 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1307 afwMath.MEDIAN | afwMath.STDEVCLIP)
1308 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1309 qaStats.getValue(afwMath.MEDIAN))
1310 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1311 qaStats.getValue(afwMath.STDEVCLIP))
1312 self.log.debug(
" Background stats for amplifer %s: %f +/- %f" %
1313 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1314 qaStats.getValue(afwMath.STDEVCLIP)))
1316 self.
debugView(ccdExposure,
"postISRCCD")
1318 return pipeBase.Struct(
1319 exposure=ccdExposure,
1321 flattenedThumb=flattenedThumb,
1323 outputExposure=ccdExposure,
1324 outputOssThumbnail=ossThumb,
1325 outputFlattenedThumbnail=flattenedThumb,
1328 @pipeBase.timeMethod
1330 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1332 This method contains the `CmdLineTask` interface to the ISR 1333 processing. All IO is handled here, freeing the `run()` method 1334 to manage only pixel-level calculations. The steps performed 1336 - Read in necessary detrending/isr/calibration data. 1337 - Process raw exposure in `run()`. 1338 - Persist the ISR-corrected exposure as "postISRCCD" if 1339 config.doWrite=True. 1343 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1344 DataRef of the detector data to be processed 1348 result : `lsst.pipe.base.Struct` 1349 Result struct with component: 1350 - ``exposure`` : `afw.image.Exposure` 1351 The fully ISR corrected exposure. 1356 Raised if a configuration option is set to True, but the 1357 required calibration data does not exist. 1360 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1362 ccdExposure = sensorRef.get(self.config.datasetType)
1364 camera = sensorRef.get(
"camera")
1365 if camera
is None and self.config.doAddDistortionModel:
1366 raise RuntimeError(
"config.doAddDistortionModel is True " 1367 "but could not get a camera from the butler")
1368 isrData = self.
readIsrData(sensorRef, ccdExposure)
1370 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1372 if self.config.doWrite:
1373 sensorRef.put(result.exposure,
"postISRCCD")
1374 if result.ossThumb
is not None:
1375 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1376 if result.flattenedThumb
is not None:
1377 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1382 """!Retrieve a calibration dataset for removing instrument signature. 1387 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1388 DataRef of the detector data to find calibration datasets 1391 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1393 If True, disable butler proxies to enable error handling 1394 within this routine. 1398 exposure : `lsst.afw.image.Exposure` 1399 Requested calibration frame. 1404 Raised if no matching calibration frame can be found. 1407 exp = dataRef.get(datasetType, immediate=immediate)
1408 except Exception
as exc1:
1409 if not self.config.fallbackFilterName:
1410 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1412 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1413 except Exception
as exc2:
1414 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1415 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1416 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1418 if self.config.doAssembleIsrExposures:
1419 exp = self.assembleCcd.assembleCcd(exp)
1423 """Ensure that the data returned by Butler is a fully constructed exposure. 1425 ISR requires exposure-level image data for historical reasons, so if we did 1426 not recieve that from Butler, construct it from what we have, modifying the 1431 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1432 `lsst.afw.image.ImageF` 1433 The input data structure obtained from Butler. 1434 camera : `lsst.afw.cameraGeom.camera` 1435 The camera associated with the image. Used to find the appropriate 1438 The detector this exposure should match. 1442 inputExp : `lsst.afw.image.Exposure` 1443 The re-constructed exposure, with appropriate detector parameters. 1448 Raised if the input data cannot be used to construct an exposure. 1450 if isinstance(inputExp, afwImage.DecoratedImageU):
1451 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1452 elif isinstance(inputExp, afwImage.ImageF):
1453 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1454 elif isinstance(inputExp, afwImage.MaskedImageF):
1455 inputExp = afwImage.makeExposure(inputExp)
1456 elif isinstance(inputExp, afwImage.Exposure):
1459 raise TypeError(f
"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1461 if inputExp.getDetector()
is None:
1462 inputExp.setDetector(camera[detectorNum])
1467 """Convert exposure image from uint16 to float. 1469 If the exposure does not need to be converted, the input is 1470 immediately returned. For exposures that are converted to use 1471 floating point pixels, the variance is set to unity and the 1476 exposure : `lsst.afw.image.Exposure` 1477 The raw exposure to be converted. 1481 newexposure : `lsst.afw.image.Exposure` 1482 The input ``exposure``, converted to floating point pixels. 1487 Raised if the exposure type cannot be converted to float. 1490 if isinstance(exposure, afwImage.ExposureF):
1493 if not hasattr(exposure,
"convertF"):
1494 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1496 newexposure = exposure.convertF()
1497 newexposure.variance[:] = 1
1498 newexposure.mask[:] = 0x0
1503 """Identify bad amplifiers, saturated and suspect pixels. 1507 ccdExposure : `lsst.afw.image.Exposure` 1508 Input exposure to be masked. 1509 amp : `lsst.afw.table.AmpInfoCatalog` 1510 Catalog of parameters defining the amplifier on this 1513 List of defects. Used to determine if the entire 1519 If this is true, the entire amplifier area is covered by 1520 defects and unusable. 1523 maskedImage = ccdExposure.getMaskedImage()
1529 if defects
is not None:
1530 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1535 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1537 maskView = dataView.getMask()
1538 maskView |= maskView.getPlaneBitMask(
"BAD")
1545 if self.config.doSaturation
and not badAmp:
1546 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1547 if self.config.doSuspect
and not badAmp:
1548 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1550 for maskName, maskThreshold
in limits.items():
1551 if not math.isnan(maskThreshold):
1552 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1553 isrFunctions.makeThresholdMask(
1554 maskedImage=dataView,
1555 threshold=maskThreshold,
1561 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1563 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1564 self.config.suspectMaskName])
1565 if numpy.all(maskView.getArray() & maskVal > 0):
1571 """Apply overscan correction in place. 1573 This method does initial pixel rejection of the overscan 1574 region. The overscan can also be optionally segmented to 1575 allow for discontinuous overscan responses to be fit 1576 separately. The actual overscan subtraction is performed by 1577 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1578 which is called here after the amplifier is preprocessed. 1582 ccdExposure : `lsst.afw.image.Exposure` 1583 Exposure to have overscan correction performed. 1584 amp : `lsst.afw.table.AmpInfoCatalog` 1585 The amplifier to consider while correcting the overscan. 1589 overscanResults : `lsst.pipe.base.Struct` 1590 Result struct with components: 1591 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1592 Value or fit subtracted from the amplifier image data. 1593 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1594 Value or fit subtracted from the overscan image data. 1595 - ``overscanImage`` : `lsst.afw.image.Image` 1596 Image of the overscan region with the overscan 1597 correction applied. This quantity is used to estimate 1598 the amplifier read noise empirically. 1603 Raised if the ``amp`` does not contain raw pixel information. 1607 lsst.ip.isr.isrFunctions.overscanCorrection 1609 if not amp.getHasRawInfo():
1610 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1612 if amp.getRawHorizontalOverscanBBox().isEmpty():
1613 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1616 statControl = afwMath.StatisticsControl()
1617 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1620 dataBBox = amp.getRawDataBBox()
1621 oscanBBox = amp.getRawHorizontalOverscanBBox()
1625 prescanBBox = amp.getRawPrescanBBox()
1626 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1627 dx0 += self.config.overscanNumLeadingColumnsToSkip
1628 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1630 dx0 += self.config.overscanNumTrailingColumnsToSkip
1631 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1637 if ((self.config.overscanBiasJump
and 1638 self.config.overscanBiasJumpLocation)
and 1639 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1640 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1641 self.config.overscanBiasJumpDevices)):
1642 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1643 yLower = self.config.overscanBiasJumpLocation
1644 yUpper = dataBBox.getHeight() - yLower
1646 yUpper = self.config.overscanBiasJumpLocation
1647 yLower = dataBBox.getHeight() - yUpper
1649 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1650 afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1651 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1652 afwGeom.Extent2I(dx0, 0),
1653 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1656 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1657 afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1658 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1659 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1662 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1663 afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1664 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1665 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1666 oscanBBox.getHeight())))
1669 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1670 ampImage = ccdExposure.maskedImage[imageBBox]
1671 overscanImage = ccdExposure.maskedImage[overscanBBox]
1673 overscanArray = overscanImage.image.array
1674 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1675 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1676 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1678 statControl = afwMath.StatisticsControl()
1679 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1681 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1682 overscanImage=overscanImage,
1683 fitType=self.config.overscanFitType,
1684 order=self.config.overscanOrder,
1685 collapseRej=self.config.overscanNumSigmaClip,
1686 statControl=statControl,
1687 overscanIsInt=self.config.overscanIsInt
1691 levelStat = afwMath.MEDIAN
1692 sigmaStat = afwMath.STDEVCLIP
1694 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1695 self.config.qa.flatness.nIter)
1696 metadata = ccdExposure.getMetadata()
1697 ampNum = amp.getName()
1698 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1699 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1700 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1702 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1703 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1704 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1706 return overscanResults
1709 """Set the variance plane using the amplifier gain and read noise 1711 The read noise is calculated from the ``overscanImage`` if the 1712 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1713 the value from the amplifier data is used. 1717 ampExposure : `lsst.afw.image.Exposure` 1718 Exposure to process. 1719 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1720 Amplifier detector data. 1721 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1722 Image of overscan, required only for empirical read noise. 1726 lsst.ip.isr.isrFunctions.updateVariance 1728 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1729 gain = amp.getGain()
1731 if math.isnan(gain):
1733 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1736 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1737 (amp.getName(), gain, patchedGain))
1740 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1741 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1743 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1744 stats = afwMath.StatisticsControl()
1745 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1746 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1747 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1749 readNoise = amp.getReadNoise()
1751 isrFunctions.updateVariance(
1752 maskedImage=ampExposure.getMaskedImage(),
1754 readNoise=readNoise,
1758 """!Apply dark correction in place. 1762 exposure : `lsst.afw.image.Exposure` 1763 Exposure to process. 1764 darkExposure : `lsst.afw.image.Exposure` 1765 Dark exposure of the same size as ``exposure``. 1766 invert : `Bool`, optional 1767 If True, re-add the dark to an already corrected image. 1772 Raised if either ``exposure`` or ``darkExposure`` do not 1773 have their dark time defined. 1777 lsst.ip.isr.isrFunctions.darkCorrection 1779 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1780 if math.isnan(expScale):
1781 raise RuntimeError(
"Exposure darktime is NAN")
1782 if darkExposure.getInfo().getVisitInfo()
is not None:
1783 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1789 if math.isnan(darkScale):
1790 raise RuntimeError(
"Dark calib darktime is NAN")
1791 isrFunctions.darkCorrection(
1792 maskedImage=exposure.getMaskedImage(),
1793 darkMaskedImage=darkExposure.getMaskedImage(),
1795 darkScale=darkScale,
1797 trimToFit=self.config.doTrimToMatchCalib
1801 """!Check if linearization is needed for the detector cameraGeom. 1803 Checks config.doLinearize and the linearity type of the first 1808 detector : `lsst.afw.cameraGeom.Detector` 1809 Detector to get linearity type from. 1813 doLinearize : `Bool` 1814 If True, linearization should be performed. 1816 return self.config.doLinearize
and \
1817 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1820 """!Apply flat correction in place. 1824 exposure : `lsst.afw.image.Exposure` 1825 Exposure to process. 1826 flatExposure : `lsst.afw.image.Exposure` 1827 Flat exposure of the same size as ``exposure``. 1828 invert : `Bool`, optional 1829 If True, unflatten an already flattened image. 1833 lsst.ip.isr.isrFunctions.flatCorrection 1835 isrFunctions.flatCorrection(
1836 maskedImage=exposure.getMaskedImage(),
1837 flatMaskedImage=flatExposure.getMaskedImage(),
1838 scalingType=self.config.flatScalingType,
1839 userScale=self.config.flatUserScale,
1841 trimToFit=self.config.doTrimToMatchCalib
1845 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1849 exposure : `lsst.afw.image.Exposure` 1850 Exposure to process. Only the amplifier DataSec is processed. 1851 amp : `lsst.afw.table.AmpInfoCatalog` 1852 Amplifier detector data. 1856 lsst.ip.isr.isrFunctions.makeThresholdMask 1858 if not math.isnan(amp.getSaturation()):
1859 maskedImage = exposure.getMaskedImage()
1860 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1861 isrFunctions.makeThresholdMask(
1862 maskedImage=dataView,
1863 threshold=amp.getSaturation(),
1865 maskName=self.config.saturatedMaskName,
1869 """!Interpolate over saturated pixels, in place. 1871 This method should be called after `saturationDetection`, to 1872 ensure that the saturated pixels have been identified in the 1873 SAT mask. It should also be called after `assembleCcd`, since 1874 saturated regions may cross amplifier boundaries. 1878 exposure : `lsst.afw.image.Exposure` 1879 Exposure to process. 1883 lsst.ip.isr.isrTask.saturationDetection 1884 lsst.ip.isr.isrFunctions.interpolateFromMask 1886 isrFunctions.interpolateFromMask(
1887 maskedImage=ccdExposure.getMaskedImage(),
1888 fwhm=self.config.fwhm,
1889 growFootprints=self.config.growSaturationFootprintSize,
1890 maskName=self.config.saturatedMaskName,
1894 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1898 exposure : `lsst.afw.image.Exposure` 1899 Exposure to process. Only the amplifier DataSec is processed. 1900 amp : `lsst.afw.table.AmpInfoCatalog` 1901 Amplifier detector data. 1905 lsst.ip.isr.isrFunctions.makeThresholdMask 1909 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1910 This is intended to indicate pixels that may be affected by unknown systematics; 1911 for example if non-linearity corrections above a certain level are unstable 1912 then that would be a useful value for suspectLevel. A value of `nan` indicates 1913 that no such level exists and no pixels are to be masked as suspicious. 1915 suspectLevel = amp.getSuspectLevel()
1916 if math.isnan(suspectLevel):
1919 maskedImage = exposure.getMaskedImage()
1920 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1921 isrFunctions.makeThresholdMask(
1922 maskedImage=dataView,
1923 threshold=suspectLevel,
1925 maskName=self.config.suspectMaskName,
1929 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1933 ccdExposure : `lsst.afw.image.Exposure` 1934 Exposure to process. 1935 defectBaseList : `List` 1936 List of defects to mask and interpolate. 1940 Call this after CCD assembly, since defects may cross amplifier boundaries. 1942 maskedImage = ccdExposure.getMaskedImage()
1944 for d
in defectBaseList:
1946 nd = measAlg.Defect(bbox)
1947 defectList.append(nd)
1948 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
1949 isrFunctions.interpolateDefectList(
1950 maskedImage=maskedImage,
1951 defectList=defectList,
1952 fwhm=self.config.fwhm,
1955 if self.config.numEdgeSuspect > 0:
1956 goodBBox = maskedImage.getBBox()
1958 goodBBox.grow(-self.config.numEdgeSuspect)
1960 SourceDetectionTask.setEdgeBits(
1963 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1967 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1971 exposure : `lsst.afw.image.Exposure` 1972 Exposure to process. 1976 We mask and interpolate over all NaNs, including those 1977 that are masked with other bits (because those may or may 1978 not be interpolated over later, and we want to remove all 1979 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1980 is used to preserve the historical name. 1982 maskedImage = exposure.getMaskedImage()
1985 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
1986 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
1987 numNans =
maskNans(maskedImage, maskVal)
1988 self.metadata.set(
"NUMNANS", numNans)
1992 self.log.warn(
"There were %i unmasked NaNs", numNans)
1993 nanDefectList = isrFunctions.getDefectListFromMask(
1994 maskedImage=maskedImage,
1995 maskName=
'UNMASKEDNAN',
1997 isrFunctions.interpolateDefectList(
1998 maskedImage=exposure.getMaskedImage(),
1999 defectList=nanDefectList,
2000 fwhm=self.config.fwhm,
2004 """Measure the image background in subgrids, for quality control purposes. 2008 exposure : `lsst.afw.image.Exposure` 2009 Exposure to process. 2010 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 2011 Configuration object containing parameters on which background 2012 statistics and subgrids to use. 2014 if IsrQaConfig
is not None:
2015 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2016 IsrQaConfig.flatness.nIter)
2017 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2018 statsControl.setAndMask(maskVal)
2019 maskedImage = exposure.getMaskedImage()
2020 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2021 skyLevel = stats.getValue(afwMath.MEDIAN)
2022 skySigma = stats.getValue(afwMath.STDEVCLIP)
2023 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2024 metadata = exposure.getMetadata()
2025 metadata.set(
'SKYLEVEL', skyLevel)
2026 metadata.set(
'SKYSIGMA', skySigma)
2029 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2030 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2031 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2032 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2033 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2034 skyLevels = numpy.zeros((nX, nY))
2037 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2039 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2041 xLLC = xc - meshXHalf
2042 yLLC = yc - meshYHalf
2043 xURC = xc + meshXHalf - 1
2044 yURC = yc + meshYHalf - 1
2046 bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2047 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2049 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2051 good = numpy.where(numpy.isfinite(skyLevels))
2052 skyMedian = numpy.median(skyLevels[good])
2053 flatness = (skyLevels[good] - skyMedian) / skyMedian
2054 flatness_rms = numpy.std(flatness)
2055 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2057 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2058 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
2059 (nX, nY, flatness_pp, flatness_rms))
2061 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2062 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2063 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2064 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2065 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2068 """Set an approximate magnitude zero point for the exposure. 2072 exposure : `lsst.afw.image.Exposure` 2073 Exposure to process. 2075 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2076 if filterName
in self.config.fluxMag0T1:
2077 fluxMag0 = self.config.fluxMag0T1[filterName]
2079 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
2080 fluxMag0 = self.config.defaultFluxMag0T1
2082 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2084 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
2087 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2088 exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2091 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2095 ccdExposure : `lsst.afw.image.Exposure` 2096 Exposure to process. 2097 fpPolygon : `lsst.afw.geom.Polygon` 2098 Polygon in focal plane coordinates. 2101 ccd = ccdExposure.getDetector()
2102 fpCorners = ccd.getCorners(FOCAL_PLANE)
2103 ccdPolygon = Polygon(fpCorners)
2106 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2109 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2110 validPolygon = Polygon(ccdPoints)
2111 ccdExposure.getInfo().setValidPolygon(validPolygon)
2115 """Context manager that applies and removes flats and darks, 2116 if the task is configured to apply them. 2120 exp : `lsst.afw.image.Exposure` 2121 Exposure to process. 2122 flat : `lsst.afw.image.Exposure` 2123 Flat exposure the same size as ``exp``. 2124 dark : `lsst.afw.image.Exposure`, optional 2125 Dark exposure the same size as ``exp``. 2129 exp : `lsst.afw.image.Exposure` 2130 The flat and dark corrected exposure. 2132 if self.config.doDark
and dark
is not None:
2134 if self.config.doFlat:
2139 if self.config.doFlat:
2141 if self.config.doDark
and dark
is not None:
2145 """Utility function to examine ISR exposure at different stages. 2149 exposure : `lsst.afw.image.Exposure` 2152 State of processing to view. 2154 frame = getDebugFrame(self._display, stepname)
2156 display = getDisplay(frame)
2157 display.scale(
'asinh',
'zscale')
2158 display.mtv(exposure)
2162 """A Detector-like object that supports returning gain and saturation level 2164 This is used when the input exposure does not have a detector. 2168 exposure : `lsst.afw.image.Exposure` 2169 Exposure to generate a fake amplifier for. 2170 config : `lsst.ip.isr.isrTaskConfig` 2171 Configuration to apply to the fake amplifier. 2175 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2177 self.
_gain = config.gain
2207 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2211 """Task to wrap the default IsrTask to allow it to be retargeted. 2213 The standard IsrTask can be called directly from a command line 2214 program, but doing so removes the ability of the task to be 2215 retargeted. As most cameras override some set of the IsrTask 2216 methods, this would remove those data-specific methods in the 2217 output post-ISR images. This wrapping class fixes the issue, 2218 allowing identical post-ISR images to be generated by both the 2219 processCcd and isrTask code. 2221 ConfigClass = RunIsrConfig
2222 _DefaultName =
"runIsr" 2226 self.makeSubtask(
"isr")
2232 dataRef : `lsst.daf.persistence.ButlerDataRef` 2233 data reference of the detector data to be processed 2237 result : `pipeBase.Struct` 2238 Result struct with component: 2240 - exposure : `lsst.afw.image.Exposure` 2241 Post-ISR processed exposure. def getInputDatasetTypes(cls, config)
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def getPrerequisiteDatasetTypes(cls, config)
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def getOutputDatasetTypes(cls, config)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def makeDatasetType(self, dsConfig)
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def getPerDatasetTypeDimensions(cls, config)
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)