33 from contextlib
import contextmanager
34 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .straylight
import StrayLightTask
54 from .vignette
import VignetteTask
56 __all__ = [
"IsrTask",
"RunIsrTask"]
60 """Configuration parameters for IsrTask. 62 Items are grouped in the order in which they are executed by the task. 67 isrName = pexConfig.Field(
74 ccdExposure = pipeBase.InputDatasetField(
75 doc=
"Input exposure to process",
78 storageClass=
"ExposureU",
79 dimensions=[
"Instrument",
"Exposure",
"Detector"],
81 camera = pipeBase.InputDatasetField(
82 doc=
"Input camera to construct complete exposures.",
85 storageClass=
"TablePersistableCamera",
86 dimensions=[
"Instrument",
"CalibrationLabel"],
88 bias = pipeBase.InputDatasetField(
89 doc=
"Input bias calibration.",
92 storageClass=
"ImageF",
93 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
95 dark = pipeBase.InputDatasetField(
96 doc=
"Input dark calibration.",
99 storageClass=
"ImageF",
100 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
102 flat = pipeBase.InputDatasetField(
103 doc=
"Input flat calibration.",
106 storageClass=
"MaskedImageF",
107 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel",
"Detector"],
109 bfKernel = pipeBase.InputDatasetField(
110 doc=
"Input brighter-fatter kernel.",
113 storageClass=
"NumpyArray",
114 dimensions=[
"Instrument",
"CalibrationLabel"],
116 defects = pipeBase.InputDatasetField(
117 doc=
"Input defect tables.",
120 storageClass=
"Catalog",
121 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
123 opticsTransmission = pipeBase.InputDatasetField(
124 doc=
"Transmission curve due to the optics.",
125 name=
"transmission_optics",
127 storageClass=
"TablePersistableTransmissionCurve",
128 dimensions=[
"Instrument",
"CalibrationLabel"],
130 filterTransmission = pipeBase.InputDatasetField(
131 doc=
"Transmission curve due to the filter.",
132 name=
"transmission_filter",
134 storageClass=
"TablePersistableTransmissionCurve",
135 dimensions=[
"Instrument",
"PhysicalFilter",
"CalibrationLabel"],
137 sensorTransmission = pipeBase.InputDatasetField(
138 doc=
"Transmission curve due to the sensor.",
139 name=
"transmission_sensor",
141 storageClass=
"TablePersistableTransmissionCurve",
142 dimensions=[
"Instrument",
"CalibrationLabel",
"Detector"],
144 atmosphereTransmission = pipeBase.InputDatasetField(
145 doc=
"Transmission curve due to the atmosphere.",
146 name=
"transmission_atmosphere",
148 storageClass=
"TablePersistableTransmissionCurve",
149 dimensions=[
"Instrument"],
153 outputExposure = pipeBase.OutputDatasetField(
154 doc=
"Output ISR processed exposure.",
157 storageClass=
"ExposureF",
158 dimensions=[
"Instrument",
"Visit",
"Detector"],
160 outputOssThumbnail = pipeBase.OutputDatasetField(
161 doc=
"Output Overscan-subtracted thumbnail image.",
164 storageClass=
"Thumbnail",
165 dimensions=[
"Instrument",
"Visit",
"Detector"],
167 outputFlattenedThumbnail = pipeBase.OutputDatasetField(
168 doc=
"Output flat-corrected thumbnail image.",
169 name=
"FlattenedThumb",
171 storageClass=
"TextStorage",
172 dimensions=[
"Instrument",
"Visit",
"Detector"],
175 quantum = pipeBase.QuantumConfig(
176 dimensions=[
"Visit",
"Detector",
"Instrument"],
180 datasetType = pexConfig.Field(
182 doc=
"Dataset type for input data; users will typically leave this alone, " 183 "but camera-specific ISR tasks will override it",
187 fallbackFilterName = pexConfig.Field(
189 doc=
"Fallback default filter name for calibrations.",
192 expectWcs = pexConfig.Field(
195 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 197 fwhm = pexConfig.Field(
199 doc=
"FWHM of PSF in arcseconds.",
202 qa = pexConfig.ConfigField(
204 doc=
"QA related configuration options.",
208 doConvertIntToFloat = pexConfig.Field(
210 doc=
"Convert integer raw images to floating point values?",
215 doSaturation = pexConfig.Field(
217 doc=
"Mask saturated pixels? NB: this is totally independent of the" 218 " interpolation option - this is ONLY setting the bits in the mask." 219 " To have them interpolated make sure doSaturationInterpolation=True",
222 saturatedMaskName = pexConfig.Field(
224 doc=
"Name of mask plane to use in saturation detection and interpolation",
227 saturation = pexConfig.Field(
229 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
230 default=float(
"NaN"),
232 growSaturationFootprintSize = pexConfig.Field(
234 doc=
"Number of pixels by which to grow the saturation footprints",
239 doSuspect = pexConfig.Field(
241 doc=
"Mask suspect pixels?",
244 suspectMaskName = pexConfig.Field(
246 doc=
"Name of mask plane to use for suspect pixels",
249 numEdgeSuspect = pexConfig.Field(
251 doc=
"Number of edge pixels to be flagged as untrustworthy.",
256 doSetBadRegions = pexConfig.Field(
258 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
261 badStatistic = pexConfig.ChoiceField(
263 doc=
"How to estimate the average value for BAD regions.",
266 "MEANCLIP":
"Correct using the (clipped) mean of good data",
267 "MEDIAN":
"Correct using the median of the good data",
272 doOverscan = pexConfig.Field(
274 doc=
"Do overscan subtraction?",
277 overscanFitType = pexConfig.ChoiceField(
279 doc=
"The method for fitting the overscan bias level.",
282 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
283 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
284 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
285 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
286 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
287 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
288 "MEAN":
"Correct using the mean of the overscan region",
289 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
290 "MEDIAN":
"Correct using the median of the overscan region",
293 overscanOrder = pexConfig.Field(
295 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
296 "or number of spline knots if overscan fit type is a spline."),
299 overscanNumSigmaClip = pexConfig.Field(
301 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
304 overscanIsInt = pexConfig.Field(
306 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
309 overscanNumLeadingColumnsToSkip = pexConfig.Field(
311 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
314 overscanNumTrailingColumnsToSkip = pexConfig.Field(
316 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
319 overscanMaxDev = pexConfig.Field(
321 doc=
"Maximum deviation from the median for overscan",
322 default=1000.0, check=
lambda x: x > 0
324 overscanBiasJump = pexConfig.Field(
326 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
329 overscanBiasJumpKeyword = pexConfig.Field(
331 doc=
"Header keyword containing information about devices.",
332 default=
"NO_SUCH_KEY",
334 overscanBiasJumpDevices = pexConfig.ListField(
336 doc=
"List of devices that need piecewise overscan correction.",
339 overscanBiasJumpLocation = pexConfig.Field(
341 doc=
"Location of bias jump along y-axis.",
346 doAssembleCcd = pexConfig.Field(
349 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 351 assembleCcd = pexConfig.ConfigurableField(
352 target=AssembleCcdTask,
353 doc=
"CCD assembly task",
357 doAssembleIsrExposures = pexConfig.Field(
360 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 362 doTrimToMatchCalib = pexConfig.Field(
365 doc=
"Trim raw data to match calibration bounding boxes?" 369 doBias = pexConfig.Field(
371 doc=
"Apply bias frame correction?",
374 biasDataProductName = pexConfig.Field(
376 doc=
"Name of the bias data product",
381 doVariance = pexConfig.Field(
383 doc=
"Calculate variance?",
386 gain = pexConfig.Field(
388 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
389 default=float(
"NaN"),
391 readNoise = pexConfig.Field(
393 doc=
"The read noise to use if no Detector is present in the Exposure",
396 doEmpiricalReadNoise = pexConfig.Field(
399 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 403 doLinearize = pexConfig.Field(
405 doc=
"Correct for nonlinearity of the detector's response?",
410 doCrosstalk = pexConfig.Field(
412 doc=
"Apply intra-CCD crosstalk correction?",
415 doCrosstalkBeforeAssemble = pexConfig.Field(
417 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
420 crosstalk = pexConfig.ConfigurableField(
421 target=CrosstalkTask,
422 doc=
"Intra-CCD crosstalk correction",
426 doWidenSaturationTrails = pexConfig.Field(
428 doc=
"Widen bleed trails based on their width?",
433 doBrighterFatter = pexConfig.Field(
436 doc=
"Apply the brighter fatter correction" 438 brighterFatterLevel = pexConfig.ChoiceField(
441 doc=
"The level at which to correct for brighter-fatter.",
443 "AMP":
"Every amplifier treated separately.",
444 "DETECTOR":
"One kernel per detector",
447 brighterFatterKernelFile = pexConfig.Field(
450 doc=
"Kernel file used for the brighter fatter correction" 452 brighterFatterMaxIter = pexConfig.Field(
455 doc=
"Maximum number of iterations for the brighter fatter correction" 457 brighterFatterThreshold = pexConfig.Field(
460 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 461 " absolute value of the difference between the current corrected image and the one" 462 " from the previous iteration summed over all the pixels." 464 brighterFatterApplyGain = pexConfig.Field(
467 doc=
"Should the gain be applied when applying the brighter fatter correction?" 471 doDefect = pexConfig.Field(
473 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
476 doSaturationInterpolation = pexConfig.Field(
478 doc=
"Perform interpolation over pixels masked as saturated?" 479 " NB: This is independent of doSaturation; if that is False this plane" 480 " will likely be blank, resulting in a no-op here.",
483 numEdgeSuspect = pexConfig.Field(
485 doc=
"Number of edge pixels to be flagged as untrustworthy.",
490 doDark = pexConfig.Field(
492 doc=
"Apply dark frame correction?",
495 darkDataProductName = pexConfig.Field(
497 doc=
"Name of the dark data product",
502 doStrayLight = pexConfig.Field(
504 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
507 strayLight = pexConfig.ConfigurableField(
508 target=StrayLightTask,
509 doc=
"y-band stray light correction" 513 doFlat = pexConfig.Field(
515 doc=
"Apply flat field correction?",
518 flatDataProductName = pexConfig.Field(
520 doc=
"Name of the flat data product",
523 flatScalingType = pexConfig.ChoiceField(
525 doc=
"The method for scaling the flat on the fly.",
528 "USER":
"Scale by flatUserScale",
529 "MEAN":
"Scale by the inverse of the mean",
530 "MEDIAN":
"Scale by the inverse of the median",
533 flatUserScale = pexConfig.Field(
535 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
538 doTweakFlat = pexConfig.Field(
540 doc=
"Tweak flats to match observed amplifier ratios?",
545 doApplyGains = pexConfig.Field(
547 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
550 normalizeGains = pexConfig.Field(
552 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
557 doFringe = pexConfig.Field(
559 doc=
"Apply fringe correction?",
562 fringe = pexConfig.ConfigurableField(
564 doc=
"Fringe subtraction task",
566 fringeAfterFlat = pexConfig.Field(
568 doc=
"Do fringe subtraction after flat-fielding?",
573 doNanInterpAfterFlat = pexConfig.Field(
575 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 576 "also have to interpolate them before flat-fielding."),
581 doAddDistortionModel = pexConfig.Field(
583 doc=
"Apply a distortion model based on camera geometry to the WCS?",
588 doMeasureBackground = pexConfig.Field(
590 doc=
"Measure the background level on the reduced image?",
595 doCameraSpecificMasking = pexConfig.Field(
597 doc=
"Mask camera-specific bad regions?",
600 masking = pexConfig.ConfigurableField(
606 fluxMag0T1 = pexConfig.DictField(
609 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
610 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
613 defaultFluxMag0T1 = pexConfig.Field(
615 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
616 default=pow(10.0, 0.4*28.0)
620 doVignette = pexConfig.Field(
622 doc=
"Apply vignetting parameters?",
625 vignette = pexConfig.ConfigurableField(
627 doc=
"Vignetting task.",
631 doAttachTransmissionCurve = pexConfig.Field(
634 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 636 doUseOpticsTransmission = pexConfig.Field(
639 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 641 doUseFilterTransmission = pexConfig.Field(
644 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 646 doUseSensorTransmission = pexConfig.Field(
649 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 651 doUseAtmosphereTransmission = pexConfig.Field(
654 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 658 doWrite = pexConfig.Field(
660 doc=
"Persist postISRCCD?",
667 raise ValueError(
"You may not specify both doFlat and doApplyGains")
670 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
671 r"""Apply common instrument signature correction algorithms to a raw frame. 673 The process for correcting imaging data is very similar from 674 camera to camera. This task provides a vanilla implementation of 675 doing these corrections, including the ability to turn certain 676 corrections off if they are not needed. The inputs to the primary 677 method, `run()`, are a raw exposure to be corrected and the 678 calibration data products. The raw input is a single chip sized 679 mosaic of all amps including overscans and other non-science 680 pixels. The method `runDataRef()` identifies and defines the 681 calibration data products, and is intended for use by a 682 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 683 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 684 subclassed for different camera, although the most camera specific 685 methods have been split into subtasks that can be redirected 688 The __init__ method sets up the subtasks for ISR processing, using 689 the defaults from `lsst.ip.isr`. 694 Positional arguments passed to the Task constructor. None used at this time. 695 kwargs : `dict`, optional 696 Keyword arguments passed on to the Task constructor. None used at this time. 698 ConfigClass = IsrTaskConfig
703 self.makeSubtask(
"assembleCcd")
704 self.makeSubtask(
"crosstalk")
705 self.makeSubtask(
"strayLight")
706 self.makeSubtask(
"fringe")
707 self.makeSubtask(
"masking")
708 self.makeSubtask(
"vignette")
717 if config.doBias
is not True:
718 inputTypeDict.pop(
"bias",
None)
719 if config.doLinearize
is not True:
720 inputTypeDict.pop(
"linearizer",
None)
721 if config.doCrosstalk
is not True:
722 inputTypeDict.pop(
"crosstalkSources",
None)
723 if config.doBrighterFatter
is not True:
724 inputTypeDict.pop(
"bfKernel",
None)
725 if config.doDefect
is not True:
726 inputTypeDict.pop(
"defects",
None)
727 if config.doDark
is not True:
728 inputTypeDict.pop(
"dark",
None)
729 if config.doFlat
is not True:
730 inputTypeDict.pop(
"flat",
None)
731 if config.doAttachTransmissionCurve
is not True:
732 inputTypeDict.pop(
"opticsTransmission",
None)
733 inputTypeDict.pop(
"filterTransmission",
None)
734 inputTypeDict.pop(
"sensorTransmission",
None)
735 inputTypeDict.pop(
"atmosphereTransmission",
None)
736 if config.doUseOpticsTransmission
is not True:
737 inputTypeDict.pop(
"opticsTransmission",
None)
738 if config.doUseFilterTransmission
is not True:
739 inputTypeDict.pop(
"filterTransmission",
None)
740 if config.doUseSensorTransmission
is not True:
741 inputTypeDict.pop(
"sensorTransmission",
None)
742 if config.doUseAtmosphereTransmission
is not True:
743 inputTypeDict.pop(
"atmosphereTransmission",
None)
751 if config.qa.doThumbnailOss
is not True:
752 outputTypeDict.pop(
"outputOssThumbnail",
None)
753 if config.qa.doThumbnailFlattened
is not True:
754 outputTypeDict.pop(
"outputFlattenedThumbnail",
None)
755 if config.doWrite
is not True:
756 outputTypeDict.pop(
"outputExposure",
None)
758 return outputTypeDict
767 names.remove(
"ccdExposure")
776 return frozenset([
"CalibrationLabel"])
780 inputData[
'detectorNum'] = int(inputDataIds[
'ccdExposure'][
'detector'])
781 except Exception
as e:
782 raise ValueError(f
"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
784 inputData[
'isGen3'] =
True 786 if self.config.doLinearize
is True:
787 if 'linearizer' not in inputData.keys():
788 detector = inputData[
'camera'][inputData[
'detectorNum']]
789 linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
790 inputData[
'linearizer'] = linearize.getLinearityTypeByName(linearityName)()
792 if inputData[
'defects']
is not None:
797 for r
in inputData[
'defects']:
798 bbox = afwGeom.BoxI(afwGeom.PointI(r.get(
"x0"), r.get(
"y0")),
799 afwGeom.ExtentI(r.get(
"width"), r.get(
"height")))
800 defectList.append(
Defect(bbox))
802 inputData[
'defects'] = defectList
819 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
825 """!Retrieve necessary frames for instrument signature removal. 827 Pre-fetching all required ISR data products limits the IO 828 required by the ISR. Any conflict between the calibration data 829 available and that needed for ISR is also detected prior to 830 doing processing, allowing it to fail quickly. 834 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 835 Butler reference of the detector data to be processed 836 rawExposure : `afw.image.Exposure` 837 The raw exposure that will later be corrected with the 838 retrieved calibration data; should not be modified in this 843 result : `lsst.pipe.base.Struct` 844 Result struct with components (which may be `None`): 845 - ``bias``: bias calibration frame (`afw.image.Exposure`) 846 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 847 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 848 - ``dark``: dark calibration frame (`afw.image.Exposure`) 849 - ``flat``: flat calibration frame (`afw.image.Exposure`) 850 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 851 - ``defects``: list of defects (`list`) 852 - ``fringes``: `lsst.pipe.base.Struct` with components: 853 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 854 - ``seed``: random seed derived from the ccdExposureId for random 855 number generator (`uint32`) 856 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 857 A ``TransmissionCurve`` that represents the throughput of the optics, 858 to be evaluated in focal-plane coordinates. 859 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 860 A ``TransmissionCurve`` that represents the throughput of the filter 861 itself, to be evaluated in focal-plane coordinates. 862 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 863 A ``TransmissionCurve`` that represents the throughput of the sensor 864 itself, to be evaluated in post-assembly trimmed detector coordinates. 865 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 866 A ``TransmissionCurve`` that represents the throughput of the 867 atmosphere, assumed to be spatially constant. 868 - ``strayLightData`` : `object` 869 An opaque object containing calibration information for 870 stray-light correction. If `None`, no correction will be 875 NotImplementedError : 876 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration. 878 ccd = rawExposure.getDetector()
879 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
880 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
881 if self.config.doBias
else None)
883 linearizer = (dataRef.get(
"linearizer", immediate=
True)
885 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
886 if self.config.doCrosstalk
else None)
887 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
888 if self.config.doDark
else None)
889 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
890 if self.config.doFlat
else None)
892 brighterFatterKernel =
None 893 if self.config.doBrighterFatter
is True:
897 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
901 brighterFatterKernel = dataRef.get(
"bfKernel")
903 brighterFatterKernel =
None 904 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
907 if self.config.brighterFatterLevel ==
'DETECTOR':
908 brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
911 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
913 defectList = (dataRef.get(
"defects")
914 if self.config.doDefect
else None)
915 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
916 if self.config.doAssembleIsrExposures
else None)
917 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
918 else pipeBase.Struct(fringes=
None))
920 if self.config.doAttachTransmissionCurve:
921 opticsTransmission = (dataRef.get(
"transmission_optics")
922 if self.config.doUseOpticsTransmission
else None)
923 filterTransmission = (dataRef.get(
"transmission_filter")
924 if self.config.doUseFilterTransmission
else None)
925 sensorTransmission = (dataRef.get(
"transmission_sensor")
926 if self.config.doUseSensorTransmission
else None)
927 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
928 if self.config.doUseAtmosphereTransmission
else None)
930 opticsTransmission =
None 931 filterTransmission =
None 932 sensorTransmission =
None 933 atmosphereTransmission =
None 935 if self.config.doStrayLight:
936 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
938 strayLightData =
None 941 return pipeBase.Struct(bias=biasExposure,
942 linearizer=linearizer,
943 crosstalkSources=crosstalkSources,
946 bfKernel=brighterFatterKernel,
948 fringes=fringeStruct,
949 opticsTransmission=opticsTransmission,
950 filterTransmission=filterTransmission,
951 sensorTransmission=sensorTransmission,
952 atmosphereTransmission=atmosphereTransmission,
953 strayLightData=strayLightData
957 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
958 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
959 opticsTransmission=None, filterTransmission=None,
960 sensorTransmission=None, atmosphereTransmission=None,
961 detectorNum=None, strayLightData=None, isGen3=False,
963 """!Perform instrument signature removal on an exposure. 965 Steps included in the ISR processing, in order performed, are: 966 - saturation and suspect pixel masking 967 - overscan subtraction 968 - CCD assembly of individual amplifiers 970 - variance image construction 971 - linearization of non-linear response 973 - brighter-fatter correction 976 - stray light subtraction 978 - masking of known defects and camera specific features 979 - vignette calculation 980 - appending transmission curve and distortion model 984 ccdExposure : `lsst.afw.image.Exposure` 985 The raw exposure that is to be run through ISR. The 986 exposure is modified by this method. 987 camera : `lsst.afw.cameraGeom.Camera`, optional 988 The camera geometry for this exposure. Used to select the 989 distortion model appropriate for this data. 990 bias : `lsst.afw.image.Exposure`, optional 991 Bias calibration frame. 992 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 993 Functor for linearization. 994 crosstalkSources : `list`, optional 995 List of possible crosstalk sources. 996 dark : `lsst.afw.image.Exposure`, optional 997 Dark calibration frame. 998 flat : `lsst.afw.image.Exposure`, optional 999 Flat calibration frame. 1000 bfKernel : `numpy.ndarray`, optional 1001 Brighter-fatter kernel. 1002 defects : `list`, optional 1004 fringes : `lsst.pipe.base.Struct`, optional 1005 Struct containing the fringe correction data, with 1007 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 1008 - ``seed``: random seed derived from the ccdExposureId for random 1009 number generator (`uint32`) 1010 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 1011 A ``TransmissionCurve`` that represents the throughput of the optics, 1012 to be evaluated in focal-plane coordinates. 1013 filterTransmission : `lsst.afw.image.TransmissionCurve` 1014 A ``TransmissionCurve`` that represents the throughput of the filter 1015 itself, to be evaluated in focal-plane coordinates. 1016 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1017 A ``TransmissionCurve`` that represents the throughput of the sensor 1018 itself, to be evaluated in post-assembly trimmed detector coordinates. 1019 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1020 A ``TransmissionCurve`` that represents the throughput of the 1021 atmosphere, assumed to be spatially constant. 1022 detectorNum : `int`, optional 1023 The integer number for the detector to process. 1024 isGen3 : bool, optional 1025 Flag this call to run() as using the Gen3 butler environment. 1026 strayLightData : `object`, optional 1027 Opaque object containing calibration information for stray-light 1028 correction. If `None`, no correction will be performed. 1032 result : `lsst.pipe.base.Struct` 1033 Result struct with component: 1034 - ``exposure`` : `afw.image.Exposure` 1035 The fully ISR corrected exposure. 1036 - ``outputExposure`` : `afw.image.Exposure` 1037 An alias for `exposure` 1038 - ``ossThumb`` : `numpy.ndarray` 1039 Thumbnail image of the exposure after overscan subtraction. 1040 - ``flattenedThumb`` : `numpy.ndarray` 1041 Thumbnail image of the exposure after flat-field correction. 1046 Raised if a configuration option is set to True, but the 1047 required calibration data has not been specified. 1051 The current processed exposure can be viewed by setting the 1052 appropriate lsstDebug entries in the `debug.display` 1053 dictionary. The names of these entries correspond to some of 1054 the IsrTaskConfig Boolean options, with the value denoting the 1055 frame to use. The exposure is shown inside the matching 1056 option check and after the processing of that step has 1057 finished. The steps with debug points are: 1068 In addition, setting the "postISRCCD" entry displays the 1069 exposure after all ISR processing has finished. 1077 self.config.doFringe =
False 1080 if detectorNum
is None:
1081 raise RuntimeError(
"Must supply the detectorNum if running as Gen3")
1083 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1088 if isinstance(ccdExposure, ButlerDataRef):
1091 ccd = ccdExposure.getDetector()
1094 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 1095 ccd = [
FakeAmp(ccdExposure, self.config)]
1098 if self.config.doBias
and bias
is None:
1099 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1101 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1102 if self.config.doBrighterFatter
and bfKernel
is None:
1103 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1104 if self.config.doDark
and dark
is None:
1105 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1107 fringes = pipeBase.Struct(fringes=
None)
1108 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
1109 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1110 if self.config.doFlat
and flat
is None:
1111 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1112 if self.config.doDefect
and defects
is None:
1113 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1114 if self.config.doAddDistortionModel
and camera
is None:
1115 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
1118 if self.config.doConvertIntToFloat:
1119 self.log.info(
"Converting exposure to floating point values")
1126 if ccdExposure.getBBox().contains(amp.getBBox()):
1130 if self.config.doOverscan
and not badAmp:
1133 self.log.debug(
"Corrected overscan for amplifier %s" % (amp.getName()))
1134 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1135 if isinstance(overscanResults.overscanFit, float):
1136 qaMedian = overscanResults.overscanFit
1137 qaStdev = float(
"NaN")
1139 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1140 afwMath.MEDIAN | afwMath.STDEVCLIP)
1141 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1142 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1144 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1145 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1146 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f" %
1147 (amp.getName(), qaMedian, qaStdev))
1148 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1150 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
1151 overscanResults =
None 1153 overscans.append(overscanResults
if overscanResults
is not None else None)
1155 self.log.info(
"Skipped OSCAN")
1157 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1158 self.log.info(
"Applying crosstalk correction.")
1159 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
1160 self.
debugView(ccdExposure,
"doCrosstalk")
1162 if self.config.doAssembleCcd:
1163 self.log.info(
"Assembling CCD from amplifiers")
1164 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1166 if self.config.expectWcs
and not ccdExposure.getWcs():
1167 self.log.warn(
"No WCS found in input exposure")
1168 self.
debugView(ccdExposure,
"doAssembleCcd")
1171 if self.config.qa.doThumbnailOss:
1172 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1174 if self.config.doBias:
1175 self.log.info(
"Applying bias correction.")
1176 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1177 trimToFit=self.config.doTrimToMatchCalib)
1180 if self.config.doVariance:
1181 for amp, overscanResults
in zip(ccd, overscans):
1182 if ccdExposure.getBBox().contains(amp.getBBox()):
1183 self.log.debug(
"Constructing variance map for amplifer %s" % (amp.getName()))
1184 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1185 if overscanResults
is not None:
1187 overscanImage=overscanResults.overscanImage)
1191 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1192 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1193 afwMath.MEDIAN | afwMath.STDEVCLIP)
1194 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
1195 qaStats.getValue(afwMath.MEDIAN))
1196 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
1197 qaStats.getValue(afwMath.STDEVCLIP))
1198 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f" %
1199 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1200 qaStats.getValue(afwMath.STDEVCLIP)))
1203 self.log.info(
"Applying linearizer.")
1204 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1206 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1207 self.log.info(
"Applying crosstalk correction.")
1208 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=
True)
1209 self.
debugView(ccdExposure,
"doCrosstalk")
1211 if self.config.doWidenSaturationTrails:
1212 self.log.info(
"Widening saturation trails.")
1213 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1215 interpolationDone =
False 1216 if self.config.doBrighterFatter:
1222 if self.config.doDefect:
1225 if self.config.doSaturationInterpolation:
1229 interpolationDone =
True 1231 self.log.info(
"Applying brighter fatter correction.")
1232 isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1233 self.config.brighterFatterMaxIter,
1234 self.config.brighterFatterThreshold,
1235 self.config.brighterFatterApplyGain,
1237 self.
debugView(ccdExposure,
"doBrighterFatter")
1239 if self.config.doDark:
1240 self.log.info(
"Applying dark correction.")
1244 if self.config.doFringe
and not self.config.fringeAfterFlat:
1245 self.log.info(
"Applying fringe correction before flat.")
1246 self.fringe.
run(ccdExposure, **fringes.getDict())
1249 if self.config.doStrayLight:
1250 if strayLightData
is not None:
1251 self.log.info(
"Applying stray light correction.")
1252 self.strayLight.
run(ccdExposure, strayLightData)
1253 self.
debugView(ccdExposure,
"doStrayLight")
1255 self.log.debug(
"Skipping stray light correction: no data found for this image.")
1257 if self.config.doFlat:
1258 self.log.info(
"Applying flat correction.")
1262 if self.config.doApplyGains:
1263 self.log.info(
"Applying gain correction instead of flat.")
1264 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1266 if self.config.doDefect
and not interpolationDone:
1267 self.log.info(
"Masking and interpolating defects.")
1270 if self.config.doSaturationInterpolation
and not interpolationDone:
1271 self.log.info(
"Interpolating saturated pixels.")
1274 if self.config.doNanInterpAfterFlat
or not interpolationDone:
1275 self.log.info(
"Masking and interpolating NAN value pixels.")
1278 if self.config.doFringe
and self.config.fringeAfterFlat:
1279 self.log.info(
"Applying fringe correction after flat.")
1280 self.fringe.
run(ccdExposure, **fringes.getDict())
1282 if self.config.doSetBadRegions:
1283 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1284 if badPixelCount > 0:
1285 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1287 flattenedThumb =
None 1288 if self.config.qa.doThumbnailFlattened:
1289 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1291 if self.config.doCameraSpecificMasking:
1292 self.log.info(
"Masking regions for camera specific reasons.")
1293 self.masking.
run(ccdExposure)
1297 if self.config.doVignette:
1298 self.log.info(
"Constructing Vignette polygon.")
1301 if self.config.vignette.doWriteVignettePolygon:
1304 if self.config.doAttachTransmissionCurve:
1305 self.log.info(
"Adding transmission curves.")
1306 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1307 filterTransmission=filterTransmission,
1308 sensorTransmission=sensorTransmission,
1309 atmosphereTransmission=atmosphereTransmission)
1311 if self.config.doAddDistortionModel:
1312 self.log.info(
"Adding a distortion model to the WCS.")
1313 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1315 if self.config.doMeasureBackground:
1316 self.log.info(
"Measuring background level:")
1319 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1321 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1322 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1323 afwMath.MEDIAN | afwMath.STDEVCLIP)
1324 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1325 qaStats.getValue(afwMath.MEDIAN))
1326 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1327 qaStats.getValue(afwMath.STDEVCLIP))
1328 self.log.debug(
" Background stats for amplifer %s: %f +/- %f" %
1329 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1330 qaStats.getValue(afwMath.STDEVCLIP)))
1332 self.
debugView(ccdExposure,
"postISRCCD")
1334 return pipeBase.Struct(
1335 exposure=ccdExposure,
1337 flattenedThumb=flattenedThumb,
1339 outputExposure=ccdExposure,
1340 outputOssThumbnail=ossThumb,
1341 outputFlattenedThumbnail=flattenedThumb,
1344 @pipeBase.timeMethod
1346 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1348 This method contains the `CmdLineTask` interface to the ISR 1349 processing. All IO is handled here, freeing the `run()` method 1350 to manage only pixel-level calculations. The steps performed 1352 - Read in necessary detrending/isr/calibration data. 1353 - Process raw exposure in `run()`. 1354 - Persist the ISR-corrected exposure as "postISRCCD" if 1355 config.doWrite=True. 1359 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1360 DataRef of the detector data to be processed 1364 result : `lsst.pipe.base.Struct` 1365 Result struct with component: 1366 - ``exposure`` : `afw.image.Exposure` 1367 The fully ISR corrected exposure. 1372 Raised if a configuration option is set to True, but the 1373 required calibration data does not exist. 1376 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1378 ccdExposure = sensorRef.get(self.config.datasetType)
1380 camera = sensorRef.get(
"camera")
1381 if camera
is None and self.config.doAddDistortionModel:
1382 raise RuntimeError(
"config.doAddDistortionModel is True " 1383 "but could not get a camera from the butler")
1384 isrData = self.
readIsrData(sensorRef, ccdExposure)
1386 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1388 if self.config.doWrite:
1389 sensorRef.put(result.exposure,
"postISRCCD")
1390 if result.ossThumb
is not None:
1391 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1392 if result.flattenedThumb
is not None:
1393 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1398 """!Retrieve a calibration dataset for removing instrument signature. 1403 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1404 DataRef of the detector data to find calibration datasets 1407 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1409 If True, disable butler proxies to enable error handling 1410 within this routine. 1414 exposure : `lsst.afw.image.Exposure` 1415 Requested calibration frame. 1420 Raised if no matching calibration frame can be found. 1423 exp = dataRef.get(datasetType, immediate=immediate)
1424 except Exception
as exc1:
1425 if not self.config.fallbackFilterName:
1426 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1428 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1429 except Exception
as exc2:
1430 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1431 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1432 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1434 if self.config.doAssembleIsrExposures:
1435 exp = self.assembleCcd.assembleCcd(exp)
1439 """Ensure that the data returned by Butler is a fully constructed exposure. 1441 ISR requires exposure-level image data for historical reasons, so if we did 1442 not recieve that from Butler, construct it from what we have, modifying the 1447 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or 1448 `lsst.afw.image.ImageF` 1449 The input data structure obtained from Butler. 1450 camera : `lsst.afw.cameraGeom.camera` 1451 The camera associated with the image. Used to find the appropriate 1454 The detector this exposure should match. 1458 inputExp : `lsst.afw.image.Exposure` 1459 The re-constructed exposure, with appropriate detector parameters. 1464 Raised if the input data cannot be used to construct an exposure. 1466 if isinstance(inputExp, afwImage.DecoratedImageU):
1467 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1468 elif isinstance(inputExp, afwImage.ImageF):
1469 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1470 elif isinstance(inputExp, afwImage.MaskedImageF):
1471 inputExp = afwImage.makeExposure(inputExp)
1472 elif isinstance(inputExp, afwImage.Exposure):
1475 raise TypeError(f
"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1477 if inputExp.getDetector()
is None:
1478 inputExp.setDetector(camera[detectorNum])
1483 """Convert exposure image from uint16 to float. 1485 If the exposure does not need to be converted, the input is 1486 immediately returned. For exposures that are converted to use 1487 floating point pixels, the variance is set to unity and the 1492 exposure : `lsst.afw.image.Exposure` 1493 The raw exposure to be converted. 1497 newexposure : `lsst.afw.image.Exposure` 1498 The input ``exposure``, converted to floating point pixels. 1503 Raised if the exposure type cannot be converted to float. 1506 if isinstance(exposure, afwImage.ExposureF):
1509 if not hasattr(exposure,
"convertF"):
1510 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1512 newexposure = exposure.convertF()
1513 newexposure.variance[:] = 1
1514 newexposure.mask[:] = 0x0
1519 """Identify bad amplifiers, saturated and suspect pixels. 1523 ccdExposure : `lsst.afw.image.Exposure` 1524 Input exposure to be masked. 1525 amp : `lsst.afw.table.AmpInfoCatalog` 1526 Catalog of parameters defining the amplifier on this 1529 List of defects. Used to determine if the entire 1535 If this is true, the entire amplifier area is covered by 1536 defects and unusable. 1539 maskedImage = ccdExposure.getMaskedImage()
1545 if defects
is not None:
1546 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1551 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1553 maskView = dataView.getMask()
1554 maskView |= maskView.getPlaneBitMask(
"BAD")
1561 if self.config.doSaturation
and not badAmp:
1562 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1563 if self.config.doSuspect
and not badAmp:
1564 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1566 for maskName, maskThreshold
in limits.items():
1567 if not math.isnan(maskThreshold):
1568 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1569 isrFunctions.makeThresholdMask(
1570 maskedImage=dataView,
1571 threshold=maskThreshold,
1577 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1579 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1580 self.config.suspectMaskName])
1581 if numpy.all(maskView.getArray() & maskVal > 0):
1587 """Apply overscan correction in place. 1589 This method does initial pixel rejection of the overscan 1590 region. The overscan can also be optionally segmented to 1591 allow for discontinuous overscan responses to be fit 1592 separately. The actual overscan subtraction is performed by 1593 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1594 which is called here after the amplifier is preprocessed. 1598 ccdExposure : `lsst.afw.image.Exposure` 1599 Exposure to have overscan correction performed. 1600 amp : `lsst.afw.table.AmpInfoCatalog` 1601 The amplifier to consider while correcting the overscan. 1605 overscanResults : `lsst.pipe.base.Struct` 1606 Result struct with components: 1607 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1608 Value or fit subtracted from the amplifier image data. 1609 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1610 Value or fit subtracted from the overscan image data. 1611 - ``overscanImage`` : `lsst.afw.image.Image` 1612 Image of the overscan region with the overscan 1613 correction applied. This quantity is used to estimate 1614 the amplifier read noise empirically. 1619 Raised if the ``amp`` does not contain raw pixel information. 1623 lsst.ip.isr.isrFunctions.overscanCorrection 1625 if not amp.getHasRawInfo():
1626 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1628 if amp.getRawHorizontalOverscanBBox().isEmpty():
1629 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1632 statControl = afwMath.StatisticsControl()
1633 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1636 dataBBox = amp.getRawDataBBox()
1637 oscanBBox = amp.getRawHorizontalOverscanBBox()
1641 prescanBBox = amp.getRawPrescanBBox()
1642 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1643 dx0 += self.config.overscanNumLeadingColumnsToSkip
1644 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1646 dx0 += self.config.overscanNumTrailingColumnsToSkip
1647 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1653 if ((self.config.overscanBiasJump
and 1654 self.config.overscanBiasJumpLocation)
and 1655 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1656 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1657 self.config.overscanBiasJumpDevices)):
1658 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1659 yLower = self.config.overscanBiasJumpLocation
1660 yUpper = dataBBox.getHeight() - yLower
1662 yUpper = self.config.overscanBiasJumpLocation
1663 yLower = dataBBox.getHeight() - yUpper
1665 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1666 afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1667 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1668 afwGeom.Extent2I(dx0, 0),
1669 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1672 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1673 afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1674 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1675 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1678 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1679 afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1680 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1681 afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1682 oscanBBox.getHeight())))
1685 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1686 ampImage = ccdExposure.maskedImage[imageBBox]
1687 overscanImage = ccdExposure.maskedImage[overscanBBox]
1689 overscanArray = overscanImage.image.array
1690 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1691 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1692 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1694 statControl = afwMath.StatisticsControl()
1695 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1697 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1698 overscanImage=overscanImage,
1699 fitType=self.config.overscanFitType,
1700 order=self.config.overscanOrder,
1701 collapseRej=self.config.overscanNumSigmaClip,
1702 statControl=statControl,
1703 overscanIsInt=self.config.overscanIsInt
1707 levelStat = afwMath.MEDIAN
1708 sigmaStat = afwMath.STDEVCLIP
1710 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1711 self.config.qa.flatness.nIter)
1712 metadata = ccdExposure.getMetadata()
1713 ampNum = amp.getName()
1714 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1715 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1716 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1718 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1719 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1720 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1722 return overscanResults
1725 """Set the variance plane using the amplifier gain and read noise 1727 The read noise is calculated from the ``overscanImage`` if the 1728 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1729 the value from the amplifier data is used. 1733 ampExposure : `lsst.afw.image.Exposure` 1734 Exposure to process. 1735 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1736 Amplifier detector data. 1737 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1738 Image of overscan, required only for empirical read noise. 1742 lsst.ip.isr.isrFunctions.updateVariance 1744 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1745 gain = amp.getGain()
1747 if math.isnan(gain):
1749 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1752 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1753 (amp.getName(), gain, patchedGain))
1756 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1757 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1759 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1760 stats = afwMath.StatisticsControl()
1761 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1762 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1763 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1765 readNoise = amp.getReadNoise()
1767 isrFunctions.updateVariance(
1768 maskedImage=ampExposure.getMaskedImage(),
1770 readNoise=readNoise,
1774 """!Apply dark correction in place. 1778 exposure : `lsst.afw.image.Exposure` 1779 Exposure to process. 1780 darkExposure : `lsst.afw.image.Exposure` 1781 Dark exposure of the same size as ``exposure``. 1782 invert : `Bool`, optional 1783 If True, re-add the dark to an already corrected image. 1788 Raised if either ``exposure`` or ``darkExposure`` do not 1789 have their dark time defined. 1793 lsst.ip.isr.isrFunctions.darkCorrection 1795 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1796 if math.isnan(expScale):
1797 raise RuntimeError(
"Exposure darktime is NAN")
1798 if darkExposure.getInfo().getVisitInfo()
is not None:
1799 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1805 if math.isnan(darkScale):
1806 raise RuntimeError(
"Dark calib darktime is NAN")
1807 isrFunctions.darkCorrection(
1808 maskedImage=exposure.getMaskedImage(),
1809 darkMaskedImage=darkExposure.getMaskedImage(),
1811 darkScale=darkScale,
1813 trimToFit=self.config.doTrimToMatchCalib
1817 """!Check if linearization is needed for the detector cameraGeom. 1819 Checks config.doLinearize and the linearity type of the first 1824 detector : `lsst.afw.cameraGeom.Detector` 1825 Detector to get linearity type from. 1829 doLinearize : `Bool` 1830 If True, linearization should be performed. 1832 return self.config.doLinearize
and \
1833 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1836 """!Apply flat correction in place. 1840 exposure : `lsst.afw.image.Exposure` 1841 Exposure to process. 1842 flatExposure : `lsst.afw.image.Exposure` 1843 Flat exposure of the same size as ``exposure``. 1844 invert : `Bool`, optional 1845 If True, unflatten an already flattened image. 1849 lsst.ip.isr.isrFunctions.flatCorrection 1851 isrFunctions.flatCorrection(
1852 maskedImage=exposure.getMaskedImage(),
1853 flatMaskedImage=flatExposure.getMaskedImage(),
1854 scalingType=self.config.flatScalingType,
1855 userScale=self.config.flatUserScale,
1857 trimToFit=self.config.doTrimToMatchCalib
1861 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1865 exposure : `lsst.afw.image.Exposure` 1866 Exposure to process. Only the amplifier DataSec is processed. 1867 amp : `lsst.afw.table.AmpInfoCatalog` 1868 Amplifier detector data. 1872 lsst.ip.isr.isrFunctions.makeThresholdMask 1874 if not math.isnan(amp.getSaturation()):
1875 maskedImage = exposure.getMaskedImage()
1876 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1877 isrFunctions.makeThresholdMask(
1878 maskedImage=dataView,
1879 threshold=amp.getSaturation(),
1881 maskName=self.config.saturatedMaskName,
1885 """!Interpolate over saturated pixels, in place. 1887 This method should be called after `saturationDetection`, to 1888 ensure that the saturated pixels have been identified in the 1889 SAT mask. It should also be called after `assembleCcd`, since 1890 saturated regions may cross amplifier boundaries. 1894 exposure : `lsst.afw.image.Exposure` 1895 Exposure to process. 1899 lsst.ip.isr.isrTask.saturationDetection 1900 lsst.ip.isr.isrFunctions.interpolateFromMask 1902 isrFunctions.interpolateFromMask(
1903 maskedImage=ccdExposure.getMaskedImage(),
1904 fwhm=self.config.fwhm,
1905 growFootprints=self.config.growSaturationFootprintSize,
1906 maskName=self.config.saturatedMaskName,
1910 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1914 exposure : `lsst.afw.image.Exposure` 1915 Exposure to process. Only the amplifier DataSec is processed. 1916 amp : `lsst.afw.table.AmpInfoCatalog` 1917 Amplifier detector data. 1921 lsst.ip.isr.isrFunctions.makeThresholdMask 1925 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1926 This is intended to indicate pixels that may be affected by unknown systematics; 1927 for example if non-linearity corrections above a certain level are unstable 1928 then that would be a useful value for suspectLevel. A value of `nan` indicates 1929 that no such level exists and no pixels are to be masked as suspicious. 1931 suspectLevel = amp.getSuspectLevel()
1932 if math.isnan(suspectLevel):
1935 maskedImage = exposure.getMaskedImage()
1936 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1937 isrFunctions.makeThresholdMask(
1938 maskedImage=dataView,
1939 threshold=suspectLevel,
1941 maskName=self.config.suspectMaskName,
1945 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1949 ccdExposure : `lsst.afw.image.Exposure` 1950 Exposure to process. 1951 defectBaseList : `List` 1952 List of defects to mask and interpolate. 1956 Call this after CCD assembly, since defects may cross amplifier boundaries. 1958 maskedImage = ccdExposure.getMaskedImage()
1960 for d
in defectBaseList:
1962 nd = measAlg.Defect(bbox)
1963 defectList.append(nd)
1964 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
1965 isrFunctions.interpolateDefectList(
1966 maskedImage=maskedImage,
1967 defectList=defectList,
1968 fwhm=self.config.fwhm,
1971 if self.config.numEdgeSuspect > 0:
1972 goodBBox = maskedImage.getBBox()
1974 goodBBox.grow(-self.config.numEdgeSuspect)
1976 SourceDetectionTask.setEdgeBits(
1979 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1983 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1987 exposure : `lsst.afw.image.Exposure` 1988 Exposure to process. 1992 We mask and interpolate over all NaNs, including those 1993 that are masked with other bits (because those may or may 1994 not be interpolated over later, and we want to remove all 1995 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1996 is used to preserve the historical name. 1998 maskedImage = exposure.getMaskedImage()
2001 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2002 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2003 numNans =
maskNans(maskedImage, maskVal)
2004 self.metadata.set(
"NUMNANS", numNans)
2008 self.log.warn(
"There were %i unmasked NaNs", numNans)
2009 nanDefectList = isrFunctions.getDefectListFromMask(
2010 maskedImage=maskedImage,
2011 maskName=
'UNMASKEDNAN',
2013 isrFunctions.interpolateDefectList(
2014 maskedImage=exposure.getMaskedImage(),
2015 defectList=nanDefectList,
2016 fwhm=self.config.fwhm,
2020 """Measure the image background in subgrids, for quality control purposes. 2024 exposure : `lsst.afw.image.Exposure` 2025 Exposure to process. 2026 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 2027 Configuration object containing parameters on which background 2028 statistics and subgrids to use. 2030 if IsrQaConfig
is not None:
2031 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2032 IsrQaConfig.flatness.nIter)
2033 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2034 statsControl.setAndMask(maskVal)
2035 maskedImage = exposure.getMaskedImage()
2036 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2037 skyLevel = stats.getValue(afwMath.MEDIAN)
2038 skySigma = stats.getValue(afwMath.STDEVCLIP)
2039 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2040 metadata = exposure.getMetadata()
2041 metadata.set(
'SKYLEVEL', skyLevel)
2042 metadata.set(
'SKYSIGMA', skySigma)
2045 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2046 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2047 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2048 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2049 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2050 skyLevels = numpy.zeros((nX, nY))
2053 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2055 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2057 xLLC = xc - meshXHalf
2058 yLLC = yc - meshYHalf
2059 xURC = xc + meshXHalf - 1
2060 yURC = yc + meshYHalf - 1
2062 bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2063 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2065 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2067 good = numpy.where(numpy.isfinite(skyLevels))
2068 skyMedian = numpy.median(skyLevels[good])
2069 flatness = (skyLevels[good] - skyMedian) / skyMedian
2070 flatness_rms = numpy.std(flatness)
2071 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2073 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2074 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
2075 (nX, nY, flatness_pp, flatness_rms))
2077 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2078 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2079 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2080 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2081 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2084 """Set an approximate magnitude zero point for the exposure. 2088 exposure : `lsst.afw.image.Exposure` 2089 Exposure to process. 2091 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2092 if filterName
in self.config.fluxMag0T1:
2093 fluxMag0 = self.config.fluxMag0T1[filterName]
2095 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
2096 fluxMag0 = self.config.defaultFluxMag0T1
2098 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2100 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
2103 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2104 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2107 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 2111 ccdExposure : `lsst.afw.image.Exposure` 2112 Exposure to process. 2113 fpPolygon : `lsst.afw.geom.Polygon` 2114 Polygon in focal plane coordinates. 2117 ccd = ccdExposure.getDetector()
2118 fpCorners = ccd.getCorners(FOCAL_PLANE)
2119 ccdPolygon = Polygon(fpCorners)
2122 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2125 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2126 validPolygon = Polygon(ccdPoints)
2127 ccdExposure.getInfo().setValidPolygon(validPolygon)
2131 """Context manager that applies and removes flats and darks, 2132 if the task is configured to apply them. 2136 exp : `lsst.afw.image.Exposure` 2137 Exposure to process. 2138 flat : `lsst.afw.image.Exposure` 2139 Flat exposure the same size as ``exp``. 2140 dark : `lsst.afw.image.Exposure`, optional 2141 Dark exposure the same size as ``exp``. 2145 exp : `lsst.afw.image.Exposure` 2146 The flat and dark corrected exposure. 2148 if self.config.doDark
and dark
is not None:
2150 if self.config.doFlat:
2155 if self.config.doFlat:
2157 if self.config.doDark
and dark
is not None:
2161 """Utility function to examine ISR exposure at different stages. 2165 exposure : `lsst.afw.image.Exposure` 2168 State of processing to view. 2170 frame = getDebugFrame(self._display, stepname)
2172 display = getDisplay(frame)
2173 display.scale(
'asinh',
'zscale')
2174 display.mtv(exposure)
2178 """A Detector-like object that supports returning gain and saturation level 2180 This is used when the input exposure does not have a detector. 2184 exposure : `lsst.afw.image.Exposure` 2185 Exposure to generate a fake amplifier for. 2186 config : `lsst.ip.isr.isrTaskConfig` 2187 Configuration to apply to the fake amplifier. 2191 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2193 self.
_gain = config.gain
2223 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2227 """Task to wrap the default IsrTask to allow it to be retargeted. 2229 The standard IsrTask can be called directly from a command line 2230 program, but doing so removes the ability of the task to be 2231 retargeted. As most cameras override some set of the IsrTask 2232 methods, this would remove those data-specific methods in the 2233 output post-ISR images. This wrapping class fixes the issue, 2234 allowing identical post-ISR images to be generated by both the 2235 processCcd and isrTask code. 2237 ConfigClass = RunIsrConfig
2238 _DefaultName =
"runIsr" 2242 self.makeSubtask(
"isr")
2248 dataRef : `lsst.daf.persistence.ButlerDataRef` 2249 data reference of the detector data to be processed 2253 result : `pipeBase.Struct` 2254 Result struct with component: 2256 - exposure : `lsst.afw.image.Exposure` 2257 Post-ISR processed exposure. def getInputDatasetTypes(cls, config)
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, kwargs)
def ensureExposure(self, inputExp, camera, detectorNum)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def getPrerequisiteDatasetTypes(cls, config)
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def getOutputDatasetTypes(cls, config)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def makeDatasetType(self, dsConfig)
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def getPerDatasetTypeDimensions(cls, config)
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, isGen3=False)
Perform instrument signature removal on an exposure.
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)