28 import lsst.pex.config
as pexConfig
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DataCoordinate, DimensionGraph, DimensionUniverse
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.afw.image.Exposure`]
87 List of datasets that match the query that will be used as
90 newDataId = DataCoordinate(DimensionGraph(DimensionUniverse(),
91 names=(
'instrument',
'exposure')),
92 (quantumDataId[
'instrument'], quantumDataId[
'exposure']))
93 results = list(registry.queryDatasets(datasetType,
94 collections=collections,
102 dimensions={
"instrument",
"exposure",
"detector"},
103 defaultTemplates={}):
104 ccdExposure = cT.Input(
106 doc=
"Input exposure to process.",
107 storageClass=
"Exposure",
108 dimensions=[
"instrument",
"exposure",
"detector"],
110 camera = cT.PrerequisiteInput(
112 storageClass=
"Camera",
113 doc=
"Input camera to construct complete exposures.",
114 dimensions=[
"instrument",
"calibration_label"],
117 crosstalk = cT.PrerequisiteInput(
119 doc=
"Input crosstalk object",
120 storageClass=
"CrosstalkCalib",
121 dimensions=[
"instrument",
"calibration_label",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
134 bias = cT.PrerequisiteInput(
136 doc=
"Input bias calibration.",
137 storageClass=
"ExposureF",
138 dimensions=[
"instrument",
"calibration_label",
"detector"],
140 dark = cT.PrerequisiteInput(
142 doc=
"Input dark calibration.",
143 storageClass=
"ExposureF",
144 dimensions=[
"instrument",
"calibration_label",
"detector"],
146 flat = cT.PrerequisiteInput(
148 doc=
"Input flat calibration.",
149 storageClass=
"ExposureF",
150 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
152 fringes = cT.PrerequisiteInput(
154 doc=
"Input fringe calibration.",
155 storageClass=
"ExposureF",
156 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
158 strayLightData = cT.PrerequisiteInput(
160 doc=
"Input stray light calibration.",
161 storageClass=
"StrayLightData",
162 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
164 bfKernel = cT.PrerequisiteInput(
166 doc=
"Input brighter-fatter kernel.",
167 storageClass=
"NumpyArray",
168 dimensions=[
"instrument",
"calibration_label"],
170 newBFKernel = cT.PrerequisiteInput(
171 name=
'brighterFatterKernel',
172 doc=
"Newer complete kernel + gain solutions.",
173 storageClass=
"BrighterFatterKernel",
174 dimensions=[
"instrument",
"calibration_label",
"detector"],
176 defects = cT.PrerequisiteInput(
178 doc=
"Input defect tables.",
179 storageClass=
"Defects",
180 dimensions=[
"instrument",
"calibration_label",
"detector"],
182 opticsTransmission = cT.PrerequisiteInput(
183 name=
"transmission_optics",
184 storageClass=
"TransmissionCurve",
185 doc=
"Transmission curve due to the optics.",
186 dimensions=[
"instrument",
"calibration_label"],
188 filterTransmission = cT.PrerequisiteInput(
189 name=
"transmission_filter",
190 storageClass=
"TransmissionCurve",
191 doc=
"Transmission curve due to the filter.",
192 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
194 sensorTransmission = cT.PrerequisiteInput(
195 name=
"transmission_sensor",
196 storageClass=
"TransmissionCurve",
197 doc=
"Transmission curve due to the sensor.",
198 dimensions=[
"instrument",
"calibration_label",
"detector"],
200 atmosphereTransmission = cT.PrerequisiteInput(
201 name=
"transmission_atmosphere",
202 storageClass=
"TransmissionCurve",
203 doc=
"Transmission curve due to the atmosphere.",
204 dimensions=[
"instrument"],
206 illumMaskedImage = cT.PrerequisiteInput(
208 doc=
"Input illumination correction.",
209 storageClass=
"MaskedImageF",
210 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
213 outputExposure = cT.Output(
215 doc=
"Output ISR processed exposure.",
216 storageClass=
"Exposure",
217 dimensions=[
"instrument",
"exposure",
"detector"],
219 preInterpExposure = cT.Output(
220 name=
'preInterpISRCCD',
221 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
222 storageClass=
"ExposureF",
223 dimensions=[
"instrument",
"exposure",
"detector"],
225 outputOssThumbnail = cT.Output(
227 doc=
"Output Overscan-subtracted thumbnail image.",
228 storageClass=
"Thumbnail",
229 dimensions=[
"instrument",
"exposure",
"detector"],
231 outputFlattenedThumbnail = cT.Output(
232 name=
"FlattenedThumb",
233 doc=
"Output flat-corrected thumbnail image.",
234 storageClass=
"Thumbnail",
235 dimensions=[
"instrument",
"exposure",
"detector"],
241 if config.doBias
is not True:
242 self.prerequisiteInputs.discard(
"bias")
243 if config.doLinearize
is not True:
244 self.prerequisiteInputs.discard(
"linearizer")
245 if config.doCrosstalk
is not True:
246 self.inputs.discard(
"crosstalkSources")
247 self.prerequisiteInputs.discard(
"crosstalk")
248 if config.doBrighterFatter
is not True:
249 self.prerequisiteInputs.discard(
"bfKernel")
250 self.prerequisiteInputs.discard(
"newBFKernel")
251 if config.doDefect
is not True:
252 self.prerequisiteInputs.discard(
"defects")
253 if config.doDark
is not True:
254 self.prerequisiteInputs.discard(
"dark")
255 if config.doFlat
is not True:
256 self.prerequisiteInputs.discard(
"flat")
257 if config.doAttachTransmissionCurve
is not True:
258 self.prerequisiteInputs.discard(
"opticsTransmission")
259 self.prerequisiteInputs.discard(
"filterTransmission")
260 self.prerequisiteInputs.discard(
"sensorTransmission")
261 self.prerequisiteInputs.discard(
"atmosphereTransmission")
262 if config.doUseOpticsTransmission
is not True:
263 self.prerequisiteInputs.discard(
"opticsTransmission")
264 if config.doUseFilterTransmission
is not True:
265 self.prerequisiteInputs.discard(
"filterTransmission")
266 if config.doUseSensorTransmission
is not True:
267 self.prerequisiteInputs.discard(
"sensorTransmission")
268 if config.doUseAtmosphereTransmission
is not True:
269 self.prerequisiteInputs.discard(
"atmosphereTransmission")
270 if config.doIlluminationCorrection
is not True:
271 self.prerequisiteInputs.discard(
"illumMaskedImage")
273 if config.doWrite
is not True:
274 self.outputs.discard(
"outputExposure")
275 self.outputs.discard(
"preInterpExposure")
276 self.outputs.discard(
"outputFlattenedThumbnail")
277 self.outputs.discard(
"outputOssThumbnail")
278 if config.doSaveInterpPixels
is not True:
279 self.outputs.discard(
"preInterpExposure")
280 if config.qa.doThumbnailOss
is not True:
281 self.outputs.discard(
"outputOssThumbnail")
282 if config.qa.doThumbnailFlattened
is not True:
283 self.outputs.discard(
"outputFlattenedThumbnail")
287 pipelineConnections=IsrTaskConnections):
288 """Configuration parameters for IsrTask.
290 Items are grouped in the order in which they are executed by the task.
292 datasetType = pexConfig.Field(
294 doc=
"Dataset type for input data; users will typically leave this alone, "
295 "but camera-specific ISR tasks will override it",
299 fallbackFilterName = pexConfig.Field(
301 doc=
"Fallback default filter name for calibrations.",
304 useFallbackDate = pexConfig.Field(
306 doc=
"Pass observation date when using fallback filter.",
309 expectWcs = pexConfig.Field(
312 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
314 fwhm = pexConfig.Field(
316 doc=
"FWHM of PSF in arcseconds.",
319 qa = pexConfig.ConfigField(
321 doc=
"QA related configuration options.",
325 doConvertIntToFloat = pexConfig.Field(
327 doc=
"Convert integer raw images to floating point values?",
332 doSaturation = pexConfig.Field(
334 doc=
"Mask saturated pixels? NB: this is totally independent of the"
335 " interpolation option - this is ONLY setting the bits in the mask."
336 " To have them interpolated make sure doSaturationInterpolation=True",
339 saturatedMaskName = pexConfig.Field(
341 doc=
"Name of mask plane to use in saturation detection and interpolation",
344 saturation = pexConfig.Field(
346 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
347 default=float(
"NaN"),
349 growSaturationFootprintSize = pexConfig.Field(
351 doc=
"Number of pixels by which to grow the saturation footprints",
356 doSuspect = pexConfig.Field(
358 doc=
"Mask suspect pixels?",
361 suspectMaskName = pexConfig.Field(
363 doc=
"Name of mask plane to use for suspect pixels",
366 numEdgeSuspect = pexConfig.Field(
368 doc=
"Number of edge pixels to be flagged as untrustworthy.",
373 doSetBadRegions = pexConfig.Field(
375 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
378 badStatistic = pexConfig.ChoiceField(
380 doc=
"How to estimate the average value for BAD regions.",
383 "MEANCLIP":
"Correct using the (clipped) mean of good data",
384 "MEDIAN":
"Correct using the median of the good data",
389 doOverscan = pexConfig.Field(
391 doc=
"Do overscan subtraction?",
394 overscan = pexConfig.ConfigurableField(
395 target=OverscanCorrectionTask,
396 doc=
"Overscan subtraction task for image segments.",
399 overscanFitType = pexConfig.ChoiceField(
401 doc=
"The method for fitting the overscan bias level.",
404 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
405 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
406 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
407 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
408 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
409 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
410 "MEAN":
"Correct using the mean of the overscan region",
411 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
412 "MEDIAN":
"Correct using the median of the overscan region",
413 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
415 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
416 " This option will no longer be used, and will be removed after v20.")
418 overscanOrder = pexConfig.Field(
420 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
421 "or number of spline knots if overscan fit type is a spline."),
423 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
424 " This option will no longer be used, and will be removed after v20.")
426 overscanNumSigmaClip = pexConfig.Field(
428 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
430 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
431 " This option will no longer be used, and will be removed after v20.")
433 overscanIsInt = pexConfig.Field(
435 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
436 " and overscan.FitType=MEDIAN_PER_ROW.",
438 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
439 " This option will no longer be used, and will be removed after v20.")
442 overscanNumLeadingColumnsToSkip = pexConfig.Field(
444 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
447 overscanNumTrailingColumnsToSkip = pexConfig.Field(
449 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
452 overscanMaxDev = pexConfig.Field(
454 doc=
"Maximum deviation from the median for overscan",
455 default=1000.0, check=
lambda x: x > 0
457 overscanBiasJump = pexConfig.Field(
459 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
462 overscanBiasJumpKeyword = pexConfig.Field(
464 doc=
"Header keyword containing information about devices.",
465 default=
"NO_SUCH_KEY",
467 overscanBiasJumpDevices = pexConfig.ListField(
469 doc=
"List of devices that need piecewise overscan correction.",
472 overscanBiasJumpLocation = pexConfig.Field(
474 doc=
"Location of bias jump along y-axis.",
479 doAssembleCcd = pexConfig.Field(
482 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
484 assembleCcd = pexConfig.ConfigurableField(
485 target=AssembleCcdTask,
486 doc=
"CCD assembly task",
490 doAssembleIsrExposures = pexConfig.Field(
493 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
495 doTrimToMatchCalib = pexConfig.Field(
498 doc=
"Trim raw data to match calibration bounding boxes?"
502 doBias = pexConfig.Field(
504 doc=
"Apply bias frame correction?",
507 biasDataProductName = pexConfig.Field(
509 doc=
"Name of the bias data product",
514 doVariance = pexConfig.Field(
516 doc=
"Calculate variance?",
519 gain = pexConfig.Field(
521 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
522 default=float(
"NaN"),
524 readNoise = pexConfig.Field(
526 doc=
"The read noise to use if no Detector is present in the Exposure",
529 doEmpiricalReadNoise = pexConfig.Field(
532 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
536 doLinearize = pexConfig.Field(
538 doc=
"Correct for nonlinearity of the detector's response?",
543 doCrosstalk = pexConfig.Field(
545 doc=
"Apply intra-CCD crosstalk correction?",
548 doCrosstalkBeforeAssemble = pexConfig.Field(
550 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
553 crosstalk = pexConfig.ConfigurableField(
554 target=CrosstalkTask,
555 doc=
"Intra-CCD crosstalk correction",
559 doDefect = pexConfig.Field(
561 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
564 doNanMasking = pexConfig.Field(
566 doc=
"Mask NAN pixels?",
569 doWidenSaturationTrails = pexConfig.Field(
571 doc=
"Widen bleed trails based on their width?",
576 doBrighterFatter = pexConfig.Field(
579 doc=
"Apply the brighter fatter correction"
581 brighterFatterLevel = pexConfig.ChoiceField(
584 doc=
"The level at which to correct for brighter-fatter.",
586 "AMP":
"Every amplifier treated separately.",
587 "DETECTOR":
"One kernel per detector",
590 brighterFatterMaxIter = pexConfig.Field(
593 doc=
"Maximum number of iterations for the brighter fatter correction"
595 brighterFatterThreshold = pexConfig.Field(
598 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
599 " absolute value of the difference between the current corrected image and the one"
600 " from the previous iteration summed over all the pixels."
602 brighterFatterApplyGain = pexConfig.Field(
605 doc=
"Should the gain be applied when applying the brighter fatter correction?"
607 brighterFatterMaskGrowSize = pexConfig.Field(
610 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
611 " when brighter-fatter correction is applied."
615 doDark = pexConfig.Field(
617 doc=
"Apply dark frame correction?",
620 darkDataProductName = pexConfig.Field(
622 doc=
"Name of the dark data product",
627 doStrayLight = pexConfig.Field(
629 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
632 strayLight = pexConfig.ConfigurableField(
633 target=StrayLightTask,
634 doc=
"y-band stray light correction"
638 doFlat = pexConfig.Field(
640 doc=
"Apply flat field correction?",
643 flatDataProductName = pexConfig.Field(
645 doc=
"Name of the flat data product",
648 flatScalingType = pexConfig.ChoiceField(
650 doc=
"The method for scaling the flat on the fly.",
653 "USER":
"Scale by flatUserScale",
654 "MEAN":
"Scale by the inverse of the mean",
655 "MEDIAN":
"Scale by the inverse of the median",
658 flatUserScale = pexConfig.Field(
660 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
663 doTweakFlat = pexConfig.Field(
665 doc=
"Tweak flats to match observed amplifier ratios?",
670 doApplyGains = pexConfig.Field(
672 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
675 normalizeGains = pexConfig.Field(
677 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
682 doFringe = pexConfig.Field(
684 doc=
"Apply fringe correction?",
687 fringe = pexConfig.ConfigurableField(
689 doc=
"Fringe subtraction task",
691 fringeAfterFlat = pexConfig.Field(
693 doc=
"Do fringe subtraction after flat-fielding?",
698 doMeasureBackground = pexConfig.Field(
700 doc=
"Measure the background level on the reduced image?",
705 doCameraSpecificMasking = pexConfig.Field(
707 doc=
"Mask camera-specific bad regions?",
710 masking = pexConfig.ConfigurableField(
717 doInterpolate = pexConfig.Field(
719 doc=
"Interpolate masked pixels?",
722 doSaturationInterpolation = pexConfig.Field(
724 doc=
"Perform interpolation over pixels masked as saturated?"
725 " NB: This is independent of doSaturation; if that is False this plane"
726 " will likely be blank, resulting in a no-op here.",
729 doNanInterpolation = pexConfig.Field(
731 doc=
"Perform interpolation over pixels masked as NaN?"
732 " NB: This is independent of doNanMasking; if that is False this plane"
733 " will likely be blank, resulting in a no-op here.",
736 doNanInterpAfterFlat = pexConfig.Field(
738 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
739 "also have to interpolate them before flat-fielding."),
742 maskListToInterpolate = pexConfig.ListField(
744 doc=
"List of mask planes that should be interpolated.",
745 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
747 doSaveInterpPixels = pexConfig.Field(
749 doc=
"Save a copy of the pre-interpolated pixel values?",
754 fluxMag0T1 = pexConfig.DictField(
757 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
758 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
761 defaultFluxMag0T1 = pexConfig.Field(
763 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
764 default=pow(10.0, 0.4*28.0)
768 doVignette = pexConfig.Field(
770 doc=
"Apply vignetting parameters?",
773 vignette = pexConfig.ConfigurableField(
775 doc=
"Vignetting task.",
779 doAttachTransmissionCurve = pexConfig.Field(
782 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
784 doUseOpticsTransmission = pexConfig.Field(
787 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
789 doUseFilterTransmission = pexConfig.Field(
792 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
794 doUseSensorTransmission = pexConfig.Field(
797 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
799 doUseAtmosphereTransmission = pexConfig.Field(
802 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
806 doIlluminationCorrection = pexConfig.Field(
809 doc=
"Perform illumination correction?"
811 illuminationCorrectionDataProductName = pexConfig.Field(
813 doc=
"Name of the illumination correction data product.",
816 illumScale = pexConfig.Field(
818 doc=
"Scale factor for the illumination correction.",
821 illumFilters = pexConfig.ListField(
824 doc=
"Only perform illumination correction for these filters."
828 doWrite = pexConfig.Field(
830 doc=
"Persist postISRCCD?",
837 raise ValueError(
"You may not specify both doFlat and doApplyGains")
839 self.config.maskListToInterpolate.append(
"SAT")
841 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
844 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
845 """Apply common instrument signature correction algorithms to a raw frame.
847 The process for correcting imaging data is very similar from
848 camera to camera. This task provides a vanilla implementation of
849 doing these corrections, including the ability to turn certain
850 corrections off if they are not needed. The inputs to the primary
851 method, `run()`, are a raw exposure to be corrected and the
852 calibration data products. The raw input is a single chip sized
853 mosaic of all amps including overscans and other non-science
854 pixels. The method `runDataRef()` identifies and defines the
855 calibration data products, and is intended for use by a
856 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
857 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
858 subclassed for different camera, although the most camera specific
859 methods have been split into subtasks that can be redirected
862 The __init__ method sets up the subtasks for ISR processing, using
863 the defaults from `lsst.ip.isr`.
868 Positional arguments passed to the Task constructor. None used at this time.
869 kwargs : `dict`, optional
870 Keyword arguments passed on to the Task constructor. None used at this time.
872 ConfigClass = IsrTaskConfig
877 self.makeSubtask(
"assembleCcd")
878 self.makeSubtask(
"crosstalk")
879 self.makeSubtask(
"strayLight")
880 self.makeSubtask(
"fringe")
881 self.makeSubtask(
"masking")
882 self.makeSubtask(
"overscan")
883 self.makeSubtask(
"vignette")
886 inputs = butlerQC.get(inputRefs)
889 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
890 except Exception
as e:
891 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
894 inputs[
'isGen3'] =
True
896 detector = inputs[
'ccdExposure'].getDetector()
898 if self.config.doCrosstalk
is True:
901 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
902 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
903 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
905 coeffVector = (self.config.crosstalk.crosstalkValues
906 if self.config.crosstalk.useConfigCoefficients
else None)
907 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
908 inputs[
'crosstalk'] = crosstalkCalib
909 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
910 if 'crosstalkSources' not in inputs:
911 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
914 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
916 linearizer.fromYaml(inputs[
'linearizer'])
920 inputs[
'linearizer'] = linearizer
922 if self.config.doDefect
is True:
923 if "defects" in inputs
and inputs[
'defects']
is not None:
926 if not isinstance(inputs[
"defects"], Defects):
927 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
931 if self.config.doBrighterFatter:
932 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
933 if brighterFatterKernel
is None:
934 brighterFatterKernel = inputs.get(
'bfKernel',
None)
936 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
937 detId = detector.getId()
938 inputs[
'bfGains'] = brighterFatterKernel.gain
941 if self.config.brighterFatterLevel ==
'DETECTOR':
942 if brighterFatterKernel.detectorKernel:
943 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
944 elif brighterFatterKernel.detectorKernelFromAmpKernels:
945 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
947 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
950 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
952 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
953 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
954 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
956 assembler=self.assembleCcd
957 if self.config.doAssembleIsrExposures
else None)
959 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
961 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
962 if 'strayLightData' not in inputs:
963 inputs[
'strayLightData'] =
None
965 outputs = self.
run(**inputs)
966 butlerQC.put(outputs, outputRefs)
969 """!Retrieve necessary frames for instrument signature removal.
971 Pre-fetching all required ISR data products limits the IO
972 required by the ISR. Any conflict between the calibration data
973 available and that needed for ISR is also detected prior to
974 doing processing, allowing it to fail quickly.
978 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
979 Butler reference of the detector data to be processed
980 rawExposure : `afw.image.Exposure`
981 The raw exposure that will later be corrected with the
982 retrieved calibration data; should not be modified in this
987 result : `lsst.pipe.base.Struct`
988 Result struct with components (which may be `None`):
989 - ``bias``: bias calibration frame (`afw.image.Exposure`)
990 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
991 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
992 - ``dark``: dark calibration frame (`afw.image.Exposure`)
993 - ``flat``: flat calibration frame (`afw.image.Exposure`)
994 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
995 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
996 - ``fringes``: `lsst.pipe.base.Struct` with components:
997 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
998 - ``seed``: random seed derived from the ccdExposureId for random
999 number generator (`uint32`).
1000 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1001 A ``TransmissionCurve`` that represents the throughput of the optics,
1002 to be evaluated in focal-plane coordinates.
1003 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1004 A ``TransmissionCurve`` that represents the throughput of the filter
1005 itself, to be evaluated in focal-plane coordinates.
1006 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1007 A ``TransmissionCurve`` that represents the throughput of the sensor
1008 itself, to be evaluated in post-assembly trimmed detector coordinates.
1009 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1010 A ``TransmissionCurve`` that represents the throughput of the
1011 atmosphere, assumed to be spatially constant.
1012 - ``strayLightData`` : `object`
1013 An opaque object containing calibration information for
1014 stray-light correction. If `None`, no correction will be
1016 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1020 NotImplementedError :
1021 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1024 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1025 dateObs = dateObs.toPython().isoformat()
1026 except RuntimeError:
1027 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1030 ccd = rawExposure.getDetector()
1031 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
1032 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1033 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
1034 if self.config.doBias
else None)
1036 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1038 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1039 linearizer.log = self.log
1040 if isinstance(linearizer, numpy.ndarray):
1043 crosstalkCalib =
None
1044 if self.config.doCrosstalk:
1046 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1048 coeffVector = (self.config.crosstalk.crosstalkValues
1049 if self.config.crosstalk.useConfigCoefficients
else None)
1050 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1051 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1052 if self.config.doCrosstalk
else None)
1054 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
1055 if self.config.doDark
else None)
1056 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
1058 if self.config.doFlat
else None)
1060 brighterFatterKernel =
None
1061 brighterFatterGains =
None
1062 if self.config.doBrighterFatter
is True:
1067 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1068 brighterFatterGains = brighterFatterKernel.gain
1069 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1072 brighterFatterKernel = dataRef.get(
"bfKernel")
1073 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
1075 brighterFatterKernel =
None
1076 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1079 if self.config.brighterFatterLevel ==
'DETECTOR':
1080 if brighterFatterKernel.detectorKernel:
1081 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1082 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1083 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1085 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1088 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1090 defectList = (dataRef.get(
"defects")
1091 if self.config.doDefect
else None)
1092 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1093 if self.config.doAssembleIsrExposures
else None)
1094 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1095 else pipeBase.Struct(fringes=
None))
1097 if self.config.doAttachTransmissionCurve:
1098 opticsTransmission = (dataRef.get(
"transmission_optics")
1099 if self.config.doUseOpticsTransmission
else None)
1100 filterTransmission = (dataRef.get(
"transmission_filter")
1101 if self.config.doUseFilterTransmission
else None)
1102 sensorTransmission = (dataRef.get(
"transmission_sensor")
1103 if self.config.doUseSensorTransmission
else None)
1104 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1105 if self.config.doUseAtmosphereTransmission
else None)
1107 opticsTransmission =
None
1108 filterTransmission =
None
1109 sensorTransmission =
None
1110 atmosphereTransmission =
None
1112 if self.config.doStrayLight:
1113 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1115 strayLightData =
None
1118 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1119 if (self.config.doIlluminationCorrection
1120 and filterName
in self.config.illumFilters)
1124 return pipeBase.Struct(bias=biasExposure,
1125 linearizer=linearizer,
1126 crosstalk=crosstalkCalib,
1127 crosstalkSources=crosstalkSources,
1130 bfKernel=brighterFatterKernel,
1131 bfGains=brighterFatterGains,
1133 fringes=fringeStruct,
1134 opticsTransmission=opticsTransmission,
1135 filterTransmission=filterTransmission,
1136 sensorTransmission=sensorTransmission,
1137 atmosphereTransmission=atmosphereTransmission,
1138 strayLightData=strayLightData,
1139 illumMaskedImage=illumMaskedImage
1142 @pipeBase.timeMethod
1143 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1144 crosstalk=None, crosstalkSources=None,
1145 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1146 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1147 sensorTransmission=
None, atmosphereTransmission=
None,
1148 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1151 """!Perform instrument signature removal on an exposure.
1153 Steps included in the ISR processing, in order performed, are:
1154 - saturation and suspect pixel masking
1155 - overscan subtraction
1156 - CCD assembly of individual amplifiers
1158 - variance image construction
1159 - linearization of non-linear response
1161 - brighter-fatter correction
1164 - stray light subtraction
1166 - masking of known defects and camera specific features
1167 - vignette calculation
1168 - appending transmission curve and distortion model
1172 ccdExposure : `lsst.afw.image.Exposure`
1173 The raw exposure that is to be run through ISR. The
1174 exposure is modified by this method.
1175 camera : `lsst.afw.cameraGeom.Camera`, optional
1176 The camera geometry for this exposure. Required if ``isGen3`` is
1177 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1178 ``flat`` does not have an associated detector.
1179 bias : `lsst.afw.image.Exposure`, optional
1180 Bias calibration frame.
1181 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1182 Functor for linearization.
1183 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1184 Calibration for crosstalk.
1185 crosstalkSources : `list`, optional
1186 List of possible crosstalk sources.
1187 dark : `lsst.afw.image.Exposure`, optional
1188 Dark calibration frame.
1189 flat : `lsst.afw.image.Exposure`, optional
1190 Flat calibration frame.
1191 bfKernel : `numpy.ndarray`, optional
1192 Brighter-fatter kernel.
1193 bfGains : `dict` of `float`, optional
1194 Gains used to override the detector's nominal gains for the
1195 brighter-fatter correction. A dict keyed by amplifier name for
1196 the detector in question.
1197 defects : `lsst.meas.algorithms.Defects`, optional
1199 fringes : `lsst.pipe.base.Struct`, optional
1200 Struct containing the fringe correction data, with
1202 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1203 - ``seed``: random seed derived from the ccdExposureId for random
1204 number generator (`uint32`)
1205 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1206 A ``TransmissionCurve`` that represents the throughput of the optics,
1207 to be evaluated in focal-plane coordinates.
1208 filterTransmission : `lsst.afw.image.TransmissionCurve`
1209 A ``TransmissionCurve`` that represents the throughput of the filter
1210 itself, to be evaluated in focal-plane coordinates.
1211 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1212 A ``TransmissionCurve`` that represents the throughput of the sensor
1213 itself, to be evaluated in post-assembly trimmed detector coordinates.
1214 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1215 A ``TransmissionCurve`` that represents the throughput of the
1216 atmosphere, assumed to be spatially constant.
1217 detectorNum : `int`, optional
1218 The integer number for the detector to process.
1219 isGen3 : bool, optional
1220 Flag this call to run() as using the Gen3 butler environment.
1221 strayLightData : `object`, optional
1222 Opaque object containing calibration information for stray-light
1223 correction. If `None`, no correction will be performed.
1224 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1225 Illumination correction image.
1229 result : `lsst.pipe.base.Struct`
1230 Result struct with component:
1231 - ``exposure`` : `afw.image.Exposure`
1232 The fully ISR corrected exposure.
1233 - ``outputExposure`` : `afw.image.Exposure`
1234 An alias for `exposure`
1235 - ``ossThumb`` : `numpy.ndarray`
1236 Thumbnail image of the exposure after overscan subtraction.
1237 - ``flattenedThumb`` : `numpy.ndarray`
1238 Thumbnail image of the exposure after flat-field correction.
1243 Raised if a configuration option is set to True, but the
1244 required calibration data has not been specified.
1248 The current processed exposure can be viewed by setting the
1249 appropriate lsstDebug entries in the `debug.display`
1250 dictionary. The names of these entries correspond to some of
1251 the IsrTaskConfig Boolean options, with the value denoting the
1252 frame to use. The exposure is shown inside the matching
1253 option check and after the processing of that step has
1254 finished. The steps with debug points are:
1265 In addition, setting the "postISRCCD" entry displays the
1266 exposure after all ISR processing has finished.
1274 if detectorNum
is None:
1275 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1277 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1282 if isinstance(ccdExposure, ButlerDataRef):
1285 ccd = ccdExposure.getDetector()
1286 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1289 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1290 ccd = [
FakeAmp(ccdExposure, self.config)]
1293 if self.config.doBias
and bias
is None:
1294 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1296 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1297 if self.config.doBrighterFatter
and bfKernel
is None:
1298 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1299 if self.config.doDark
and dark
is None:
1300 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1301 if self.config.doFlat
and flat
is None:
1302 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1303 if self.config.doDefect
and defects
is None:
1304 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1305 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1306 and fringes.fringes
is None):
1311 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1312 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1313 and illumMaskedImage
is None):
1314 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1317 if self.config.doConvertIntToFloat:
1318 self.log.info(
"Converting exposure to floating point values.")
1325 if ccdExposure.getBBox().contains(amp.getBBox()):
1329 if self.config.doOverscan
and not badAmp:
1332 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1333 if overscanResults
is not None and \
1334 self.config.qa
is not None and self.config.qa.saveStats
is True:
1335 if isinstance(overscanResults.overscanFit, float):
1336 qaMedian = overscanResults.overscanFit
1337 qaStdev = float(
"NaN")
1339 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1340 afwMath.MEDIAN | afwMath.STDEVCLIP)
1341 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1342 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1344 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1345 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1346 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1347 amp.getName(), qaMedian, qaStdev)
1348 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1351 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1352 overscanResults =
None
1354 overscans.append(overscanResults
if overscanResults
is not None else None)
1356 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1358 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1359 self.log.info(
"Applying crosstalk correction.")
1360 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1361 crosstalkSources=crosstalkSources)
1362 self.
debugView(ccdExposure,
"doCrosstalk")
1364 if self.config.doAssembleCcd:
1365 self.log.info(
"Assembling CCD from amplifiers.")
1366 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1368 if self.config.expectWcs
and not ccdExposure.getWcs():
1369 self.log.warn(
"No WCS found in input exposure.")
1370 self.
debugView(ccdExposure,
"doAssembleCcd")
1373 if self.config.qa.doThumbnailOss:
1374 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1376 if self.config.doBias:
1377 self.log.info(
"Applying bias correction.")
1378 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1379 trimToFit=self.config.doTrimToMatchCalib)
1382 if self.config.doVariance:
1383 for amp, overscanResults
in zip(ccd, overscans):
1384 if ccdExposure.getBBox().contains(amp.getBBox()):
1385 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1386 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1387 if overscanResults
is not None:
1389 overscanImage=overscanResults.overscanImage)
1393 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1394 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1395 afwMath.MEDIAN | afwMath.STDEVCLIP)
1396 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1397 qaStats.getValue(afwMath.MEDIAN))
1398 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1399 qaStats.getValue(afwMath.STDEVCLIP))
1400 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1401 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1402 qaStats.getValue(afwMath.STDEVCLIP))
1405 self.log.info(
"Applying linearizer.")
1406 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1407 detector=ccd, log=self.log)
1409 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1410 self.log.info(
"Applying crosstalk correction.")
1411 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1412 crosstalkSources=crosstalkSources, isTrimmed=
True)
1413 self.
debugView(ccdExposure,
"doCrosstalk")
1417 if self.config.doDefect:
1418 self.log.info(
"Masking defects.")
1421 if self.config.numEdgeSuspect > 0:
1422 self.log.info(
"Masking edges as SUSPECT.")
1423 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1424 maskPlane=
"SUSPECT")
1426 if self.config.doNanMasking:
1427 self.log.info(
"Masking NAN value pixels.")
1430 if self.config.doWidenSaturationTrails:
1431 self.log.info(
"Widening saturation trails.")
1432 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1434 if self.config.doCameraSpecificMasking:
1435 self.log.info(
"Masking regions for camera specific reasons.")
1436 self.masking.
run(ccdExposure)
1438 if self.config.doBrighterFatter:
1447 interpExp = ccdExposure.clone()
1449 isrFunctions.interpolateFromMask(
1450 maskedImage=interpExp.getMaskedImage(),
1451 fwhm=self.config.fwhm,
1452 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1453 maskNameList=self.config.maskListToInterpolate
1455 bfExp = interpExp.clone()
1457 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1458 type(bfKernel), type(bfGains))
1459 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1460 self.config.brighterFatterMaxIter,
1461 self.config.brighterFatterThreshold,
1462 self.config.brighterFatterApplyGain,
1464 if bfResults[1] == self.config.brighterFatterMaxIter:
1465 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1468 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1470 image = ccdExposure.getMaskedImage().getImage()
1471 bfCorr = bfExp.getMaskedImage().getImage()
1472 bfCorr -= interpExp.getMaskedImage().getImage()
1481 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1482 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1485 if self.config.brighterFatterMaskGrowSize > 0:
1486 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1487 for maskPlane
in self.config.maskListToInterpolate:
1488 isrFunctions.growMasks(ccdExposure.getMask(),
1489 radius=self.config.brighterFatterMaskGrowSize,
1490 maskNameList=maskPlane,
1491 maskValue=maskPlane)
1493 self.
debugView(ccdExposure,
"doBrighterFatter")
1495 if self.config.doDark:
1496 self.log.info(
"Applying dark correction.")
1500 if self.config.doFringe
and not self.config.fringeAfterFlat:
1501 self.log.info(
"Applying fringe correction before flat.")
1502 self.fringe.
run(ccdExposure, **fringes.getDict())
1505 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1506 self.log.info(
"Checking strayLight correction.")
1507 self.strayLight.
run(ccdExposure, strayLightData)
1508 self.
debugView(ccdExposure,
"doStrayLight")
1510 if self.config.doFlat:
1511 self.log.info(
"Applying flat correction.")
1515 if self.config.doApplyGains:
1516 self.log.info(
"Applying gain correction instead of flat.")
1517 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1519 if self.config.doFringe
and self.config.fringeAfterFlat:
1520 self.log.info(
"Applying fringe correction after flat.")
1521 self.fringe.
run(ccdExposure, **fringes.getDict())
1523 if self.config.doVignette:
1524 self.log.info(
"Constructing Vignette polygon.")
1527 if self.config.vignette.doWriteVignettePolygon:
1530 if self.config.doAttachTransmissionCurve:
1531 self.log.info(
"Adding transmission curves.")
1532 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1533 filterTransmission=filterTransmission,
1534 sensorTransmission=sensorTransmission,
1535 atmosphereTransmission=atmosphereTransmission)
1537 flattenedThumb =
None
1538 if self.config.qa.doThumbnailFlattened:
1539 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1541 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1542 self.log.info(
"Performing illumination correction.")
1543 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1544 illumMaskedImage, illumScale=self.config.illumScale,
1545 trimToFit=self.config.doTrimToMatchCalib)
1548 if self.config.doSaveInterpPixels:
1549 preInterpExp = ccdExposure.clone()
1564 if self.config.doSetBadRegions:
1565 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1566 if badPixelCount > 0:
1567 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1569 if self.config.doInterpolate:
1570 self.log.info(
"Interpolating masked pixels.")
1571 isrFunctions.interpolateFromMask(
1572 maskedImage=ccdExposure.getMaskedImage(),
1573 fwhm=self.config.fwhm,
1574 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1575 maskNameList=list(self.config.maskListToInterpolate)
1580 if self.config.doMeasureBackground:
1581 self.log.info(
"Measuring background level.")
1584 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1586 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1587 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1588 afwMath.MEDIAN | afwMath.STDEVCLIP)
1589 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1590 qaStats.getValue(afwMath.MEDIAN))
1591 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1592 qaStats.getValue(afwMath.STDEVCLIP))
1593 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1594 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1595 qaStats.getValue(afwMath.STDEVCLIP))
1597 self.
debugView(ccdExposure,
"postISRCCD")
1599 return pipeBase.Struct(
1600 exposure=ccdExposure,
1602 flattenedThumb=flattenedThumb,
1604 preInterpolatedExposure=preInterpExp,
1605 outputExposure=ccdExposure,
1606 outputOssThumbnail=ossThumb,
1607 outputFlattenedThumbnail=flattenedThumb,
1610 @pipeBase.timeMethod
1612 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1614 This method contains the `CmdLineTask` interface to the ISR
1615 processing. All IO is handled here, freeing the `run()` method
1616 to manage only pixel-level calculations. The steps performed
1618 - Read in necessary detrending/isr/calibration data.
1619 - Process raw exposure in `run()`.
1620 - Persist the ISR-corrected exposure as "postISRCCD" if
1621 config.doWrite=True.
1625 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1626 DataRef of the detector data to be processed
1630 result : `lsst.pipe.base.Struct`
1631 Result struct with component:
1632 - ``exposure`` : `afw.image.Exposure`
1633 The fully ISR corrected exposure.
1638 Raised if a configuration option is set to True, but the
1639 required calibration data does not exist.
1642 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1644 ccdExposure = sensorRef.get(self.config.datasetType)
1646 camera = sensorRef.get(
"camera")
1647 isrData = self.
readIsrData(sensorRef, ccdExposure)
1649 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1651 if self.config.doWrite:
1652 sensorRef.put(result.exposure,
"postISRCCD")
1653 if result.preInterpolatedExposure
is not None:
1654 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1655 if result.ossThumb
is not None:
1656 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1657 if result.flattenedThumb
is not None:
1658 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1663 """!Retrieve a calibration dataset for removing instrument signature.
1668 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1669 DataRef of the detector data to find calibration datasets
1672 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1673 dateObs : `str`, optional
1674 Date of the observation. Used to correct butler failures
1675 when using fallback filters.
1677 If True, disable butler proxies to enable error handling
1678 within this routine.
1682 exposure : `lsst.afw.image.Exposure`
1683 Requested calibration frame.
1688 Raised if no matching calibration frame can be found.
1691 exp = dataRef.get(datasetType, immediate=immediate)
1692 except Exception
as exc1:
1693 if not self.config.fallbackFilterName:
1694 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1696 if self.config.useFallbackDate
and dateObs:
1697 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1698 dateObs=dateObs, immediate=immediate)
1700 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1701 except Exception
as exc2:
1702 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1703 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1704 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1706 if self.config.doAssembleIsrExposures:
1707 exp = self.assembleCcd.assembleCcd(exp)
1711 """Ensure that the data returned by Butler is a fully constructed exposure.
1713 ISR requires exposure-level image data for historical reasons, so if we did
1714 not recieve that from Butler, construct it from what we have, modifying the
1719 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1720 `lsst.afw.image.ImageF`
1721 The input data structure obtained from Butler.
1722 camera : `lsst.afw.cameraGeom.camera`
1723 The camera associated with the image. Used to find the appropriate
1726 The detector this exposure should match.
1730 inputExp : `lsst.afw.image.Exposure`
1731 The re-constructed exposure, with appropriate detector parameters.
1736 Raised if the input data cannot be used to construct an exposure.
1738 if isinstance(inputExp, afwImage.DecoratedImageU):
1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1740 elif isinstance(inputExp, afwImage.ImageF):
1741 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1742 elif isinstance(inputExp, afwImage.MaskedImageF):
1743 inputExp = afwImage.makeExposure(inputExp)
1744 elif isinstance(inputExp, afwImage.Exposure):
1746 elif inputExp
is None:
1750 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1753 if inputExp.getDetector()
is None:
1754 inputExp.setDetector(camera[detectorNum])
1759 """Convert exposure image from uint16 to float.
1761 If the exposure does not need to be converted, the input is
1762 immediately returned. For exposures that are converted to use
1763 floating point pixels, the variance is set to unity and the
1768 exposure : `lsst.afw.image.Exposure`
1769 The raw exposure to be converted.
1773 newexposure : `lsst.afw.image.Exposure`
1774 The input ``exposure``, converted to floating point pixels.
1779 Raised if the exposure type cannot be converted to float.
1782 if isinstance(exposure, afwImage.ExposureF):
1784 self.log.debug(
"Exposure already of type float.")
1786 if not hasattr(exposure,
"convertF"):
1787 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1789 newexposure = exposure.convertF()
1790 newexposure.variance[:] = 1
1791 newexposure.mask[:] = 0x0
1796 """Identify bad amplifiers, saturated and suspect pixels.
1800 ccdExposure : `lsst.afw.image.Exposure`
1801 Input exposure to be masked.
1802 amp : `lsst.afw.table.AmpInfoCatalog`
1803 Catalog of parameters defining the amplifier on this
1805 defects : `lsst.meas.algorithms.Defects`
1806 List of defects. Used to determine if the entire
1812 If this is true, the entire amplifier area is covered by
1813 defects and unusable.
1816 maskedImage = ccdExposure.getMaskedImage()
1822 if defects
is not None:
1823 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1828 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1830 maskView = dataView.getMask()
1831 maskView |= maskView.getPlaneBitMask(
"BAD")
1838 if self.config.doSaturation
and not badAmp:
1839 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1840 if self.config.doSuspect
and not badAmp:
1841 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1842 if math.isfinite(self.config.saturation):
1843 limits.update({self.config.saturatedMaskName: self.config.saturation})
1845 for maskName, maskThreshold
in limits.items():
1846 if not math.isnan(maskThreshold):
1847 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1848 isrFunctions.makeThresholdMask(
1849 maskedImage=dataView,
1850 threshold=maskThreshold,
1856 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1858 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1859 self.config.suspectMaskName])
1860 if numpy.all(maskView.getArray() & maskVal > 0):
1862 maskView |= maskView.getPlaneBitMask(
"BAD")
1867 """Apply overscan correction in place.
1869 This method does initial pixel rejection of the overscan
1870 region. The overscan can also be optionally segmented to
1871 allow for discontinuous overscan responses to be fit
1872 separately. The actual overscan subtraction is performed by
1873 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1874 which is called here after the amplifier is preprocessed.
1878 ccdExposure : `lsst.afw.image.Exposure`
1879 Exposure to have overscan correction performed.
1880 amp : `lsst.afw.table.AmpInfoCatalog`
1881 The amplifier to consider while correcting the overscan.
1885 overscanResults : `lsst.pipe.base.Struct`
1886 Result struct with components:
1887 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1888 Value or fit subtracted from the amplifier image data.
1889 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1890 Value or fit subtracted from the overscan image data.
1891 - ``overscanImage`` : `lsst.afw.image.Image`
1892 Image of the overscan region with the overscan
1893 correction applied. This quantity is used to estimate
1894 the amplifier read noise empirically.
1899 Raised if the ``amp`` does not contain raw pixel information.
1903 lsst.ip.isr.isrFunctions.overscanCorrection
1905 if amp.getRawHorizontalOverscanBBox().isEmpty():
1906 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1909 statControl = afwMath.StatisticsControl()
1910 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1913 dataBBox = amp.getRawDataBBox()
1914 oscanBBox = amp.getRawHorizontalOverscanBBox()
1918 prescanBBox = amp.getRawPrescanBBox()
1919 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1920 dx0 += self.config.overscanNumLeadingColumnsToSkip
1921 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1923 dx0 += self.config.overscanNumTrailingColumnsToSkip
1924 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1930 if ((self.config.overscanBiasJump
1931 and self.config.overscanBiasJumpLocation)
1932 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1933 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
1934 self.config.overscanBiasJumpDevices)):
1935 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1936 yLower = self.config.overscanBiasJumpLocation
1937 yUpper = dataBBox.getHeight() - yLower
1939 yUpper = self.config.overscanBiasJumpLocation
1940 yLower = dataBBox.getHeight() - yUpper
1958 oscanBBox.getHeight())))
1961 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1962 ampImage = ccdExposure.maskedImage[imageBBox]
1963 overscanImage = ccdExposure.maskedImage[overscanBBox]
1965 overscanArray = overscanImage.image.array
1966 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1967 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1968 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1970 statControl = afwMath.StatisticsControl()
1971 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1973 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage)
1976 levelStat = afwMath.MEDIAN
1977 sigmaStat = afwMath.STDEVCLIP
1979 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1980 self.config.qa.flatness.nIter)
1981 metadata = ccdExposure.getMetadata()
1982 ampNum = amp.getName()
1984 if isinstance(overscanResults.overscanFit, float):
1985 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1986 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1988 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1989 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1990 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1992 return overscanResults
1995 """Set the variance plane using the amplifier gain and read noise
1997 The read noise is calculated from the ``overscanImage`` if the
1998 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1999 the value from the amplifier data is used.
2003 ampExposure : `lsst.afw.image.Exposure`
2004 Exposure to process.
2005 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2006 Amplifier detector data.
2007 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2008 Image of overscan, required only for empirical read noise.
2012 lsst.ip.isr.isrFunctions.updateVariance
2014 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2015 gain = amp.getGain()
2017 if math.isnan(gain):
2019 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2022 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2023 amp.getName(), gain, patchedGain)
2026 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2027 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
2029 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2030 stats = afwMath.StatisticsControl()
2031 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2032 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2033 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2034 amp.getName(), readNoise)
2036 readNoise = amp.getReadNoise()
2038 isrFunctions.updateVariance(
2039 maskedImage=ampExposure.getMaskedImage(),
2041 readNoise=readNoise,
2045 """!Apply dark correction in place.
2049 exposure : `lsst.afw.image.Exposure`
2050 Exposure to process.
2051 darkExposure : `lsst.afw.image.Exposure`
2052 Dark exposure of the same size as ``exposure``.
2053 invert : `Bool`, optional
2054 If True, re-add the dark to an already corrected image.
2059 Raised if either ``exposure`` or ``darkExposure`` do not
2060 have their dark time defined.
2064 lsst.ip.isr.isrFunctions.darkCorrection
2066 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2067 if math.isnan(expScale):
2068 raise RuntimeError(
"Exposure darktime is NAN.")
2069 if darkExposure.getInfo().getVisitInfo()
is not None \
2070 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2071 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2075 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2078 isrFunctions.darkCorrection(
2079 maskedImage=exposure.getMaskedImage(),
2080 darkMaskedImage=darkExposure.getMaskedImage(),
2082 darkScale=darkScale,
2084 trimToFit=self.config.doTrimToMatchCalib
2088 """!Check if linearization is needed for the detector cameraGeom.
2090 Checks config.doLinearize and the linearity type of the first
2095 detector : `lsst.afw.cameraGeom.Detector`
2096 Detector to get linearity type from.
2100 doLinearize : `Bool`
2101 If True, linearization should be performed.
2103 return self.config.doLinearize
and \
2104 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2107 """!Apply flat correction in place.
2111 exposure : `lsst.afw.image.Exposure`
2112 Exposure to process.
2113 flatExposure : `lsst.afw.image.Exposure`
2114 Flat exposure of the same size as ``exposure``.
2115 invert : `Bool`, optional
2116 If True, unflatten an already flattened image.
2120 lsst.ip.isr.isrFunctions.flatCorrection
2122 isrFunctions.flatCorrection(
2123 maskedImage=exposure.getMaskedImage(),
2124 flatMaskedImage=flatExposure.getMaskedImage(),
2125 scalingType=self.config.flatScalingType,
2126 userScale=self.config.flatUserScale,
2128 trimToFit=self.config.doTrimToMatchCalib
2132 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2136 exposure : `lsst.afw.image.Exposure`
2137 Exposure to process. Only the amplifier DataSec is processed.
2138 amp : `lsst.afw.table.AmpInfoCatalog`
2139 Amplifier detector data.
2143 lsst.ip.isr.isrFunctions.makeThresholdMask
2145 if not math.isnan(amp.getSaturation()):
2146 maskedImage = exposure.getMaskedImage()
2147 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2148 isrFunctions.makeThresholdMask(
2149 maskedImage=dataView,
2150 threshold=amp.getSaturation(),
2152 maskName=self.config.saturatedMaskName,
2156 """!Interpolate over saturated pixels, in place.
2158 This method should be called after `saturationDetection`, to
2159 ensure that the saturated pixels have been identified in the
2160 SAT mask. It should also be called after `assembleCcd`, since
2161 saturated regions may cross amplifier boundaries.
2165 exposure : `lsst.afw.image.Exposure`
2166 Exposure to process.
2170 lsst.ip.isr.isrTask.saturationDetection
2171 lsst.ip.isr.isrFunctions.interpolateFromMask
2173 isrFunctions.interpolateFromMask(
2174 maskedImage=exposure.getMaskedImage(),
2175 fwhm=self.config.fwhm,
2176 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2177 maskNameList=list(self.config.saturatedMaskName),
2181 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2185 exposure : `lsst.afw.image.Exposure`
2186 Exposure to process. Only the amplifier DataSec is processed.
2187 amp : `lsst.afw.table.AmpInfoCatalog`
2188 Amplifier detector data.
2192 lsst.ip.isr.isrFunctions.makeThresholdMask
2196 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2197 This is intended to indicate pixels that may be affected by unknown systematics;
2198 for example if non-linearity corrections above a certain level are unstable
2199 then that would be a useful value for suspectLevel. A value of `nan` indicates
2200 that no such level exists and no pixels are to be masked as suspicious.
2202 suspectLevel = amp.getSuspectLevel()
2203 if math.isnan(suspectLevel):
2206 maskedImage = exposure.getMaskedImage()
2207 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2208 isrFunctions.makeThresholdMask(
2209 maskedImage=dataView,
2210 threshold=suspectLevel,
2212 maskName=self.config.suspectMaskName,
2216 """!Mask defects using mask plane "BAD", in place.
2220 exposure : `lsst.afw.image.Exposure`
2221 Exposure to process.
2222 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2223 `lsst.afw.image.DefectBase`.
2224 List of defects to mask.
2228 Call this after CCD assembly, since defects may cross amplifier boundaries.
2230 maskedImage = exposure.getMaskedImage()
2231 if not isinstance(defectBaseList, Defects):
2233 defectList = Defects(defectBaseList)
2235 defectList = defectBaseList
2236 defectList.maskPixels(maskedImage, maskName=
"BAD")
2238 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2239 """!Mask edge pixels with applicable mask plane.
2243 exposure : `lsst.afw.image.Exposure`
2244 Exposure to process.
2245 numEdgePixels : `int`, optional
2246 Number of edge pixels to mask.
2247 maskPlane : `str`, optional
2248 Mask plane name to use.
2250 maskedImage = exposure.getMaskedImage()
2251 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2253 if numEdgePixels > 0:
2254 goodBBox = maskedImage.getBBox()
2256 goodBBox.grow(-numEdgePixels)
2258 SourceDetectionTask.setEdgeBits(
2265 """Mask and interpolate defects using mask plane "BAD", in place.
2269 exposure : `lsst.afw.image.Exposure`
2270 Exposure to process.
2271 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2272 `lsst.afw.image.DefectBase`.
2273 List of defects to mask and interpolate.
2277 lsst.ip.isr.isrTask.maskDefect()
2280 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2281 maskPlane=
"SUSPECT")
2282 isrFunctions.interpolateFromMask(
2283 maskedImage=exposure.getMaskedImage(),
2284 fwhm=self.config.fwhm,
2285 growSaturatedFootprints=0,
2286 maskNameList=[
"BAD"],
2290 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2294 exposure : `lsst.afw.image.Exposure`
2295 Exposure to process.
2299 We mask over all NaNs, including those that are masked with
2300 other bits (because those may or may not be interpolated over
2301 later, and we want to remove all NaNs). Despite this
2302 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2303 the historical name.
2305 maskedImage = exposure.getMaskedImage()
2308 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2309 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2310 numNans =
maskNans(maskedImage, maskVal)
2311 self.metadata.set(
"NUMNANS", numNans)
2313 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2316 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2320 exposure : `lsst.afw.image.Exposure`
2321 Exposure to process.
2325 lsst.ip.isr.isrTask.maskNan()
2328 isrFunctions.interpolateFromMask(
2329 maskedImage=exposure.getMaskedImage(),
2330 fwhm=self.config.fwhm,
2331 growSaturatedFootprints=0,
2332 maskNameList=[
"UNMASKEDNAN"],
2336 """Measure the image background in subgrids, for quality control purposes.
2340 exposure : `lsst.afw.image.Exposure`
2341 Exposure to process.
2342 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2343 Configuration object containing parameters on which background
2344 statistics and subgrids to use.
2346 if IsrQaConfig
is not None:
2347 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2348 IsrQaConfig.flatness.nIter)
2349 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2350 statsControl.setAndMask(maskVal)
2351 maskedImage = exposure.getMaskedImage()
2352 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2353 skyLevel = stats.getValue(afwMath.MEDIAN)
2354 skySigma = stats.getValue(afwMath.STDEVCLIP)
2355 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2356 metadata = exposure.getMetadata()
2357 metadata.set(
'SKYLEVEL', skyLevel)
2358 metadata.set(
'SKYSIGMA', skySigma)
2361 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2362 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2363 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2364 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2365 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2366 skyLevels = numpy.zeros((nX, nY))
2369 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2371 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2373 xLLC = xc - meshXHalf
2374 yLLC = yc - meshYHalf
2375 xURC = xc + meshXHalf - 1
2376 yURC = yc + meshYHalf - 1
2379 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2381 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2383 good = numpy.where(numpy.isfinite(skyLevels))
2384 skyMedian = numpy.median(skyLevels[good])
2385 flatness = (skyLevels[good] - skyMedian) / skyMedian
2386 flatness_rms = numpy.std(flatness)
2387 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2389 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2390 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2391 nX, nY, flatness_pp, flatness_rms)
2393 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2394 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2395 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2396 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2397 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2400 """Set an approximate magnitude zero point for the exposure.
2404 exposure : `lsst.afw.image.Exposure`
2405 Exposure to process.
2407 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2408 if filterName
in self.config.fluxMag0T1:
2409 fluxMag0 = self.config.fluxMag0T1[filterName]
2411 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2412 fluxMag0 = self.config.defaultFluxMag0T1
2414 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2416 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2419 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2420 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2423 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2427 ccdExposure : `lsst.afw.image.Exposure`
2428 Exposure to process.
2429 fpPolygon : `lsst.afw.geom.Polygon`
2430 Polygon in focal plane coordinates.
2433 ccd = ccdExposure.getDetector()
2434 fpCorners = ccd.getCorners(FOCAL_PLANE)
2435 ccdPolygon = Polygon(fpCorners)
2438 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2441 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2442 validPolygon = Polygon(ccdPoints)
2443 ccdExposure.getInfo().setValidPolygon(validPolygon)
2447 """Context manager that applies and removes flats and darks,
2448 if the task is configured to apply them.
2452 exp : `lsst.afw.image.Exposure`
2453 Exposure to process.
2454 flat : `lsst.afw.image.Exposure`
2455 Flat exposure the same size as ``exp``.
2456 dark : `lsst.afw.image.Exposure`, optional
2457 Dark exposure the same size as ``exp``.
2461 exp : `lsst.afw.image.Exposure`
2462 The flat and dark corrected exposure.
2464 if self.config.doDark
and dark
is not None:
2466 if self.config.doFlat:
2471 if self.config.doFlat:
2473 if self.config.doDark
and dark
is not None:
2477 """Utility function to examine ISR exposure at different stages.
2481 exposure : `lsst.afw.image.Exposure`
2484 State of processing to view.
2486 frame = getDebugFrame(self._display, stepname)
2488 display = getDisplay(frame)
2489 display.scale(
'asinh',
'zscale')
2490 display.mtv(exposure)
2491 prompt =
"Press Enter to continue [c]... "
2493 ans = input(prompt).lower()
2494 if ans
in (
"",
"c",):
2499 """A Detector-like object that supports returning gain and saturation level
2501 This is used when the input exposure does not have a detector.
2505 exposure : `lsst.afw.image.Exposure`
2506 Exposure to generate a fake amplifier for.
2507 config : `lsst.ip.isr.isrTaskConfig`
2508 Configuration to apply to the fake amplifier.
2512 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2514 self.
_gain = config.gain
2541 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2545 """Task to wrap the default IsrTask to allow it to be retargeted.
2547 The standard IsrTask can be called directly from a command line
2548 program, but doing so removes the ability of the task to be
2549 retargeted. As most cameras override some set of the IsrTask
2550 methods, this would remove those data-specific methods in the
2551 output post-ISR images. This wrapping class fixes the issue,
2552 allowing identical post-ISR images to be generated by both the
2553 processCcd and isrTask code.
2555 ConfigClass = RunIsrConfig
2556 _DefaultName =
"runIsr"
2560 self.makeSubtask(
"isr")
2566 dataRef : `lsst.daf.persistence.ButlerDataRef`
2567 data reference of the detector data to be processed
2571 result : `pipeBase.Struct`
2572 Result struct with component:
2574 - exposure : `lsst.afw.image.Exposure`
2575 Post-ISR processed exposure.