28 import lsst.pex.config
as pexConfig
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument",
"calibration_label"],
115 crosstalk = cT.PrerequisiteInput(
117 doc=
"Input crosstalk object",
118 storageClass=
"CrosstalkCalib",
119 dimensions=[
"instrument",
"calibration_label",
"detector"],
123 crosstalkSources = cT.PrerequisiteInput(
124 name=
"isrOverscanCorrected",
125 doc=
"Overscan corrected input images.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
130 lookupFunction=crosstalkSourceLookup,
132 bias = cT.PrerequisiteInput(
134 doc=
"Input bias calibration.",
135 storageClass=
"ExposureF",
136 dimensions=[
"instrument",
"calibration_label",
"detector"],
138 dark = cT.PrerequisiteInput(
140 doc=
"Input dark calibration.",
141 storageClass=
"ExposureF",
142 dimensions=[
"instrument",
"calibration_label",
"detector"],
144 flat = cT.PrerequisiteInput(
146 doc=
"Input flat calibration.",
147 storageClass=
"ExposureF",
148 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
150 fringes = cT.PrerequisiteInput(
152 doc=
"Input fringe calibration.",
153 storageClass=
"ExposureF",
154 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
156 strayLightData = cT.PrerequisiteInput(
158 doc=
"Input stray light calibration.",
159 storageClass=
"StrayLightData",
160 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
162 bfKernel = cT.PrerequisiteInput(
164 doc=
"Input brighter-fatter kernel.",
165 storageClass=
"NumpyArray",
166 dimensions=[
"instrument",
"calibration_label"],
168 newBFKernel = cT.PrerequisiteInput(
169 name=
'brighterFatterKernel',
170 doc=
"Newer complete kernel + gain solutions.",
171 storageClass=
"BrighterFatterKernel",
172 dimensions=[
"instrument",
"calibration_label",
"detector"],
174 defects = cT.PrerequisiteInput(
176 doc=
"Input defect tables.",
177 storageClass=
"Defects",
178 dimensions=[
"instrument",
"calibration_label",
"detector"],
180 opticsTransmission = cT.PrerequisiteInput(
181 name=
"transmission_optics",
182 storageClass=
"TransmissionCurve",
183 doc=
"Transmission curve due to the optics.",
184 dimensions=[
"instrument",
"calibration_label"],
186 filterTransmission = cT.PrerequisiteInput(
187 name=
"transmission_filter",
188 storageClass=
"TransmissionCurve",
189 doc=
"Transmission curve due to the filter.",
190 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
192 sensorTransmission = cT.PrerequisiteInput(
193 name=
"transmission_sensor",
194 storageClass=
"TransmissionCurve",
195 doc=
"Transmission curve due to the sensor.",
196 dimensions=[
"instrument",
"calibration_label",
"detector"],
198 atmosphereTransmission = cT.PrerequisiteInput(
199 name=
"transmission_atmosphere",
200 storageClass=
"TransmissionCurve",
201 doc=
"Transmission curve due to the atmosphere.",
202 dimensions=[
"instrument"],
204 illumMaskedImage = cT.PrerequisiteInput(
206 doc=
"Input illumination correction.",
207 storageClass=
"MaskedImageF",
208 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
211 outputExposure = cT.Output(
213 doc=
"Output ISR processed exposure.",
214 storageClass=
"Exposure",
215 dimensions=[
"instrument",
"exposure",
"detector"],
217 preInterpExposure = cT.Output(
218 name=
'preInterpISRCCD',
219 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
220 storageClass=
"ExposureF",
221 dimensions=[
"instrument",
"exposure",
"detector"],
223 outputOssThumbnail = cT.Output(
225 doc=
"Output Overscan-subtracted thumbnail image.",
226 storageClass=
"Thumbnail",
227 dimensions=[
"instrument",
"exposure",
"detector"],
229 outputFlattenedThumbnail = cT.Output(
230 name=
"FlattenedThumb",
231 doc=
"Output flat-corrected thumbnail image.",
232 storageClass=
"Thumbnail",
233 dimensions=[
"instrument",
"exposure",
"detector"],
239 if config.doBias
is not True:
240 self.prerequisiteInputs.discard(
"bias")
241 if config.doLinearize
is not True:
242 self.prerequisiteInputs.discard(
"linearizer")
243 if config.doCrosstalk
is not True:
244 self.inputs.discard(
"crosstalkSources")
245 self.prerequisiteInputs.discard(
"crosstalk")
246 if config.doBrighterFatter
is not True:
247 self.prerequisiteInputs.discard(
"bfKernel")
248 self.prerequisiteInputs.discard(
"newBFKernel")
249 if config.doDefect
is not True:
250 self.prerequisiteInputs.discard(
"defects")
251 if config.doDark
is not True:
252 self.prerequisiteInputs.discard(
"dark")
253 if config.doFlat
is not True:
254 self.prerequisiteInputs.discard(
"flat")
255 if config.doAttachTransmissionCurve
is not True:
256 self.prerequisiteInputs.discard(
"opticsTransmission")
257 self.prerequisiteInputs.discard(
"filterTransmission")
258 self.prerequisiteInputs.discard(
"sensorTransmission")
259 self.prerequisiteInputs.discard(
"atmosphereTransmission")
260 if config.doUseOpticsTransmission
is not True:
261 self.prerequisiteInputs.discard(
"opticsTransmission")
262 if config.doUseFilterTransmission
is not True:
263 self.prerequisiteInputs.discard(
"filterTransmission")
264 if config.doUseSensorTransmission
is not True:
265 self.prerequisiteInputs.discard(
"sensorTransmission")
266 if config.doUseAtmosphereTransmission
is not True:
267 self.prerequisiteInputs.discard(
"atmosphereTransmission")
268 if config.doIlluminationCorrection
is not True:
269 self.prerequisiteInputs.discard(
"illumMaskedImage")
271 if config.doWrite
is not True:
272 self.outputs.discard(
"outputExposure")
273 self.outputs.discard(
"preInterpExposure")
274 self.outputs.discard(
"outputFlattenedThumbnail")
275 self.outputs.discard(
"outputOssThumbnail")
276 if config.doSaveInterpPixels
is not True:
277 self.outputs.discard(
"preInterpExposure")
278 if config.qa.doThumbnailOss
is not True:
279 self.outputs.discard(
"outputOssThumbnail")
280 if config.qa.doThumbnailFlattened
is not True:
281 self.outputs.discard(
"outputFlattenedThumbnail")
285 pipelineConnections=IsrTaskConnections):
286 """Configuration parameters for IsrTask.
288 Items are grouped in the order in which they are executed by the task.
290 datasetType = pexConfig.Field(
292 doc=
"Dataset type for input data; users will typically leave this alone, "
293 "but camera-specific ISR tasks will override it",
297 fallbackFilterName = pexConfig.Field(
299 doc=
"Fallback default filter name for calibrations.",
302 useFallbackDate = pexConfig.Field(
304 doc=
"Pass observation date when using fallback filter.",
307 expectWcs = pexConfig.Field(
310 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
312 fwhm = pexConfig.Field(
314 doc=
"FWHM of PSF in arcseconds.",
317 qa = pexConfig.ConfigField(
319 doc=
"QA related configuration options.",
323 doConvertIntToFloat = pexConfig.Field(
325 doc=
"Convert integer raw images to floating point values?",
330 doSaturation = pexConfig.Field(
332 doc=
"Mask saturated pixels? NB: this is totally independent of the"
333 " interpolation option - this is ONLY setting the bits in the mask."
334 " To have them interpolated make sure doSaturationInterpolation=True",
337 saturatedMaskName = pexConfig.Field(
339 doc=
"Name of mask plane to use in saturation detection and interpolation",
342 saturation = pexConfig.Field(
344 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
345 default=float(
"NaN"),
347 growSaturationFootprintSize = pexConfig.Field(
349 doc=
"Number of pixels by which to grow the saturation footprints",
354 doSuspect = pexConfig.Field(
356 doc=
"Mask suspect pixels?",
359 suspectMaskName = pexConfig.Field(
361 doc=
"Name of mask plane to use for suspect pixels",
364 numEdgeSuspect = pexConfig.Field(
366 doc=
"Number of edge pixels to be flagged as untrustworthy.",
369 edgeMaskLevel = pexConfig.ChoiceField(
371 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
374 'DETECTOR':
'Mask only the edges of the full detector.',
375 'AMP':
'Mask edges of each amplifier.',
380 doSetBadRegions = pexConfig.Field(
382 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
385 badStatistic = pexConfig.ChoiceField(
387 doc=
"How to estimate the average value for BAD regions.",
390 "MEANCLIP":
"Correct using the (clipped) mean of good data",
391 "MEDIAN":
"Correct using the median of the good data",
396 doOverscan = pexConfig.Field(
398 doc=
"Do overscan subtraction?",
401 overscan = pexConfig.ConfigurableField(
402 target=OverscanCorrectionTask,
403 doc=
"Overscan subtraction task for image segments.",
406 overscanFitType = pexConfig.ChoiceField(
408 doc=
"The method for fitting the overscan bias level.",
411 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
412 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
413 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
414 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
415 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
416 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
417 "MEAN":
"Correct using the mean of the overscan region",
418 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
419 "MEDIAN":
"Correct using the median of the overscan region",
420 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
422 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
423 " This option will no longer be used, and will be removed after v20.")
425 overscanOrder = pexConfig.Field(
427 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
428 "or number of spline knots if overscan fit type is a spline."),
430 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
431 " This option will no longer be used, and will be removed after v20.")
433 overscanNumSigmaClip = pexConfig.Field(
435 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
437 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
438 " This option will no longer be used, and will be removed after v20.")
440 overscanIsInt = pexConfig.Field(
442 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
443 " and overscan.FitType=MEDIAN_PER_ROW.",
445 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
446 " This option will no longer be used, and will be removed after v20.")
449 overscanNumLeadingColumnsToSkip = pexConfig.Field(
451 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
454 overscanNumTrailingColumnsToSkip = pexConfig.Field(
456 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
459 overscanMaxDev = pexConfig.Field(
461 doc=
"Maximum deviation from the median for overscan",
462 default=1000.0, check=
lambda x: x > 0
464 overscanBiasJump = pexConfig.Field(
466 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
469 overscanBiasJumpKeyword = pexConfig.Field(
471 doc=
"Header keyword containing information about devices.",
472 default=
"NO_SUCH_KEY",
474 overscanBiasJumpDevices = pexConfig.ListField(
476 doc=
"List of devices that need piecewise overscan correction.",
479 overscanBiasJumpLocation = pexConfig.Field(
481 doc=
"Location of bias jump along y-axis.",
486 doAssembleCcd = pexConfig.Field(
489 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
491 assembleCcd = pexConfig.ConfigurableField(
492 target=AssembleCcdTask,
493 doc=
"CCD assembly task",
497 doAssembleIsrExposures = pexConfig.Field(
500 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
502 doTrimToMatchCalib = pexConfig.Field(
505 doc=
"Trim raw data to match calibration bounding boxes?"
509 doBias = pexConfig.Field(
511 doc=
"Apply bias frame correction?",
514 biasDataProductName = pexConfig.Field(
516 doc=
"Name of the bias data product",
521 doVariance = pexConfig.Field(
523 doc=
"Calculate variance?",
526 gain = pexConfig.Field(
528 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
529 default=float(
"NaN"),
531 readNoise = pexConfig.Field(
533 doc=
"The read noise to use if no Detector is present in the Exposure",
536 doEmpiricalReadNoise = pexConfig.Field(
539 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
543 doLinearize = pexConfig.Field(
545 doc=
"Correct for nonlinearity of the detector's response?",
550 doCrosstalk = pexConfig.Field(
552 doc=
"Apply intra-CCD crosstalk correction?",
555 doCrosstalkBeforeAssemble = pexConfig.Field(
557 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
560 crosstalk = pexConfig.ConfigurableField(
561 target=CrosstalkTask,
562 doc=
"Intra-CCD crosstalk correction",
566 doDefect = pexConfig.Field(
568 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
571 doNanMasking = pexConfig.Field(
573 doc=
"Mask NAN pixels?",
576 doWidenSaturationTrails = pexConfig.Field(
578 doc=
"Widen bleed trails based on their width?",
583 doBrighterFatter = pexConfig.Field(
586 doc=
"Apply the brighter fatter correction"
588 brighterFatterLevel = pexConfig.ChoiceField(
591 doc=
"The level at which to correct for brighter-fatter.",
593 "AMP":
"Every amplifier treated separately.",
594 "DETECTOR":
"One kernel per detector",
597 brighterFatterMaxIter = pexConfig.Field(
600 doc=
"Maximum number of iterations for the brighter fatter correction"
602 brighterFatterThreshold = pexConfig.Field(
605 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
606 " absolute value of the difference between the current corrected image and the one"
607 " from the previous iteration summed over all the pixels."
609 brighterFatterApplyGain = pexConfig.Field(
612 doc=
"Should the gain be applied when applying the brighter fatter correction?"
614 brighterFatterMaskGrowSize = pexConfig.Field(
617 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
618 " when brighter-fatter correction is applied."
622 doDark = pexConfig.Field(
624 doc=
"Apply dark frame correction?",
627 darkDataProductName = pexConfig.Field(
629 doc=
"Name of the dark data product",
634 doStrayLight = pexConfig.Field(
636 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
639 strayLight = pexConfig.ConfigurableField(
640 target=StrayLightTask,
641 doc=
"y-band stray light correction"
645 doFlat = pexConfig.Field(
647 doc=
"Apply flat field correction?",
650 flatDataProductName = pexConfig.Field(
652 doc=
"Name of the flat data product",
655 flatScalingType = pexConfig.ChoiceField(
657 doc=
"The method for scaling the flat on the fly.",
660 "USER":
"Scale by flatUserScale",
661 "MEAN":
"Scale by the inverse of the mean",
662 "MEDIAN":
"Scale by the inverse of the median",
665 flatUserScale = pexConfig.Field(
667 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
670 doTweakFlat = pexConfig.Field(
672 doc=
"Tweak flats to match observed amplifier ratios?",
677 doApplyGains = pexConfig.Field(
679 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
682 normalizeGains = pexConfig.Field(
684 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
689 doFringe = pexConfig.Field(
691 doc=
"Apply fringe correction?",
694 fringe = pexConfig.ConfigurableField(
696 doc=
"Fringe subtraction task",
698 fringeAfterFlat = pexConfig.Field(
700 doc=
"Do fringe subtraction after flat-fielding?",
705 doMeasureBackground = pexConfig.Field(
707 doc=
"Measure the background level on the reduced image?",
712 doCameraSpecificMasking = pexConfig.Field(
714 doc=
"Mask camera-specific bad regions?",
717 masking = pexConfig.ConfigurableField(
724 doInterpolate = pexConfig.Field(
726 doc=
"Interpolate masked pixels?",
729 doSaturationInterpolation = pexConfig.Field(
731 doc=
"Perform interpolation over pixels masked as saturated?"
732 " NB: This is independent of doSaturation; if that is False this plane"
733 " will likely be blank, resulting in a no-op here.",
736 doNanInterpolation = pexConfig.Field(
738 doc=
"Perform interpolation over pixels masked as NaN?"
739 " NB: This is independent of doNanMasking; if that is False this plane"
740 " will likely be blank, resulting in a no-op here.",
743 doNanInterpAfterFlat = pexConfig.Field(
745 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
746 "also have to interpolate them before flat-fielding."),
749 maskListToInterpolate = pexConfig.ListField(
751 doc=
"List of mask planes that should be interpolated.",
752 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
754 doSaveInterpPixels = pexConfig.Field(
756 doc=
"Save a copy of the pre-interpolated pixel values?",
761 fluxMag0T1 = pexConfig.DictField(
764 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
765 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
768 defaultFluxMag0T1 = pexConfig.Field(
770 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
771 default=pow(10.0, 0.4*28.0)
775 doVignette = pexConfig.Field(
777 doc=
"Apply vignetting parameters?",
780 vignette = pexConfig.ConfigurableField(
782 doc=
"Vignetting task.",
786 doAttachTransmissionCurve = pexConfig.Field(
789 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
791 doUseOpticsTransmission = pexConfig.Field(
794 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
796 doUseFilterTransmission = pexConfig.Field(
799 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
801 doUseSensorTransmission = pexConfig.Field(
804 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
806 doUseAtmosphereTransmission = pexConfig.Field(
809 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
813 doIlluminationCorrection = pexConfig.Field(
816 doc=
"Perform illumination correction?"
818 illuminationCorrectionDataProductName = pexConfig.Field(
820 doc=
"Name of the illumination correction data product.",
823 illumScale = pexConfig.Field(
825 doc=
"Scale factor for the illumination correction.",
828 illumFilters = pexConfig.ListField(
831 doc=
"Only perform illumination correction for these filters."
835 doWrite = pexConfig.Field(
837 doc=
"Persist postISRCCD?",
844 raise ValueError(
"You may not specify both doFlat and doApplyGains")
846 self.config.maskListToInterpolate.append(
"SAT")
848 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
851 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
852 """Apply common instrument signature correction algorithms to a raw frame.
854 The process for correcting imaging data is very similar from
855 camera to camera. This task provides a vanilla implementation of
856 doing these corrections, including the ability to turn certain
857 corrections off if they are not needed. The inputs to the primary
858 method, `run()`, are a raw exposure to be corrected and the
859 calibration data products. The raw input is a single chip sized
860 mosaic of all amps including overscans and other non-science
861 pixels. The method `runDataRef()` identifies and defines the
862 calibration data products, and is intended for use by a
863 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
864 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
865 subclassed for different camera, although the most camera specific
866 methods have been split into subtasks that can be redirected
869 The __init__ method sets up the subtasks for ISR processing, using
870 the defaults from `lsst.ip.isr`.
875 Positional arguments passed to the Task constructor. None used at this time.
876 kwargs : `dict`, optional
877 Keyword arguments passed on to the Task constructor. None used at this time.
879 ConfigClass = IsrTaskConfig
884 self.makeSubtask(
"assembleCcd")
885 self.makeSubtask(
"crosstalk")
886 self.makeSubtask(
"strayLight")
887 self.makeSubtask(
"fringe")
888 self.makeSubtask(
"masking")
889 self.makeSubtask(
"overscan")
890 self.makeSubtask(
"vignette")
893 inputs = butlerQC.get(inputRefs)
896 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
897 except Exception
as e:
898 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
901 inputs[
'isGen3'] =
True
903 detector = inputs[
'ccdExposure'].getDetector()
905 if self.config.doCrosstalk
is True:
908 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
909 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
910 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
912 coeffVector = (self.config.crosstalk.crosstalkValues
913 if self.config.crosstalk.useConfigCoefficients
else None)
914 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
915 inputs[
'crosstalk'] = crosstalkCalib
916 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
917 if 'crosstalkSources' not in inputs:
918 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
921 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
923 linearizer.fromYaml(inputs[
'linearizer'])
927 inputs[
'linearizer'] = linearizer
929 if self.config.doDefect
is True:
930 if "defects" in inputs
and inputs[
'defects']
is not None:
933 if not isinstance(inputs[
"defects"], Defects):
934 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
938 if self.config.doBrighterFatter:
939 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
940 if brighterFatterKernel
is None:
941 brighterFatterKernel = inputs.get(
'bfKernel',
None)
943 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
944 detId = detector.getId()
945 inputs[
'bfGains'] = brighterFatterKernel.gain
948 if self.config.brighterFatterLevel ==
'DETECTOR':
949 if brighterFatterKernel.detectorKernel:
950 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
951 elif brighterFatterKernel.detectorKernelFromAmpKernels:
952 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
954 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
957 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
959 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
960 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
961 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
963 assembler=self.assembleCcd
964 if self.config.doAssembleIsrExposures
else None)
966 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
968 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
969 if 'strayLightData' not in inputs:
970 inputs[
'strayLightData'] =
None
972 outputs = self.
run(**inputs)
973 butlerQC.put(outputs, outputRefs)
976 """!Retrieve necessary frames for instrument signature removal.
978 Pre-fetching all required ISR data products limits the IO
979 required by the ISR. Any conflict between the calibration data
980 available and that needed for ISR is also detected prior to
981 doing processing, allowing it to fail quickly.
985 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
986 Butler reference of the detector data to be processed
987 rawExposure : `afw.image.Exposure`
988 The raw exposure that will later be corrected with the
989 retrieved calibration data; should not be modified in this
994 result : `lsst.pipe.base.Struct`
995 Result struct with components (which may be `None`):
996 - ``bias``: bias calibration frame (`afw.image.Exposure`)
997 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
998 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
999 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1000 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1001 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1002 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
1003 - ``fringes``: `lsst.pipe.base.Struct` with components:
1004 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1005 - ``seed``: random seed derived from the ccdExposureId for random
1006 number generator (`uint32`).
1007 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1008 A ``TransmissionCurve`` that represents the throughput of the optics,
1009 to be evaluated in focal-plane coordinates.
1010 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1011 A ``TransmissionCurve`` that represents the throughput of the filter
1012 itself, to be evaluated in focal-plane coordinates.
1013 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1014 A ``TransmissionCurve`` that represents the throughput of the sensor
1015 itself, to be evaluated in post-assembly trimmed detector coordinates.
1016 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1017 A ``TransmissionCurve`` that represents the throughput of the
1018 atmosphere, assumed to be spatially constant.
1019 - ``strayLightData`` : `object`
1020 An opaque object containing calibration information for
1021 stray-light correction. If `None`, no correction will be
1023 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1027 NotImplementedError :
1028 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1031 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1032 dateObs = dateObs.toPython().isoformat()
1033 except RuntimeError:
1034 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1037 ccd = rawExposure.getDetector()
1038 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
1039 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1040 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
1041 if self.config.doBias
else None)
1043 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1045 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1046 linearizer.log = self.log
1047 if isinstance(linearizer, numpy.ndarray):
1050 crosstalkCalib =
None
1051 if self.config.doCrosstalk:
1053 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1055 coeffVector = (self.config.crosstalk.crosstalkValues
1056 if self.config.crosstalk.useConfigCoefficients
else None)
1057 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1058 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1059 if self.config.doCrosstalk
else None)
1061 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
1062 if self.config.doDark
else None)
1063 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
1065 if self.config.doFlat
else None)
1067 brighterFatterKernel =
None
1068 brighterFatterGains =
None
1069 if self.config.doBrighterFatter
is True:
1074 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1075 brighterFatterGains = brighterFatterKernel.gain
1076 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1079 brighterFatterKernel = dataRef.get(
"bfKernel")
1080 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
1082 brighterFatterKernel =
None
1083 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1086 if self.config.brighterFatterLevel ==
'DETECTOR':
1087 if brighterFatterKernel.detectorKernel:
1088 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1089 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1090 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1092 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1095 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1097 defectList = (dataRef.get(
"defects")
1098 if self.config.doDefect
else None)
1099 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1100 if self.config.doAssembleIsrExposures
else None)
1101 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1102 else pipeBase.Struct(fringes=
None))
1104 if self.config.doAttachTransmissionCurve:
1105 opticsTransmission = (dataRef.get(
"transmission_optics")
1106 if self.config.doUseOpticsTransmission
else None)
1107 filterTransmission = (dataRef.get(
"transmission_filter")
1108 if self.config.doUseFilterTransmission
else None)
1109 sensorTransmission = (dataRef.get(
"transmission_sensor")
1110 if self.config.doUseSensorTransmission
else None)
1111 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1112 if self.config.doUseAtmosphereTransmission
else None)
1114 opticsTransmission =
None
1115 filterTransmission =
None
1116 sensorTransmission =
None
1117 atmosphereTransmission =
None
1119 if self.config.doStrayLight:
1120 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1122 strayLightData =
None
1125 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1126 if (self.config.doIlluminationCorrection
1127 and filterName
in self.config.illumFilters)
1131 return pipeBase.Struct(bias=biasExposure,
1132 linearizer=linearizer,
1133 crosstalk=crosstalkCalib,
1134 crosstalkSources=crosstalkSources,
1137 bfKernel=brighterFatterKernel,
1138 bfGains=brighterFatterGains,
1140 fringes=fringeStruct,
1141 opticsTransmission=opticsTransmission,
1142 filterTransmission=filterTransmission,
1143 sensorTransmission=sensorTransmission,
1144 atmosphereTransmission=atmosphereTransmission,
1145 strayLightData=strayLightData,
1146 illumMaskedImage=illumMaskedImage
1149 @pipeBase.timeMethod
1150 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1151 crosstalk=None, crosstalkSources=None,
1152 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1153 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1154 sensorTransmission=
None, atmosphereTransmission=
None,
1155 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1158 """!Perform instrument signature removal on an exposure.
1160 Steps included in the ISR processing, in order performed, are:
1161 - saturation and suspect pixel masking
1162 - overscan subtraction
1163 - CCD assembly of individual amplifiers
1165 - variance image construction
1166 - linearization of non-linear response
1168 - brighter-fatter correction
1171 - stray light subtraction
1173 - masking of known defects and camera specific features
1174 - vignette calculation
1175 - appending transmission curve and distortion model
1179 ccdExposure : `lsst.afw.image.Exposure`
1180 The raw exposure that is to be run through ISR. The
1181 exposure is modified by this method.
1182 camera : `lsst.afw.cameraGeom.Camera`, optional
1183 The camera geometry for this exposure. Required if ``isGen3`` is
1184 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1185 ``flat`` does not have an associated detector.
1186 bias : `lsst.afw.image.Exposure`, optional
1187 Bias calibration frame.
1188 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1189 Functor for linearization.
1190 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1191 Calibration for crosstalk.
1192 crosstalkSources : `list`, optional
1193 List of possible crosstalk sources.
1194 dark : `lsst.afw.image.Exposure`, optional
1195 Dark calibration frame.
1196 flat : `lsst.afw.image.Exposure`, optional
1197 Flat calibration frame.
1198 bfKernel : `numpy.ndarray`, optional
1199 Brighter-fatter kernel.
1200 bfGains : `dict` of `float`, optional
1201 Gains used to override the detector's nominal gains for the
1202 brighter-fatter correction. A dict keyed by amplifier name for
1203 the detector in question.
1204 defects : `lsst.meas.algorithms.Defects`, optional
1206 fringes : `lsst.pipe.base.Struct`, optional
1207 Struct containing the fringe correction data, with
1209 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1210 - ``seed``: random seed derived from the ccdExposureId for random
1211 number generator (`uint32`)
1212 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1213 A ``TransmissionCurve`` that represents the throughput of the optics,
1214 to be evaluated in focal-plane coordinates.
1215 filterTransmission : `lsst.afw.image.TransmissionCurve`
1216 A ``TransmissionCurve`` that represents the throughput of the filter
1217 itself, to be evaluated in focal-plane coordinates.
1218 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1219 A ``TransmissionCurve`` that represents the throughput of the sensor
1220 itself, to be evaluated in post-assembly trimmed detector coordinates.
1221 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1222 A ``TransmissionCurve`` that represents the throughput of the
1223 atmosphere, assumed to be spatially constant.
1224 detectorNum : `int`, optional
1225 The integer number for the detector to process.
1226 isGen3 : bool, optional
1227 Flag this call to run() as using the Gen3 butler environment.
1228 strayLightData : `object`, optional
1229 Opaque object containing calibration information for stray-light
1230 correction. If `None`, no correction will be performed.
1231 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1232 Illumination correction image.
1236 result : `lsst.pipe.base.Struct`
1237 Result struct with component:
1238 - ``exposure`` : `afw.image.Exposure`
1239 The fully ISR corrected exposure.
1240 - ``outputExposure`` : `afw.image.Exposure`
1241 An alias for `exposure`
1242 - ``ossThumb`` : `numpy.ndarray`
1243 Thumbnail image of the exposure after overscan subtraction.
1244 - ``flattenedThumb`` : `numpy.ndarray`
1245 Thumbnail image of the exposure after flat-field correction.
1250 Raised if a configuration option is set to True, but the
1251 required calibration data has not been specified.
1255 The current processed exposure can be viewed by setting the
1256 appropriate lsstDebug entries in the `debug.display`
1257 dictionary. The names of these entries correspond to some of
1258 the IsrTaskConfig Boolean options, with the value denoting the
1259 frame to use. The exposure is shown inside the matching
1260 option check and after the processing of that step has
1261 finished. The steps with debug points are:
1272 In addition, setting the "postISRCCD" entry displays the
1273 exposure after all ISR processing has finished.
1281 if detectorNum
is None:
1282 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1284 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1289 if isinstance(ccdExposure, ButlerDataRef):
1292 ccd = ccdExposure.getDetector()
1293 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1296 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1297 ccd = [
FakeAmp(ccdExposure, self.config)]
1300 if self.config.doBias
and bias
is None:
1301 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1303 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1304 if self.config.doBrighterFatter
and bfKernel
is None:
1305 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1306 if self.config.doDark
and dark
is None:
1307 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1308 if self.config.doFlat
and flat
is None:
1309 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1310 if self.config.doDefect
and defects
is None:
1311 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1312 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1313 and fringes.fringes
is None):
1318 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1319 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1320 and illumMaskedImage
is None):
1321 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1324 if self.config.doConvertIntToFloat:
1325 self.log.info(
"Converting exposure to floating point values.")
1332 if ccdExposure.getBBox().contains(amp.getBBox()):
1336 if self.config.doOverscan
and not badAmp:
1339 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1340 if overscanResults
is not None and \
1341 self.config.qa
is not None and self.config.qa.saveStats
is True:
1342 if isinstance(overscanResults.overscanFit, float):
1343 qaMedian = overscanResults.overscanFit
1344 qaStdev = float(
"NaN")
1346 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1347 afwMath.MEDIAN | afwMath.STDEVCLIP)
1348 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1349 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1351 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1352 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1353 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1354 amp.getName(), qaMedian, qaStdev)
1355 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1358 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1359 overscanResults =
None
1361 overscans.append(overscanResults
if overscanResults
is not None else None)
1363 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1365 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1366 self.log.info(
"Applying crosstalk correction.")
1367 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1368 crosstalkSources=crosstalkSources)
1369 self.
debugView(ccdExposure,
"doCrosstalk")
1371 if self.config.doAssembleCcd:
1372 self.log.info(
"Assembling CCD from amplifiers.")
1373 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1375 if self.config.expectWcs
and not ccdExposure.getWcs():
1376 self.log.warn(
"No WCS found in input exposure.")
1377 self.
debugView(ccdExposure,
"doAssembleCcd")
1380 if self.config.qa.doThumbnailOss:
1381 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1383 if self.config.doBias:
1384 self.log.info(
"Applying bias correction.")
1385 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1386 trimToFit=self.config.doTrimToMatchCalib)
1389 if self.config.doVariance:
1390 for amp, overscanResults
in zip(ccd, overscans):
1391 if ccdExposure.getBBox().contains(amp.getBBox()):
1392 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1393 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1394 if overscanResults
is not None:
1396 overscanImage=overscanResults.overscanImage)
1400 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1401 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1402 afwMath.MEDIAN | afwMath.STDEVCLIP)
1403 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1404 qaStats.getValue(afwMath.MEDIAN))
1405 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1406 qaStats.getValue(afwMath.STDEVCLIP))
1407 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1408 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1409 qaStats.getValue(afwMath.STDEVCLIP))
1412 self.log.info(
"Applying linearizer.")
1413 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1414 detector=ccd, log=self.log)
1416 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1417 self.log.info(
"Applying crosstalk correction.")
1418 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1419 crosstalkSources=crosstalkSources, isTrimmed=
True)
1420 self.
debugView(ccdExposure,
"doCrosstalk")
1424 if self.config.doDefect:
1425 self.log.info(
"Masking defects.")
1428 if self.config.numEdgeSuspect > 0:
1429 self.log.info(
"Masking edges as SUSPECT.")
1430 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1431 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1433 if self.config.doNanMasking:
1434 self.log.info(
"Masking NAN value pixels.")
1437 if self.config.doWidenSaturationTrails:
1438 self.log.info(
"Widening saturation trails.")
1439 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1441 if self.config.doCameraSpecificMasking:
1442 self.log.info(
"Masking regions for camera specific reasons.")
1443 self.masking.
run(ccdExposure)
1445 if self.config.doBrighterFatter:
1454 interpExp = ccdExposure.clone()
1456 isrFunctions.interpolateFromMask(
1457 maskedImage=interpExp.getMaskedImage(),
1458 fwhm=self.config.fwhm,
1459 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1460 maskNameList=self.config.maskListToInterpolate
1462 bfExp = interpExp.clone()
1464 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1465 type(bfKernel), type(bfGains))
1466 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1467 self.config.brighterFatterMaxIter,
1468 self.config.brighterFatterThreshold,
1469 self.config.brighterFatterApplyGain,
1471 if bfResults[1] == self.config.brighterFatterMaxIter:
1472 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1475 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1477 image = ccdExposure.getMaskedImage().getImage()
1478 bfCorr = bfExp.getMaskedImage().getImage()
1479 bfCorr -= interpExp.getMaskedImage().getImage()
1488 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1489 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1492 if self.config.brighterFatterMaskGrowSize > 0:
1493 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1494 for maskPlane
in self.config.maskListToInterpolate:
1495 isrFunctions.growMasks(ccdExposure.getMask(),
1496 radius=self.config.brighterFatterMaskGrowSize,
1497 maskNameList=maskPlane,
1498 maskValue=maskPlane)
1500 self.
debugView(ccdExposure,
"doBrighterFatter")
1502 if self.config.doDark:
1503 self.log.info(
"Applying dark correction.")
1507 if self.config.doFringe
and not self.config.fringeAfterFlat:
1508 self.log.info(
"Applying fringe correction before flat.")
1509 self.fringe.
run(ccdExposure, **fringes.getDict())
1512 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1513 self.log.info(
"Checking strayLight correction.")
1514 self.strayLight.
run(ccdExposure, strayLightData)
1515 self.
debugView(ccdExposure,
"doStrayLight")
1517 if self.config.doFlat:
1518 self.log.info(
"Applying flat correction.")
1522 if self.config.doApplyGains:
1523 self.log.info(
"Applying gain correction instead of flat.")
1524 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1526 if self.config.doFringe
and self.config.fringeAfterFlat:
1527 self.log.info(
"Applying fringe correction after flat.")
1528 self.fringe.
run(ccdExposure, **fringes.getDict())
1530 if self.config.doVignette:
1531 self.log.info(
"Constructing Vignette polygon.")
1534 if self.config.vignette.doWriteVignettePolygon:
1537 if self.config.doAttachTransmissionCurve:
1538 self.log.info(
"Adding transmission curves.")
1539 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1540 filterTransmission=filterTransmission,
1541 sensorTransmission=sensorTransmission,
1542 atmosphereTransmission=atmosphereTransmission)
1544 flattenedThumb =
None
1545 if self.config.qa.doThumbnailFlattened:
1546 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1548 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1549 self.log.info(
"Performing illumination correction.")
1550 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1551 illumMaskedImage, illumScale=self.config.illumScale,
1552 trimToFit=self.config.doTrimToMatchCalib)
1555 if self.config.doSaveInterpPixels:
1556 preInterpExp = ccdExposure.clone()
1571 if self.config.doSetBadRegions:
1572 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1573 if badPixelCount > 0:
1574 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1576 if self.config.doInterpolate:
1577 self.log.info(
"Interpolating masked pixels.")
1578 isrFunctions.interpolateFromMask(
1579 maskedImage=ccdExposure.getMaskedImage(),
1580 fwhm=self.config.fwhm,
1581 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1582 maskNameList=list(self.config.maskListToInterpolate)
1587 if self.config.doMeasureBackground:
1588 self.log.info(
"Measuring background level.")
1591 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1593 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1594 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1595 afwMath.MEDIAN | afwMath.STDEVCLIP)
1596 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1597 qaStats.getValue(afwMath.MEDIAN))
1598 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1599 qaStats.getValue(afwMath.STDEVCLIP))
1600 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1601 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1602 qaStats.getValue(afwMath.STDEVCLIP))
1604 self.
debugView(ccdExposure,
"postISRCCD")
1606 return pipeBase.Struct(
1607 exposure=ccdExposure,
1609 flattenedThumb=flattenedThumb,
1611 preInterpolatedExposure=preInterpExp,
1612 outputExposure=ccdExposure,
1613 outputOssThumbnail=ossThumb,
1614 outputFlattenedThumbnail=flattenedThumb,
1617 @pipeBase.timeMethod
1619 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1621 This method contains the `CmdLineTask` interface to the ISR
1622 processing. All IO is handled here, freeing the `run()` method
1623 to manage only pixel-level calculations. The steps performed
1625 - Read in necessary detrending/isr/calibration data.
1626 - Process raw exposure in `run()`.
1627 - Persist the ISR-corrected exposure as "postISRCCD" if
1628 config.doWrite=True.
1632 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1633 DataRef of the detector data to be processed
1637 result : `lsst.pipe.base.Struct`
1638 Result struct with component:
1639 - ``exposure`` : `afw.image.Exposure`
1640 The fully ISR corrected exposure.
1645 Raised if a configuration option is set to True, but the
1646 required calibration data does not exist.
1649 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1651 ccdExposure = sensorRef.get(self.config.datasetType)
1653 camera = sensorRef.get(
"camera")
1654 isrData = self.
readIsrData(sensorRef, ccdExposure)
1656 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1658 if self.config.doWrite:
1659 sensorRef.put(result.exposure,
"postISRCCD")
1660 if result.preInterpolatedExposure
is not None:
1661 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1662 if result.ossThumb
is not None:
1663 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1664 if result.flattenedThumb
is not None:
1665 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1670 """!Retrieve a calibration dataset for removing instrument signature.
1675 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1676 DataRef of the detector data to find calibration datasets
1679 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1680 dateObs : `str`, optional
1681 Date of the observation. Used to correct butler failures
1682 when using fallback filters.
1684 If True, disable butler proxies to enable error handling
1685 within this routine.
1689 exposure : `lsst.afw.image.Exposure`
1690 Requested calibration frame.
1695 Raised if no matching calibration frame can be found.
1698 exp = dataRef.get(datasetType, immediate=immediate)
1699 except Exception
as exc1:
1700 if not self.config.fallbackFilterName:
1701 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1703 if self.config.useFallbackDate
and dateObs:
1704 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1705 dateObs=dateObs, immediate=immediate)
1707 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1708 except Exception
as exc2:
1709 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1710 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1711 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1713 if self.config.doAssembleIsrExposures:
1714 exp = self.assembleCcd.assembleCcd(exp)
1718 """Ensure that the data returned by Butler is a fully constructed exposure.
1720 ISR requires exposure-level image data for historical reasons, so if we did
1721 not recieve that from Butler, construct it from what we have, modifying the
1726 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1727 `lsst.afw.image.ImageF`
1728 The input data structure obtained from Butler.
1729 camera : `lsst.afw.cameraGeom.camera`
1730 The camera associated with the image. Used to find the appropriate
1733 The detector this exposure should match.
1737 inputExp : `lsst.afw.image.Exposure`
1738 The re-constructed exposure, with appropriate detector parameters.
1743 Raised if the input data cannot be used to construct an exposure.
1745 if isinstance(inputExp, afwImage.DecoratedImageU):
1746 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1747 elif isinstance(inputExp, afwImage.ImageF):
1748 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1749 elif isinstance(inputExp, afwImage.MaskedImageF):
1750 inputExp = afwImage.makeExposure(inputExp)
1751 elif isinstance(inputExp, afwImage.Exposure):
1753 elif inputExp
is None:
1757 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1760 if inputExp.getDetector()
is None:
1761 inputExp.setDetector(camera[detectorNum])
1766 """Convert exposure image from uint16 to float.
1768 If the exposure does not need to be converted, the input is
1769 immediately returned. For exposures that are converted to use
1770 floating point pixels, the variance is set to unity and the
1775 exposure : `lsst.afw.image.Exposure`
1776 The raw exposure to be converted.
1780 newexposure : `lsst.afw.image.Exposure`
1781 The input ``exposure``, converted to floating point pixels.
1786 Raised if the exposure type cannot be converted to float.
1789 if isinstance(exposure, afwImage.ExposureF):
1791 self.log.debug(
"Exposure already of type float.")
1793 if not hasattr(exposure,
"convertF"):
1794 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1796 newexposure = exposure.convertF()
1797 newexposure.variance[:] = 1
1798 newexposure.mask[:] = 0x0
1803 """Identify bad amplifiers, saturated and suspect pixels.
1807 ccdExposure : `lsst.afw.image.Exposure`
1808 Input exposure to be masked.
1809 amp : `lsst.afw.table.AmpInfoCatalog`
1810 Catalog of parameters defining the amplifier on this
1812 defects : `lsst.meas.algorithms.Defects`
1813 List of defects. Used to determine if the entire
1819 If this is true, the entire amplifier area is covered by
1820 defects and unusable.
1823 maskedImage = ccdExposure.getMaskedImage()
1829 if defects
is not None:
1830 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1835 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1837 maskView = dataView.getMask()
1838 maskView |= maskView.getPlaneBitMask(
"BAD")
1845 if self.config.doSaturation
and not badAmp:
1846 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1847 if self.config.doSuspect
and not badAmp:
1848 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1849 if math.isfinite(self.config.saturation):
1850 limits.update({self.config.saturatedMaskName: self.config.saturation})
1852 for maskName, maskThreshold
in limits.items():
1853 if not math.isnan(maskThreshold):
1854 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1855 isrFunctions.makeThresholdMask(
1856 maskedImage=dataView,
1857 threshold=maskThreshold,
1863 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1865 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1866 self.config.suspectMaskName])
1867 if numpy.all(maskView.getArray() & maskVal > 0):
1869 maskView |= maskView.getPlaneBitMask(
"BAD")
1874 """Apply overscan correction in place.
1876 This method does initial pixel rejection of the overscan
1877 region. The overscan can also be optionally segmented to
1878 allow for discontinuous overscan responses to be fit
1879 separately. The actual overscan subtraction is performed by
1880 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1881 which is called here after the amplifier is preprocessed.
1885 ccdExposure : `lsst.afw.image.Exposure`
1886 Exposure to have overscan correction performed.
1887 amp : `lsst.afw.table.AmpInfoCatalog`
1888 The amplifier to consider while correcting the overscan.
1892 overscanResults : `lsst.pipe.base.Struct`
1893 Result struct with components:
1894 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1895 Value or fit subtracted from the amplifier image data.
1896 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1897 Value or fit subtracted from the overscan image data.
1898 - ``overscanImage`` : `lsst.afw.image.Image`
1899 Image of the overscan region with the overscan
1900 correction applied. This quantity is used to estimate
1901 the amplifier read noise empirically.
1906 Raised if the ``amp`` does not contain raw pixel information.
1910 lsst.ip.isr.isrFunctions.overscanCorrection
1912 if amp.getRawHorizontalOverscanBBox().isEmpty():
1913 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1916 statControl = afwMath.StatisticsControl()
1917 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1920 dataBBox = amp.getRawDataBBox()
1921 oscanBBox = amp.getRawHorizontalOverscanBBox()
1925 prescanBBox = amp.getRawPrescanBBox()
1926 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1927 dx0 += self.config.overscanNumLeadingColumnsToSkip
1928 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1930 dx0 += self.config.overscanNumTrailingColumnsToSkip
1931 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1937 if ((self.config.overscanBiasJump
1938 and self.config.overscanBiasJumpLocation)
1939 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1940 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
1941 self.config.overscanBiasJumpDevices)):
1942 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1943 yLower = self.config.overscanBiasJumpLocation
1944 yUpper = dataBBox.getHeight() - yLower
1946 yUpper = self.config.overscanBiasJumpLocation
1947 yLower = dataBBox.getHeight() - yUpper
1965 oscanBBox.getHeight())))
1968 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1969 ampImage = ccdExposure.maskedImage[imageBBox]
1970 overscanImage = ccdExposure.maskedImage[overscanBBox]
1972 overscanArray = overscanImage.image.array
1973 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1974 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1975 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1977 statControl = afwMath.StatisticsControl()
1978 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1980 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage)
1983 levelStat = afwMath.MEDIAN
1984 sigmaStat = afwMath.STDEVCLIP
1986 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1987 self.config.qa.flatness.nIter)
1988 metadata = ccdExposure.getMetadata()
1989 ampNum = amp.getName()
1991 if isinstance(overscanResults.overscanFit, float):
1992 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1993 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1995 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1996 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1997 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1999 return overscanResults
2002 """Set the variance plane using the amplifier gain and read noise
2004 The read noise is calculated from the ``overscanImage`` if the
2005 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2006 the value from the amplifier data is used.
2010 ampExposure : `lsst.afw.image.Exposure`
2011 Exposure to process.
2012 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2013 Amplifier detector data.
2014 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2015 Image of overscan, required only for empirical read noise.
2019 lsst.ip.isr.isrFunctions.updateVariance
2021 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2022 gain = amp.getGain()
2024 if math.isnan(gain):
2026 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2029 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2030 amp.getName(), gain, patchedGain)
2033 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2034 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
2036 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2037 stats = afwMath.StatisticsControl()
2038 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2039 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2040 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2041 amp.getName(), readNoise)
2043 readNoise = amp.getReadNoise()
2045 isrFunctions.updateVariance(
2046 maskedImage=ampExposure.getMaskedImage(),
2048 readNoise=readNoise,
2052 """!Apply dark correction in place.
2056 exposure : `lsst.afw.image.Exposure`
2057 Exposure to process.
2058 darkExposure : `lsst.afw.image.Exposure`
2059 Dark exposure of the same size as ``exposure``.
2060 invert : `Bool`, optional
2061 If True, re-add the dark to an already corrected image.
2066 Raised if either ``exposure`` or ``darkExposure`` do not
2067 have their dark time defined.
2071 lsst.ip.isr.isrFunctions.darkCorrection
2073 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2074 if math.isnan(expScale):
2075 raise RuntimeError(
"Exposure darktime is NAN.")
2076 if darkExposure.getInfo().getVisitInfo()
is not None \
2077 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2078 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2082 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2085 isrFunctions.darkCorrection(
2086 maskedImage=exposure.getMaskedImage(),
2087 darkMaskedImage=darkExposure.getMaskedImage(),
2089 darkScale=darkScale,
2091 trimToFit=self.config.doTrimToMatchCalib
2095 """!Check if linearization is needed for the detector cameraGeom.
2097 Checks config.doLinearize and the linearity type of the first
2102 detector : `lsst.afw.cameraGeom.Detector`
2103 Detector to get linearity type from.
2107 doLinearize : `Bool`
2108 If True, linearization should be performed.
2110 return self.config.doLinearize
and \
2111 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2114 """!Apply flat correction in place.
2118 exposure : `lsst.afw.image.Exposure`
2119 Exposure to process.
2120 flatExposure : `lsst.afw.image.Exposure`
2121 Flat exposure of the same size as ``exposure``.
2122 invert : `Bool`, optional
2123 If True, unflatten an already flattened image.
2127 lsst.ip.isr.isrFunctions.flatCorrection
2129 isrFunctions.flatCorrection(
2130 maskedImage=exposure.getMaskedImage(),
2131 flatMaskedImage=flatExposure.getMaskedImage(),
2132 scalingType=self.config.flatScalingType,
2133 userScale=self.config.flatUserScale,
2135 trimToFit=self.config.doTrimToMatchCalib
2139 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2143 exposure : `lsst.afw.image.Exposure`
2144 Exposure to process. Only the amplifier DataSec is processed.
2145 amp : `lsst.afw.table.AmpInfoCatalog`
2146 Amplifier detector data.
2150 lsst.ip.isr.isrFunctions.makeThresholdMask
2152 if not math.isnan(amp.getSaturation()):
2153 maskedImage = exposure.getMaskedImage()
2154 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2155 isrFunctions.makeThresholdMask(
2156 maskedImage=dataView,
2157 threshold=amp.getSaturation(),
2159 maskName=self.config.saturatedMaskName,
2163 """!Interpolate over saturated pixels, in place.
2165 This method should be called after `saturationDetection`, to
2166 ensure that the saturated pixels have been identified in the
2167 SAT mask. It should also be called after `assembleCcd`, since
2168 saturated regions may cross amplifier boundaries.
2172 exposure : `lsst.afw.image.Exposure`
2173 Exposure to process.
2177 lsst.ip.isr.isrTask.saturationDetection
2178 lsst.ip.isr.isrFunctions.interpolateFromMask
2180 isrFunctions.interpolateFromMask(
2181 maskedImage=exposure.getMaskedImage(),
2182 fwhm=self.config.fwhm,
2183 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2184 maskNameList=list(self.config.saturatedMaskName),
2188 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2192 exposure : `lsst.afw.image.Exposure`
2193 Exposure to process. Only the amplifier DataSec is processed.
2194 amp : `lsst.afw.table.AmpInfoCatalog`
2195 Amplifier detector data.
2199 lsst.ip.isr.isrFunctions.makeThresholdMask
2203 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2204 This is intended to indicate pixels that may be affected by unknown systematics;
2205 for example if non-linearity corrections above a certain level are unstable
2206 then that would be a useful value for suspectLevel. A value of `nan` indicates
2207 that no such level exists and no pixels are to be masked as suspicious.
2209 suspectLevel = amp.getSuspectLevel()
2210 if math.isnan(suspectLevel):
2213 maskedImage = exposure.getMaskedImage()
2214 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2215 isrFunctions.makeThresholdMask(
2216 maskedImage=dataView,
2217 threshold=suspectLevel,
2219 maskName=self.config.suspectMaskName,
2223 """!Mask defects using mask plane "BAD", in place.
2227 exposure : `lsst.afw.image.Exposure`
2228 Exposure to process.
2229 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2230 `lsst.afw.image.DefectBase`.
2231 List of defects to mask.
2235 Call this after CCD assembly, since defects may cross amplifier boundaries.
2237 maskedImage = exposure.getMaskedImage()
2238 if not isinstance(defectBaseList, Defects):
2240 defectList = Defects(defectBaseList)
2242 defectList = defectBaseList
2243 defectList.maskPixels(maskedImage, maskName=
"BAD")
2245 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2246 """!Mask edge pixels with applicable mask plane.
2250 exposure : `lsst.afw.image.Exposure`
2251 Exposure to process.
2252 numEdgePixels : `int`, optional
2253 Number of edge pixels to mask.
2254 maskPlane : `str`, optional
2255 Mask plane name to use.
2256 level : `str`, optional
2257 Level at which to mask edges.
2259 maskedImage = exposure.getMaskedImage()
2260 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2262 if numEdgePixels > 0:
2263 if level ==
'DETECTOR':
2264 boxes = [maskedImage.getBBox()]
2265 elif level ==
'AMP':
2266 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2270 subImage = maskedImage[box]
2271 box.grow(-numEdgePixels)
2273 SourceDetectionTask.setEdgeBits(
2279 """Mask and interpolate defects using mask plane "BAD", in place.
2283 exposure : `lsst.afw.image.Exposure`
2284 Exposure to process.
2285 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2286 `lsst.afw.image.DefectBase`.
2287 List of defects to mask and interpolate.
2291 lsst.ip.isr.isrTask.maskDefect()
2294 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2295 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2296 isrFunctions.interpolateFromMask(
2297 maskedImage=exposure.getMaskedImage(),
2298 fwhm=self.config.fwhm,
2299 growSaturatedFootprints=0,
2300 maskNameList=[
"BAD"],
2304 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2308 exposure : `lsst.afw.image.Exposure`
2309 Exposure to process.
2313 We mask over all NaNs, including those that are masked with
2314 other bits (because those may or may not be interpolated over
2315 later, and we want to remove all NaNs). Despite this
2316 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2317 the historical name.
2319 maskedImage = exposure.getMaskedImage()
2322 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2323 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2324 numNans =
maskNans(maskedImage, maskVal)
2325 self.metadata.set(
"NUMNANS", numNans)
2327 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2330 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2334 exposure : `lsst.afw.image.Exposure`
2335 Exposure to process.
2339 lsst.ip.isr.isrTask.maskNan()
2342 isrFunctions.interpolateFromMask(
2343 maskedImage=exposure.getMaskedImage(),
2344 fwhm=self.config.fwhm,
2345 growSaturatedFootprints=0,
2346 maskNameList=[
"UNMASKEDNAN"],
2350 """Measure the image background in subgrids, for quality control purposes.
2354 exposure : `lsst.afw.image.Exposure`
2355 Exposure to process.
2356 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2357 Configuration object containing parameters on which background
2358 statistics and subgrids to use.
2360 if IsrQaConfig
is not None:
2361 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2362 IsrQaConfig.flatness.nIter)
2363 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2364 statsControl.setAndMask(maskVal)
2365 maskedImage = exposure.getMaskedImage()
2366 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2367 skyLevel = stats.getValue(afwMath.MEDIAN)
2368 skySigma = stats.getValue(afwMath.STDEVCLIP)
2369 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2370 metadata = exposure.getMetadata()
2371 metadata.set(
'SKYLEVEL', skyLevel)
2372 metadata.set(
'SKYSIGMA', skySigma)
2375 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2376 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2377 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2378 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2379 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2380 skyLevels = numpy.zeros((nX, nY))
2383 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2385 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2387 xLLC = xc - meshXHalf
2388 yLLC = yc - meshYHalf
2389 xURC = xc + meshXHalf - 1
2390 yURC = yc + meshYHalf - 1
2393 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2395 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2397 good = numpy.where(numpy.isfinite(skyLevels))
2398 skyMedian = numpy.median(skyLevels[good])
2399 flatness = (skyLevels[good] - skyMedian) / skyMedian
2400 flatness_rms = numpy.std(flatness)
2401 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2403 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2404 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2405 nX, nY, flatness_pp, flatness_rms)
2407 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2408 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2409 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2410 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2411 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2414 """Set an approximate magnitude zero point for the exposure.
2418 exposure : `lsst.afw.image.Exposure`
2419 Exposure to process.
2421 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2422 if filterName
in self.config.fluxMag0T1:
2423 fluxMag0 = self.config.fluxMag0T1[filterName]
2425 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2426 fluxMag0 = self.config.defaultFluxMag0T1
2428 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2430 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2433 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2434 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2437 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2441 ccdExposure : `lsst.afw.image.Exposure`
2442 Exposure to process.
2443 fpPolygon : `lsst.afw.geom.Polygon`
2444 Polygon in focal plane coordinates.
2447 ccd = ccdExposure.getDetector()
2448 fpCorners = ccd.getCorners(FOCAL_PLANE)
2449 ccdPolygon = Polygon(fpCorners)
2452 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2455 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2456 validPolygon = Polygon(ccdPoints)
2457 ccdExposure.getInfo().setValidPolygon(validPolygon)
2461 """Context manager that applies and removes flats and darks,
2462 if the task is configured to apply them.
2466 exp : `lsst.afw.image.Exposure`
2467 Exposure to process.
2468 flat : `lsst.afw.image.Exposure`
2469 Flat exposure the same size as ``exp``.
2470 dark : `lsst.afw.image.Exposure`, optional
2471 Dark exposure the same size as ``exp``.
2475 exp : `lsst.afw.image.Exposure`
2476 The flat and dark corrected exposure.
2478 if self.config.doDark
and dark
is not None:
2480 if self.config.doFlat:
2485 if self.config.doFlat:
2487 if self.config.doDark
and dark
is not None:
2491 """Utility function to examine ISR exposure at different stages.
2495 exposure : `lsst.afw.image.Exposure`
2498 State of processing to view.
2500 frame = getDebugFrame(self._display, stepname)
2502 display = getDisplay(frame)
2503 display.scale(
'asinh',
'zscale')
2504 display.mtv(exposure)
2505 prompt =
"Press Enter to continue [c]... "
2507 ans = input(prompt).lower()
2508 if ans
in (
"",
"c",):
2513 """A Detector-like object that supports returning gain and saturation level
2515 This is used when the input exposure does not have a detector.
2519 exposure : `lsst.afw.image.Exposure`
2520 Exposure to generate a fake amplifier for.
2521 config : `lsst.ip.isr.isrTaskConfig`
2522 Configuration to apply to the fake amplifier.
2526 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2528 self.
_gain = config.gain
2555 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2559 """Task to wrap the default IsrTask to allow it to be retargeted.
2561 The standard IsrTask can be called directly from a command line
2562 program, but doing so removes the ability of the task to be
2563 retargeted. As most cameras override some set of the IsrTask
2564 methods, this would remove those data-specific methods in the
2565 output post-ISR images. This wrapping class fixes the issue,
2566 allowing identical post-ISR images to be generated by both the
2567 processCcd and isrTask code.
2569 ConfigClass = RunIsrConfig
2570 _DefaultName =
"runIsr"
2574 self.makeSubtask(
"isr")
2580 dataRef : `lsst.daf.persistence.ButlerDataRef`
2581 data reference of the detector data to be processed
2585 result : `pipeBase.Struct`
2586 Result struct with component:
2588 - exposure : `lsst.afw.image.Exposure`
2589 Post-ISR processed exposure.