28 import lsst.pex.config
as pexConfig
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument",
"calibration_label"],
115 crosstalk = cT.PrerequisiteInput(
117 doc=
"Input crosstalk object",
118 storageClass=
"CrosstalkCalib",
119 dimensions=[
"instrument",
"calibration_label",
"detector"],
123 crosstalkSources = cT.PrerequisiteInput(
124 name=
"isrOverscanCorrected",
125 doc=
"Overscan corrected input images.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
130 lookupFunction=crosstalkSourceLookup,
132 bias = cT.PrerequisiteInput(
134 doc=
"Input bias calibration.",
135 storageClass=
"ExposureF",
136 dimensions=[
"instrument",
"calibration_label",
"detector"],
138 dark = cT.PrerequisiteInput(
140 doc=
"Input dark calibration.",
141 storageClass=
"ExposureF",
142 dimensions=[
"instrument",
"calibration_label",
"detector"],
144 flat = cT.PrerequisiteInput(
146 doc=
"Input flat calibration.",
147 storageClass=
"ExposureF",
148 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
150 fringes = cT.PrerequisiteInput(
152 doc=
"Input fringe calibration.",
153 storageClass=
"ExposureF",
154 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
156 strayLightData = cT.PrerequisiteInput(
158 doc=
"Input stray light calibration.",
159 storageClass=
"StrayLightData",
160 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
162 bfKernel = cT.PrerequisiteInput(
164 doc=
"Input brighter-fatter kernel.",
165 storageClass=
"NumpyArray",
166 dimensions=[
"instrument",
"calibration_label"],
168 newBFKernel = cT.PrerequisiteInput(
169 name=
'brighterFatterKernel',
170 doc=
"Newer complete kernel + gain solutions.",
171 storageClass=
"BrighterFatterKernel",
172 dimensions=[
"instrument",
"calibration_label",
"detector"],
174 defects = cT.PrerequisiteInput(
176 doc=
"Input defect tables.",
177 storageClass=
"Defects",
178 dimensions=[
"instrument",
"calibration_label",
"detector"],
180 opticsTransmission = cT.PrerequisiteInput(
181 name=
"transmission_optics",
182 storageClass=
"TransmissionCurve",
183 doc=
"Transmission curve due to the optics.",
184 dimensions=[
"instrument",
"calibration_label"],
186 filterTransmission = cT.PrerequisiteInput(
187 name=
"transmission_filter",
188 storageClass=
"TransmissionCurve",
189 doc=
"Transmission curve due to the filter.",
190 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
192 sensorTransmission = cT.PrerequisiteInput(
193 name=
"transmission_sensor",
194 storageClass=
"TransmissionCurve",
195 doc=
"Transmission curve due to the sensor.",
196 dimensions=[
"instrument",
"calibration_label",
"detector"],
198 atmosphereTransmission = cT.PrerequisiteInput(
199 name=
"transmission_atmosphere",
200 storageClass=
"TransmissionCurve",
201 doc=
"Transmission curve due to the atmosphere.",
202 dimensions=[
"instrument"],
204 illumMaskedImage = cT.PrerequisiteInput(
206 doc=
"Input illumination correction.",
207 storageClass=
"MaskedImageF",
208 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
211 outputExposure = cT.Output(
213 doc=
"Output ISR processed exposure.",
214 storageClass=
"Exposure",
215 dimensions=[
"instrument",
"exposure",
"detector"],
217 preInterpExposure = cT.Output(
218 name=
'preInterpISRCCD',
219 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
220 storageClass=
"ExposureF",
221 dimensions=[
"instrument",
"exposure",
"detector"],
223 outputOssThumbnail = cT.Output(
225 doc=
"Output Overscan-subtracted thumbnail image.",
226 storageClass=
"Thumbnail",
227 dimensions=[
"instrument",
"exposure",
"detector"],
229 outputFlattenedThumbnail = cT.Output(
230 name=
"FlattenedThumb",
231 doc=
"Output flat-corrected thumbnail image.",
232 storageClass=
"Thumbnail",
233 dimensions=[
"instrument",
"exposure",
"detector"],
239 if config.doBias
is not True:
240 self.prerequisiteInputs.discard(
"bias")
241 if config.doLinearize
is not True:
242 self.prerequisiteInputs.discard(
"linearizer")
243 if config.doCrosstalk
is not True:
244 self.inputs.discard(
"crosstalkSources")
245 self.prerequisiteInputs.discard(
"crosstalk")
246 if config.doBrighterFatter
is not True:
247 self.prerequisiteInputs.discard(
"bfKernel")
248 self.prerequisiteInputs.discard(
"newBFKernel")
249 if config.doDefect
is not True:
250 self.prerequisiteInputs.discard(
"defects")
251 if config.doDark
is not True:
252 self.prerequisiteInputs.discard(
"dark")
253 if config.doFlat
is not True:
254 self.prerequisiteInputs.discard(
"flat")
255 if config.doAttachTransmissionCurve
is not True:
256 self.prerequisiteInputs.discard(
"opticsTransmission")
257 self.prerequisiteInputs.discard(
"filterTransmission")
258 self.prerequisiteInputs.discard(
"sensorTransmission")
259 self.prerequisiteInputs.discard(
"atmosphereTransmission")
260 if config.doUseOpticsTransmission
is not True:
261 self.prerequisiteInputs.discard(
"opticsTransmission")
262 if config.doUseFilterTransmission
is not True:
263 self.prerequisiteInputs.discard(
"filterTransmission")
264 if config.doUseSensorTransmission
is not True:
265 self.prerequisiteInputs.discard(
"sensorTransmission")
266 if config.doUseAtmosphereTransmission
is not True:
267 self.prerequisiteInputs.discard(
"atmosphereTransmission")
268 if config.doIlluminationCorrection
is not True:
269 self.prerequisiteInputs.discard(
"illumMaskedImage")
271 if config.doWrite
is not True:
272 self.outputs.discard(
"outputExposure")
273 self.outputs.discard(
"preInterpExposure")
274 self.outputs.discard(
"outputFlattenedThumbnail")
275 self.outputs.discard(
"outputOssThumbnail")
276 if config.doSaveInterpPixels
is not True:
277 self.outputs.discard(
"preInterpExposure")
278 if config.qa.doThumbnailOss
is not True:
279 self.outputs.discard(
"outputOssThumbnail")
280 if config.qa.doThumbnailFlattened
is not True:
281 self.outputs.discard(
"outputFlattenedThumbnail")
285 pipelineConnections=IsrTaskConnections):
286 """Configuration parameters for IsrTask.
288 Items are grouped in the order in which they are executed by the task.
290 datasetType = pexConfig.Field(
292 doc=
"Dataset type for input data; users will typically leave this alone, "
293 "but camera-specific ISR tasks will override it",
297 fallbackFilterName = pexConfig.Field(
299 doc=
"Fallback default filter name for calibrations.",
302 useFallbackDate = pexConfig.Field(
304 doc=
"Pass observation date when using fallback filter.",
307 expectWcs = pexConfig.Field(
310 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
312 fwhm = pexConfig.Field(
314 doc=
"FWHM of PSF in arcseconds.",
317 qa = pexConfig.ConfigField(
319 doc=
"QA related configuration options.",
323 doConvertIntToFloat = pexConfig.Field(
325 doc=
"Convert integer raw images to floating point values?",
330 doSaturation = pexConfig.Field(
332 doc=
"Mask saturated pixels? NB: this is totally independent of the"
333 " interpolation option - this is ONLY setting the bits in the mask."
334 " To have them interpolated make sure doSaturationInterpolation=True",
337 saturatedMaskName = pexConfig.Field(
339 doc=
"Name of mask plane to use in saturation detection and interpolation",
342 saturation = pexConfig.Field(
344 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
345 default=float(
"NaN"),
347 growSaturationFootprintSize = pexConfig.Field(
349 doc=
"Number of pixels by which to grow the saturation footprints",
354 doSuspect = pexConfig.Field(
356 doc=
"Mask suspect pixels?",
359 suspectMaskName = pexConfig.Field(
361 doc=
"Name of mask plane to use for suspect pixels",
364 numEdgeSuspect = pexConfig.Field(
366 doc=
"Number of edge pixels to be flagged as untrustworthy.",
369 edgeMaskLevel = pexConfig.ChoiceField(
371 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
374 'DETECTOR':
'Mask only the edges of the full detector.',
375 'AMP':
'Mask edges of each amplifier.',
380 doSetBadRegions = pexConfig.Field(
382 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
385 badStatistic = pexConfig.ChoiceField(
387 doc=
"How to estimate the average value for BAD regions.",
390 "MEANCLIP":
"Correct using the (clipped) mean of good data",
391 "MEDIAN":
"Correct using the median of the good data",
396 doOverscan = pexConfig.Field(
398 doc=
"Do overscan subtraction?",
401 overscan = pexConfig.ConfigurableField(
402 target=OverscanCorrectionTask,
403 doc=
"Overscan subtraction task for image segments.",
406 overscanFitType = pexConfig.ChoiceField(
408 doc=
"The method for fitting the overscan bias level.",
411 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
412 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
413 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
414 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
415 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
416 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
417 "MEAN":
"Correct using the mean of the overscan region",
418 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
419 "MEDIAN":
"Correct using the median of the overscan region",
420 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
422 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
423 " This option will no longer be used, and will be removed after v20.")
425 overscanOrder = pexConfig.Field(
427 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
428 "or number of spline knots if overscan fit type is a spline."),
430 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
431 " This option will no longer be used, and will be removed after v20.")
433 overscanNumSigmaClip = pexConfig.Field(
435 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
437 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
438 " This option will no longer be used, and will be removed after v20.")
440 overscanIsInt = pexConfig.Field(
442 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
443 " and overscan.FitType=MEDIAN_PER_ROW.",
445 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
446 " This option will no longer be used, and will be removed after v20.")
449 overscanNumLeadingColumnsToSkip = pexConfig.Field(
451 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
454 overscanNumTrailingColumnsToSkip = pexConfig.Field(
456 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
459 overscanMaxDev = pexConfig.Field(
461 doc=
"Maximum deviation from the median for overscan",
462 default=1000.0, check=
lambda x: x > 0
464 overscanBiasJump = pexConfig.Field(
466 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
469 overscanBiasJumpKeyword = pexConfig.Field(
471 doc=
"Header keyword containing information about devices.",
472 default=
"NO_SUCH_KEY",
474 overscanBiasJumpDevices = pexConfig.ListField(
476 doc=
"List of devices that need piecewise overscan correction.",
479 overscanBiasJumpLocation = pexConfig.Field(
481 doc=
"Location of bias jump along y-axis.",
486 doAssembleCcd = pexConfig.Field(
489 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
491 assembleCcd = pexConfig.ConfigurableField(
492 target=AssembleCcdTask,
493 doc=
"CCD assembly task",
497 doAssembleIsrExposures = pexConfig.Field(
500 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
502 doTrimToMatchCalib = pexConfig.Field(
505 doc=
"Trim raw data to match calibration bounding boxes?"
509 doBias = pexConfig.Field(
511 doc=
"Apply bias frame correction?",
514 biasDataProductName = pexConfig.Field(
516 doc=
"Name of the bias data product",
519 doBiasBeforeOverscan = pexConfig.Field(
521 doc=
"Reverse order of overscan and bias correction.",
526 doVariance = pexConfig.Field(
528 doc=
"Calculate variance?",
531 gain = pexConfig.Field(
533 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
534 default=float(
"NaN"),
536 readNoise = pexConfig.Field(
538 doc=
"The read noise to use if no Detector is present in the Exposure",
541 doEmpiricalReadNoise = pexConfig.Field(
544 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
548 doLinearize = pexConfig.Field(
550 doc=
"Correct for nonlinearity of the detector's response?",
555 doCrosstalk = pexConfig.Field(
557 doc=
"Apply intra-CCD crosstalk correction?",
560 doCrosstalkBeforeAssemble = pexConfig.Field(
562 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
565 crosstalk = pexConfig.ConfigurableField(
566 target=CrosstalkTask,
567 doc=
"Intra-CCD crosstalk correction",
571 doDefect = pexConfig.Field(
573 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
576 doNanMasking = pexConfig.Field(
578 doc=
"Mask NAN pixels?",
581 doWidenSaturationTrails = pexConfig.Field(
583 doc=
"Widen bleed trails based on their width?",
588 doBrighterFatter = pexConfig.Field(
591 doc=
"Apply the brighter fatter correction"
593 brighterFatterLevel = pexConfig.ChoiceField(
596 doc=
"The level at which to correct for brighter-fatter.",
598 "AMP":
"Every amplifier treated separately.",
599 "DETECTOR":
"One kernel per detector",
602 brighterFatterMaxIter = pexConfig.Field(
605 doc=
"Maximum number of iterations for the brighter fatter correction"
607 brighterFatterThreshold = pexConfig.Field(
610 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
611 " absolute value of the difference between the current corrected image and the one"
612 " from the previous iteration summed over all the pixels."
614 brighterFatterApplyGain = pexConfig.Field(
617 doc=
"Should the gain be applied when applying the brighter fatter correction?"
619 brighterFatterMaskGrowSize = pexConfig.Field(
622 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
623 " when brighter-fatter correction is applied."
627 doDark = pexConfig.Field(
629 doc=
"Apply dark frame correction?",
632 darkDataProductName = pexConfig.Field(
634 doc=
"Name of the dark data product",
639 doStrayLight = pexConfig.Field(
641 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
644 strayLight = pexConfig.ConfigurableField(
645 target=StrayLightTask,
646 doc=
"y-band stray light correction"
650 doFlat = pexConfig.Field(
652 doc=
"Apply flat field correction?",
655 flatDataProductName = pexConfig.Field(
657 doc=
"Name of the flat data product",
660 flatScalingType = pexConfig.ChoiceField(
662 doc=
"The method for scaling the flat on the fly.",
665 "USER":
"Scale by flatUserScale",
666 "MEAN":
"Scale by the inverse of the mean",
667 "MEDIAN":
"Scale by the inverse of the median",
670 flatUserScale = pexConfig.Field(
672 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
675 doTweakFlat = pexConfig.Field(
677 doc=
"Tweak flats to match observed amplifier ratios?",
682 doApplyGains = pexConfig.Field(
684 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
687 normalizeGains = pexConfig.Field(
689 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
694 doFringe = pexConfig.Field(
696 doc=
"Apply fringe correction?",
699 fringe = pexConfig.ConfigurableField(
701 doc=
"Fringe subtraction task",
703 fringeAfterFlat = pexConfig.Field(
705 doc=
"Do fringe subtraction after flat-fielding?",
710 doMeasureBackground = pexConfig.Field(
712 doc=
"Measure the background level on the reduced image?",
717 doCameraSpecificMasking = pexConfig.Field(
719 doc=
"Mask camera-specific bad regions?",
722 masking = pexConfig.ConfigurableField(
729 doInterpolate = pexConfig.Field(
731 doc=
"Interpolate masked pixels?",
734 doSaturationInterpolation = pexConfig.Field(
736 doc=
"Perform interpolation over pixels masked as saturated?"
737 " NB: This is independent of doSaturation; if that is False this plane"
738 " will likely be blank, resulting in a no-op here.",
741 doNanInterpolation = pexConfig.Field(
743 doc=
"Perform interpolation over pixels masked as NaN?"
744 " NB: This is independent of doNanMasking; if that is False this plane"
745 " will likely be blank, resulting in a no-op here.",
748 doNanInterpAfterFlat = pexConfig.Field(
750 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
751 "also have to interpolate them before flat-fielding."),
754 maskListToInterpolate = pexConfig.ListField(
756 doc=
"List of mask planes that should be interpolated.",
757 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
759 doSaveInterpPixels = pexConfig.Field(
761 doc=
"Save a copy of the pre-interpolated pixel values?",
766 fluxMag0T1 = pexConfig.DictField(
769 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
770 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
773 defaultFluxMag0T1 = pexConfig.Field(
775 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
776 default=pow(10.0, 0.4*28.0)
780 doVignette = pexConfig.Field(
782 doc=
"Apply vignetting parameters?",
785 vignette = pexConfig.ConfigurableField(
787 doc=
"Vignetting task.",
791 doAttachTransmissionCurve = pexConfig.Field(
794 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
796 doUseOpticsTransmission = pexConfig.Field(
799 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
801 doUseFilterTransmission = pexConfig.Field(
804 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
806 doUseSensorTransmission = pexConfig.Field(
809 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
811 doUseAtmosphereTransmission = pexConfig.Field(
814 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
818 doIlluminationCorrection = pexConfig.Field(
821 doc=
"Perform illumination correction?"
823 illuminationCorrectionDataProductName = pexConfig.Field(
825 doc=
"Name of the illumination correction data product.",
828 illumScale = pexConfig.Field(
830 doc=
"Scale factor for the illumination correction.",
833 illumFilters = pexConfig.ListField(
836 doc=
"Only perform illumination correction for these filters."
840 doWrite = pexConfig.Field(
842 doc=
"Persist postISRCCD?",
849 raise ValueError(
"You may not specify both doFlat and doApplyGains")
851 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
853 self.config.maskListToInterpolate.append(
"SAT")
855 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
858 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
859 """Apply common instrument signature correction algorithms to a raw frame.
861 The process for correcting imaging data is very similar from
862 camera to camera. This task provides a vanilla implementation of
863 doing these corrections, including the ability to turn certain
864 corrections off if they are not needed. The inputs to the primary
865 method, `run()`, are a raw exposure to be corrected and the
866 calibration data products. The raw input is a single chip sized
867 mosaic of all amps including overscans and other non-science
868 pixels. The method `runDataRef()` identifies and defines the
869 calibration data products, and is intended for use by a
870 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
871 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
872 subclassed for different camera, although the most camera specific
873 methods have been split into subtasks that can be redirected
876 The __init__ method sets up the subtasks for ISR processing, using
877 the defaults from `lsst.ip.isr`.
882 Positional arguments passed to the Task constructor. None used at this time.
883 kwargs : `dict`, optional
884 Keyword arguments passed on to the Task constructor. None used at this time.
886 ConfigClass = IsrTaskConfig
891 self.makeSubtask(
"assembleCcd")
892 self.makeSubtask(
"crosstalk")
893 self.makeSubtask(
"strayLight")
894 self.makeSubtask(
"fringe")
895 self.makeSubtask(
"masking")
896 self.makeSubtask(
"overscan")
897 self.makeSubtask(
"vignette")
900 inputs = butlerQC.get(inputRefs)
903 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
904 except Exception
as e:
905 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
908 inputs[
'isGen3'] =
True
910 detector = inputs[
'ccdExposure'].getDetector()
912 if self.config.doCrosstalk
is True:
915 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
916 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
917 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
919 coeffVector = (self.config.crosstalk.crosstalkValues
920 if self.config.crosstalk.useConfigCoefficients
else None)
921 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
922 inputs[
'crosstalk'] = crosstalkCalib
923 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
924 if 'crosstalkSources' not in inputs:
925 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
928 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
930 linearizer.fromYaml(inputs[
'linearizer'])
934 inputs[
'linearizer'] = linearizer
936 if self.config.doDefect
is True:
937 if "defects" in inputs
and inputs[
'defects']
is not None:
940 if not isinstance(inputs[
"defects"], Defects):
941 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
945 if self.config.doBrighterFatter:
946 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
947 if brighterFatterKernel
is None:
948 brighterFatterKernel = inputs.get(
'bfKernel',
None)
950 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
951 detId = detector.getId()
952 inputs[
'bfGains'] = brighterFatterKernel.gain
955 if self.config.brighterFatterLevel ==
'DETECTOR':
956 if brighterFatterKernel.detectorKernel:
957 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
958 elif brighterFatterKernel.detectorKernelFromAmpKernels:
959 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
961 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
964 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
966 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
967 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
968 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
970 assembler=self.assembleCcd
971 if self.config.doAssembleIsrExposures
else None)
973 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
975 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
976 if 'strayLightData' not in inputs:
977 inputs[
'strayLightData'] =
None
979 outputs = self.
run(**inputs)
980 butlerQC.put(outputs, outputRefs)
983 """!Retrieve necessary frames for instrument signature removal.
985 Pre-fetching all required ISR data products limits the IO
986 required by the ISR. Any conflict between the calibration data
987 available and that needed for ISR is also detected prior to
988 doing processing, allowing it to fail quickly.
992 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
993 Butler reference of the detector data to be processed
994 rawExposure : `afw.image.Exposure`
995 The raw exposure that will later be corrected with the
996 retrieved calibration data; should not be modified in this
1001 result : `lsst.pipe.base.Struct`
1002 Result struct with components (which may be `None`):
1003 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1004 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1005 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1006 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1007 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1008 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1009 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
1010 - ``fringes``: `lsst.pipe.base.Struct` with components:
1011 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1012 - ``seed``: random seed derived from the ccdExposureId for random
1013 number generator (`uint32`).
1014 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1015 A ``TransmissionCurve`` that represents the throughput of the optics,
1016 to be evaluated in focal-plane coordinates.
1017 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1018 A ``TransmissionCurve`` that represents the throughput of the filter
1019 itself, to be evaluated in focal-plane coordinates.
1020 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1021 A ``TransmissionCurve`` that represents the throughput of the sensor
1022 itself, to be evaluated in post-assembly trimmed detector coordinates.
1023 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1024 A ``TransmissionCurve`` that represents the throughput of the
1025 atmosphere, assumed to be spatially constant.
1026 - ``strayLightData`` : `object`
1027 An opaque object containing calibration information for
1028 stray-light correction. If `None`, no correction will be
1030 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1034 NotImplementedError :
1035 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1038 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1039 dateObs = dateObs.toPython().isoformat()
1040 except RuntimeError:
1041 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1044 ccd = rawExposure.getDetector()
1045 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
1046 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1047 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
1048 if self.config.doBias
else None)
1050 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1052 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1053 linearizer.log = self.log
1054 if isinstance(linearizer, numpy.ndarray):
1057 crosstalkCalib =
None
1058 if self.config.doCrosstalk:
1060 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1062 coeffVector = (self.config.crosstalk.crosstalkValues
1063 if self.config.crosstalk.useConfigCoefficients
else None)
1064 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1065 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1066 if self.config.doCrosstalk
else None)
1068 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
1069 if self.config.doDark
else None)
1070 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
1072 if self.config.doFlat
else None)
1074 brighterFatterKernel =
None
1075 brighterFatterGains =
None
1076 if self.config.doBrighterFatter
is True:
1081 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1082 brighterFatterGains = brighterFatterKernel.gain
1083 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1086 brighterFatterKernel = dataRef.get(
"bfKernel")
1087 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
1089 brighterFatterKernel =
None
1090 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1093 if self.config.brighterFatterLevel ==
'DETECTOR':
1094 if brighterFatterKernel.detectorKernel:
1095 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1096 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1097 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1099 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1102 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1104 defectList = (dataRef.get(
"defects")
1105 if self.config.doDefect
else None)
1106 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1107 if self.config.doAssembleIsrExposures
else None)
1108 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1109 else pipeBase.Struct(fringes=
None))
1111 if self.config.doAttachTransmissionCurve:
1112 opticsTransmission = (dataRef.get(
"transmission_optics")
1113 if self.config.doUseOpticsTransmission
else None)
1114 filterTransmission = (dataRef.get(
"transmission_filter")
1115 if self.config.doUseFilterTransmission
else None)
1116 sensorTransmission = (dataRef.get(
"transmission_sensor")
1117 if self.config.doUseSensorTransmission
else None)
1118 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1119 if self.config.doUseAtmosphereTransmission
else None)
1121 opticsTransmission =
None
1122 filterTransmission =
None
1123 sensorTransmission =
None
1124 atmosphereTransmission =
None
1126 if self.config.doStrayLight:
1127 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1129 strayLightData =
None
1132 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1133 if (self.config.doIlluminationCorrection
1134 and filterName
in self.config.illumFilters)
1138 return pipeBase.Struct(bias=biasExposure,
1139 linearizer=linearizer,
1140 crosstalk=crosstalkCalib,
1141 crosstalkSources=crosstalkSources,
1144 bfKernel=brighterFatterKernel,
1145 bfGains=brighterFatterGains,
1147 fringes=fringeStruct,
1148 opticsTransmission=opticsTransmission,
1149 filterTransmission=filterTransmission,
1150 sensorTransmission=sensorTransmission,
1151 atmosphereTransmission=atmosphereTransmission,
1152 strayLightData=strayLightData,
1153 illumMaskedImage=illumMaskedImage
1156 @pipeBase.timeMethod
1157 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1158 crosstalk=None, crosstalkSources=None,
1159 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1160 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1161 sensorTransmission=
None, atmosphereTransmission=
None,
1162 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1165 """!Perform instrument signature removal on an exposure.
1167 Steps included in the ISR processing, in order performed, are:
1168 - saturation and suspect pixel masking
1169 - overscan subtraction
1170 - CCD assembly of individual amplifiers
1172 - variance image construction
1173 - linearization of non-linear response
1175 - brighter-fatter correction
1178 - stray light subtraction
1180 - masking of known defects and camera specific features
1181 - vignette calculation
1182 - appending transmission curve and distortion model
1186 ccdExposure : `lsst.afw.image.Exposure`
1187 The raw exposure that is to be run through ISR. The
1188 exposure is modified by this method.
1189 camera : `lsst.afw.cameraGeom.Camera`, optional
1190 The camera geometry for this exposure. Required if ``isGen3`` is
1191 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1192 ``flat`` does not have an associated detector.
1193 bias : `lsst.afw.image.Exposure`, optional
1194 Bias calibration frame.
1195 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1196 Functor for linearization.
1197 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1198 Calibration for crosstalk.
1199 crosstalkSources : `list`, optional
1200 List of possible crosstalk sources.
1201 dark : `lsst.afw.image.Exposure`, optional
1202 Dark calibration frame.
1203 flat : `lsst.afw.image.Exposure`, optional
1204 Flat calibration frame.
1205 bfKernel : `numpy.ndarray`, optional
1206 Brighter-fatter kernel.
1207 bfGains : `dict` of `float`, optional
1208 Gains used to override the detector's nominal gains for the
1209 brighter-fatter correction. A dict keyed by amplifier name for
1210 the detector in question.
1211 defects : `lsst.meas.algorithms.Defects`, optional
1213 fringes : `lsst.pipe.base.Struct`, optional
1214 Struct containing the fringe correction data, with
1216 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1217 - ``seed``: random seed derived from the ccdExposureId for random
1218 number generator (`uint32`)
1219 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1220 A ``TransmissionCurve`` that represents the throughput of the optics,
1221 to be evaluated in focal-plane coordinates.
1222 filterTransmission : `lsst.afw.image.TransmissionCurve`
1223 A ``TransmissionCurve`` that represents the throughput of the filter
1224 itself, to be evaluated in focal-plane coordinates.
1225 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1226 A ``TransmissionCurve`` that represents the throughput of the sensor
1227 itself, to be evaluated in post-assembly trimmed detector coordinates.
1228 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1229 A ``TransmissionCurve`` that represents the throughput of the
1230 atmosphere, assumed to be spatially constant.
1231 detectorNum : `int`, optional
1232 The integer number for the detector to process.
1233 isGen3 : bool, optional
1234 Flag this call to run() as using the Gen3 butler environment.
1235 strayLightData : `object`, optional
1236 Opaque object containing calibration information for stray-light
1237 correction. If `None`, no correction will be performed.
1238 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1239 Illumination correction image.
1243 result : `lsst.pipe.base.Struct`
1244 Result struct with component:
1245 - ``exposure`` : `afw.image.Exposure`
1246 The fully ISR corrected exposure.
1247 - ``outputExposure`` : `afw.image.Exposure`
1248 An alias for `exposure`
1249 - ``ossThumb`` : `numpy.ndarray`
1250 Thumbnail image of the exposure after overscan subtraction.
1251 - ``flattenedThumb`` : `numpy.ndarray`
1252 Thumbnail image of the exposure after flat-field correction.
1257 Raised if a configuration option is set to True, but the
1258 required calibration data has not been specified.
1262 The current processed exposure can be viewed by setting the
1263 appropriate lsstDebug entries in the `debug.display`
1264 dictionary. The names of these entries correspond to some of
1265 the IsrTaskConfig Boolean options, with the value denoting the
1266 frame to use. The exposure is shown inside the matching
1267 option check and after the processing of that step has
1268 finished. The steps with debug points are:
1279 In addition, setting the "postISRCCD" entry displays the
1280 exposure after all ISR processing has finished.
1288 if detectorNum
is None:
1289 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1291 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1296 if isinstance(ccdExposure, ButlerDataRef):
1299 ccd = ccdExposure.getDetector()
1300 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1303 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1304 ccd = [
FakeAmp(ccdExposure, self.config)]
1307 if self.config.doBias
and bias
is None:
1308 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1310 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1311 if self.config.doBrighterFatter
and bfKernel
is None:
1312 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1313 if self.config.doDark
and dark
is None:
1314 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1315 if self.config.doFlat
and flat
is None:
1316 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1317 if self.config.doDefect
and defects
is None:
1318 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1319 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1320 and fringes.fringes
is None):
1325 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1326 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1327 and illumMaskedImage
is None):
1328 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1331 if self.config.doConvertIntToFloat:
1332 self.log.info(
"Converting exposure to floating point values.")
1335 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1336 self.log.info(
"Applying bias correction.")
1337 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1338 trimToFit=self.config.doTrimToMatchCalib)
1345 if ccdExposure.getBBox().contains(amp.getBBox()):
1349 if self.config.doOverscan
and not badAmp:
1352 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1353 if overscanResults
is not None and \
1354 self.config.qa
is not None and self.config.qa.saveStats
is True:
1355 if isinstance(overscanResults.overscanFit, float):
1356 qaMedian = overscanResults.overscanFit
1357 qaStdev = float(
"NaN")
1359 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1360 afwMath.MEDIAN | afwMath.STDEVCLIP)
1361 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1362 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1364 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1365 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1366 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1367 amp.getName(), qaMedian, qaStdev)
1368 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1371 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1372 overscanResults =
None
1374 overscans.append(overscanResults
if overscanResults
is not None else None)
1376 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1378 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1379 self.log.info(
"Applying crosstalk correction.")
1380 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1381 crosstalkSources=crosstalkSources)
1382 self.
debugView(ccdExposure,
"doCrosstalk")
1384 if self.config.doAssembleCcd:
1385 self.log.info(
"Assembling CCD from amplifiers.")
1386 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1388 if self.config.expectWcs
and not ccdExposure.getWcs():
1389 self.log.warn(
"No WCS found in input exposure.")
1390 self.
debugView(ccdExposure,
"doAssembleCcd")
1393 if self.config.qa.doThumbnailOss:
1394 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1396 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1397 self.log.info(
"Applying bias correction.")
1398 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1399 trimToFit=self.config.doTrimToMatchCalib)
1402 if self.config.doVariance:
1403 for amp, overscanResults
in zip(ccd, overscans):
1404 if ccdExposure.getBBox().contains(amp.getBBox()):
1405 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1406 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1407 if overscanResults
is not None:
1409 overscanImage=overscanResults.overscanImage)
1413 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1414 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1415 afwMath.MEDIAN | afwMath.STDEVCLIP)
1416 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1417 qaStats.getValue(afwMath.MEDIAN))
1418 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1419 qaStats.getValue(afwMath.STDEVCLIP))
1420 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1421 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1422 qaStats.getValue(afwMath.STDEVCLIP))
1425 self.log.info(
"Applying linearizer.")
1426 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1427 detector=ccd, log=self.log)
1429 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1430 self.log.info(
"Applying crosstalk correction.")
1431 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1432 crosstalkSources=crosstalkSources, isTrimmed=
True)
1433 self.
debugView(ccdExposure,
"doCrosstalk")
1437 if self.config.doDefect:
1438 self.log.info(
"Masking defects.")
1441 if self.config.numEdgeSuspect > 0:
1442 self.log.info(
"Masking edges as SUSPECT.")
1443 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1444 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1446 if self.config.doNanMasking:
1447 self.log.info(
"Masking NAN value pixels.")
1450 if self.config.doWidenSaturationTrails:
1451 self.log.info(
"Widening saturation trails.")
1452 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1454 if self.config.doCameraSpecificMasking:
1455 self.log.info(
"Masking regions for camera specific reasons.")
1456 self.masking.
run(ccdExposure)
1458 if self.config.doBrighterFatter:
1467 interpExp = ccdExposure.clone()
1469 isrFunctions.interpolateFromMask(
1470 maskedImage=interpExp.getMaskedImage(),
1471 fwhm=self.config.fwhm,
1472 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1473 maskNameList=self.config.maskListToInterpolate
1475 bfExp = interpExp.clone()
1477 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1478 type(bfKernel), type(bfGains))
1479 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1480 self.config.brighterFatterMaxIter,
1481 self.config.brighterFatterThreshold,
1482 self.config.brighterFatterApplyGain,
1484 if bfResults[1] == self.config.brighterFatterMaxIter:
1485 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1488 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1490 image = ccdExposure.getMaskedImage().getImage()
1491 bfCorr = bfExp.getMaskedImage().getImage()
1492 bfCorr -= interpExp.getMaskedImage().getImage()
1501 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1502 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1505 if self.config.brighterFatterMaskGrowSize > 0:
1506 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1507 for maskPlane
in self.config.maskListToInterpolate:
1508 isrFunctions.growMasks(ccdExposure.getMask(),
1509 radius=self.config.brighterFatterMaskGrowSize,
1510 maskNameList=maskPlane,
1511 maskValue=maskPlane)
1513 self.
debugView(ccdExposure,
"doBrighterFatter")
1515 if self.config.doDark:
1516 self.log.info(
"Applying dark correction.")
1520 if self.config.doFringe
and not self.config.fringeAfterFlat:
1521 self.log.info(
"Applying fringe correction before flat.")
1522 self.fringe.
run(ccdExposure, **fringes.getDict())
1525 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1526 self.log.info(
"Checking strayLight correction.")
1527 self.strayLight.
run(ccdExposure, strayLightData)
1528 self.
debugView(ccdExposure,
"doStrayLight")
1530 if self.config.doFlat:
1531 self.log.info(
"Applying flat correction.")
1535 if self.config.doApplyGains:
1536 self.log.info(
"Applying gain correction instead of flat.")
1537 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1539 if self.config.doFringe
and self.config.fringeAfterFlat:
1540 self.log.info(
"Applying fringe correction after flat.")
1541 self.fringe.
run(ccdExposure, **fringes.getDict())
1543 if self.config.doVignette:
1544 self.log.info(
"Constructing Vignette polygon.")
1547 if self.config.vignette.doWriteVignettePolygon:
1550 if self.config.doAttachTransmissionCurve:
1551 self.log.info(
"Adding transmission curves.")
1552 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1553 filterTransmission=filterTransmission,
1554 sensorTransmission=sensorTransmission,
1555 atmosphereTransmission=atmosphereTransmission)
1557 flattenedThumb =
None
1558 if self.config.qa.doThumbnailFlattened:
1559 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1561 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1562 self.log.info(
"Performing illumination correction.")
1563 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1564 illumMaskedImage, illumScale=self.config.illumScale,
1565 trimToFit=self.config.doTrimToMatchCalib)
1568 if self.config.doSaveInterpPixels:
1569 preInterpExp = ccdExposure.clone()
1584 if self.config.doSetBadRegions:
1585 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1586 if badPixelCount > 0:
1587 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1589 if self.config.doInterpolate:
1590 self.log.info(
"Interpolating masked pixels.")
1591 isrFunctions.interpolateFromMask(
1592 maskedImage=ccdExposure.getMaskedImage(),
1593 fwhm=self.config.fwhm,
1594 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1595 maskNameList=list(self.config.maskListToInterpolate)
1600 if self.config.doMeasureBackground:
1601 self.log.info(
"Measuring background level.")
1604 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1606 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1607 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1608 afwMath.MEDIAN | afwMath.STDEVCLIP)
1609 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1610 qaStats.getValue(afwMath.MEDIAN))
1611 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1612 qaStats.getValue(afwMath.STDEVCLIP))
1613 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1614 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1615 qaStats.getValue(afwMath.STDEVCLIP))
1617 self.
debugView(ccdExposure,
"postISRCCD")
1619 return pipeBase.Struct(
1620 exposure=ccdExposure,
1622 flattenedThumb=flattenedThumb,
1624 preInterpolatedExposure=preInterpExp,
1625 outputExposure=ccdExposure,
1626 outputOssThumbnail=ossThumb,
1627 outputFlattenedThumbnail=flattenedThumb,
1630 @pipeBase.timeMethod
1632 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1634 This method contains the `CmdLineTask` interface to the ISR
1635 processing. All IO is handled here, freeing the `run()` method
1636 to manage only pixel-level calculations. The steps performed
1638 - Read in necessary detrending/isr/calibration data.
1639 - Process raw exposure in `run()`.
1640 - Persist the ISR-corrected exposure as "postISRCCD" if
1641 config.doWrite=True.
1645 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1646 DataRef of the detector data to be processed
1650 result : `lsst.pipe.base.Struct`
1651 Result struct with component:
1652 - ``exposure`` : `afw.image.Exposure`
1653 The fully ISR corrected exposure.
1658 Raised if a configuration option is set to True, but the
1659 required calibration data does not exist.
1662 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1664 ccdExposure = sensorRef.get(self.config.datasetType)
1666 camera = sensorRef.get(
"camera")
1667 isrData = self.
readIsrData(sensorRef, ccdExposure)
1669 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1671 if self.config.doWrite:
1672 sensorRef.put(result.exposure,
"postISRCCD")
1673 if result.preInterpolatedExposure
is not None:
1674 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1675 if result.ossThumb
is not None:
1676 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1677 if result.flattenedThumb
is not None:
1678 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1683 """!Retrieve a calibration dataset for removing instrument signature.
1688 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1689 DataRef of the detector data to find calibration datasets
1692 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1693 dateObs : `str`, optional
1694 Date of the observation. Used to correct butler failures
1695 when using fallback filters.
1697 If True, disable butler proxies to enable error handling
1698 within this routine.
1702 exposure : `lsst.afw.image.Exposure`
1703 Requested calibration frame.
1708 Raised if no matching calibration frame can be found.
1711 exp = dataRef.get(datasetType, immediate=immediate)
1712 except Exception
as exc1:
1713 if not self.config.fallbackFilterName:
1714 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1716 if self.config.useFallbackDate
and dateObs:
1717 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1718 dateObs=dateObs, immediate=immediate)
1720 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1721 except Exception
as exc2:
1722 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1723 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1724 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1726 if self.config.doAssembleIsrExposures:
1727 exp = self.assembleCcd.assembleCcd(exp)
1731 """Ensure that the data returned by Butler is a fully constructed exposure.
1733 ISR requires exposure-level image data for historical reasons, so if we did
1734 not recieve that from Butler, construct it from what we have, modifying the
1739 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1740 `lsst.afw.image.ImageF`
1741 The input data structure obtained from Butler.
1742 camera : `lsst.afw.cameraGeom.camera`
1743 The camera associated with the image. Used to find the appropriate
1746 The detector this exposure should match.
1750 inputExp : `lsst.afw.image.Exposure`
1751 The re-constructed exposure, with appropriate detector parameters.
1756 Raised if the input data cannot be used to construct an exposure.
1758 if isinstance(inputExp, afwImage.DecoratedImageU):
1759 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1760 elif isinstance(inputExp, afwImage.ImageF):
1761 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1762 elif isinstance(inputExp, afwImage.MaskedImageF):
1763 inputExp = afwImage.makeExposure(inputExp)
1764 elif isinstance(inputExp, afwImage.Exposure):
1766 elif inputExp
is None:
1770 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1773 if inputExp.getDetector()
is None:
1774 inputExp.setDetector(camera[detectorNum])
1779 """Convert exposure image from uint16 to float.
1781 If the exposure does not need to be converted, the input is
1782 immediately returned. For exposures that are converted to use
1783 floating point pixels, the variance is set to unity and the
1788 exposure : `lsst.afw.image.Exposure`
1789 The raw exposure to be converted.
1793 newexposure : `lsst.afw.image.Exposure`
1794 The input ``exposure``, converted to floating point pixels.
1799 Raised if the exposure type cannot be converted to float.
1802 if isinstance(exposure, afwImage.ExposureF):
1804 self.log.debug(
"Exposure already of type float.")
1806 if not hasattr(exposure,
"convertF"):
1807 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1809 newexposure = exposure.convertF()
1810 newexposure.variance[:] = 1
1811 newexposure.mask[:] = 0x0
1816 """Identify bad amplifiers, saturated and suspect pixels.
1820 ccdExposure : `lsst.afw.image.Exposure`
1821 Input exposure to be masked.
1822 amp : `lsst.afw.table.AmpInfoCatalog`
1823 Catalog of parameters defining the amplifier on this
1825 defects : `lsst.meas.algorithms.Defects`
1826 List of defects. Used to determine if the entire
1832 If this is true, the entire amplifier area is covered by
1833 defects and unusable.
1836 maskedImage = ccdExposure.getMaskedImage()
1842 if defects
is not None:
1843 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1848 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1850 maskView = dataView.getMask()
1851 maskView |= maskView.getPlaneBitMask(
"BAD")
1858 if self.config.doSaturation
and not badAmp:
1859 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1860 if self.config.doSuspect
and not badAmp:
1861 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1862 if math.isfinite(self.config.saturation):
1863 limits.update({self.config.saturatedMaskName: self.config.saturation})
1865 for maskName, maskThreshold
in limits.items():
1866 if not math.isnan(maskThreshold):
1867 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1868 isrFunctions.makeThresholdMask(
1869 maskedImage=dataView,
1870 threshold=maskThreshold,
1876 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1878 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1879 self.config.suspectMaskName])
1880 if numpy.all(maskView.getArray() & maskVal > 0):
1882 maskView |= maskView.getPlaneBitMask(
"BAD")
1887 """Apply overscan correction in place.
1889 This method does initial pixel rejection of the overscan
1890 region. The overscan can also be optionally segmented to
1891 allow for discontinuous overscan responses to be fit
1892 separately. The actual overscan subtraction is performed by
1893 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1894 which is called here after the amplifier is preprocessed.
1898 ccdExposure : `lsst.afw.image.Exposure`
1899 Exposure to have overscan correction performed.
1900 amp : `lsst.afw.table.AmpInfoCatalog`
1901 The amplifier to consider while correcting the overscan.
1905 overscanResults : `lsst.pipe.base.Struct`
1906 Result struct with components:
1907 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1908 Value or fit subtracted from the amplifier image data.
1909 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1910 Value or fit subtracted from the overscan image data.
1911 - ``overscanImage`` : `lsst.afw.image.Image`
1912 Image of the overscan region with the overscan
1913 correction applied. This quantity is used to estimate
1914 the amplifier read noise empirically.
1919 Raised if the ``amp`` does not contain raw pixel information.
1923 lsst.ip.isr.isrFunctions.overscanCorrection
1925 if amp.getRawHorizontalOverscanBBox().isEmpty():
1926 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1929 statControl = afwMath.StatisticsControl()
1930 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1933 dataBBox = amp.getRawDataBBox()
1934 oscanBBox = amp.getRawHorizontalOverscanBBox()
1938 prescanBBox = amp.getRawPrescanBBox()
1939 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1940 dx0 += self.config.overscanNumLeadingColumnsToSkip
1941 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1943 dx0 += self.config.overscanNumTrailingColumnsToSkip
1944 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1950 if ((self.config.overscanBiasJump
1951 and self.config.overscanBiasJumpLocation)
1952 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1953 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
1954 self.config.overscanBiasJumpDevices)):
1955 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1956 yLower = self.config.overscanBiasJumpLocation
1957 yUpper = dataBBox.getHeight() - yLower
1959 yUpper = self.config.overscanBiasJumpLocation
1960 yLower = dataBBox.getHeight() - yUpper
1978 oscanBBox.getHeight())))
1981 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1982 ampImage = ccdExposure.maskedImage[imageBBox]
1983 overscanImage = ccdExposure.maskedImage[overscanBBox]
1985 overscanArray = overscanImage.image.array
1986 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1987 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1988 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1990 statControl = afwMath.StatisticsControl()
1991 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1993 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage)
1996 levelStat = afwMath.MEDIAN
1997 sigmaStat = afwMath.STDEVCLIP
1999 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2000 self.config.qa.flatness.nIter)
2001 metadata = ccdExposure.getMetadata()
2002 ampNum = amp.getName()
2004 if isinstance(overscanResults.overscanFit, float):
2005 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2006 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2008 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2009 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2010 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2012 return overscanResults
2015 """Set the variance plane using the amplifier gain and read noise
2017 The read noise is calculated from the ``overscanImage`` if the
2018 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2019 the value from the amplifier data is used.
2023 ampExposure : `lsst.afw.image.Exposure`
2024 Exposure to process.
2025 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2026 Amplifier detector data.
2027 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2028 Image of overscan, required only for empirical read noise.
2032 lsst.ip.isr.isrFunctions.updateVariance
2034 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2035 gain = amp.getGain()
2037 if math.isnan(gain):
2039 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2042 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2043 amp.getName(), gain, patchedGain)
2046 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2047 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
2049 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2050 stats = afwMath.StatisticsControl()
2051 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2052 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2053 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2054 amp.getName(), readNoise)
2056 readNoise = amp.getReadNoise()
2058 isrFunctions.updateVariance(
2059 maskedImage=ampExposure.getMaskedImage(),
2061 readNoise=readNoise,
2065 """!Apply dark correction in place.
2069 exposure : `lsst.afw.image.Exposure`
2070 Exposure to process.
2071 darkExposure : `lsst.afw.image.Exposure`
2072 Dark exposure of the same size as ``exposure``.
2073 invert : `Bool`, optional
2074 If True, re-add the dark to an already corrected image.
2079 Raised if either ``exposure`` or ``darkExposure`` do not
2080 have their dark time defined.
2084 lsst.ip.isr.isrFunctions.darkCorrection
2086 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2087 if math.isnan(expScale):
2088 raise RuntimeError(
"Exposure darktime is NAN.")
2089 if darkExposure.getInfo().getVisitInfo()
is not None \
2090 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2091 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2095 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2098 isrFunctions.darkCorrection(
2099 maskedImage=exposure.getMaskedImage(),
2100 darkMaskedImage=darkExposure.getMaskedImage(),
2102 darkScale=darkScale,
2104 trimToFit=self.config.doTrimToMatchCalib
2108 """!Check if linearization is needed for the detector cameraGeom.
2110 Checks config.doLinearize and the linearity type of the first
2115 detector : `lsst.afw.cameraGeom.Detector`
2116 Detector to get linearity type from.
2120 doLinearize : `Bool`
2121 If True, linearization should be performed.
2123 return self.config.doLinearize
and \
2124 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2127 """!Apply flat correction in place.
2131 exposure : `lsst.afw.image.Exposure`
2132 Exposure to process.
2133 flatExposure : `lsst.afw.image.Exposure`
2134 Flat exposure of the same size as ``exposure``.
2135 invert : `Bool`, optional
2136 If True, unflatten an already flattened image.
2140 lsst.ip.isr.isrFunctions.flatCorrection
2142 isrFunctions.flatCorrection(
2143 maskedImage=exposure.getMaskedImage(),
2144 flatMaskedImage=flatExposure.getMaskedImage(),
2145 scalingType=self.config.flatScalingType,
2146 userScale=self.config.flatUserScale,
2148 trimToFit=self.config.doTrimToMatchCalib
2152 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2156 exposure : `lsst.afw.image.Exposure`
2157 Exposure to process. Only the amplifier DataSec is processed.
2158 amp : `lsst.afw.table.AmpInfoCatalog`
2159 Amplifier detector data.
2163 lsst.ip.isr.isrFunctions.makeThresholdMask
2165 if not math.isnan(amp.getSaturation()):
2166 maskedImage = exposure.getMaskedImage()
2167 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2168 isrFunctions.makeThresholdMask(
2169 maskedImage=dataView,
2170 threshold=amp.getSaturation(),
2172 maskName=self.config.saturatedMaskName,
2176 """!Interpolate over saturated pixels, in place.
2178 This method should be called after `saturationDetection`, to
2179 ensure that the saturated pixels have been identified in the
2180 SAT mask. It should also be called after `assembleCcd`, since
2181 saturated regions may cross amplifier boundaries.
2185 exposure : `lsst.afw.image.Exposure`
2186 Exposure to process.
2190 lsst.ip.isr.isrTask.saturationDetection
2191 lsst.ip.isr.isrFunctions.interpolateFromMask
2193 isrFunctions.interpolateFromMask(
2194 maskedImage=exposure.getMaskedImage(),
2195 fwhm=self.config.fwhm,
2196 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2197 maskNameList=list(self.config.saturatedMaskName),
2201 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2205 exposure : `lsst.afw.image.Exposure`
2206 Exposure to process. Only the amplifier DataSec is processed.
2207 amp : `lsst.afw.table.AmpInfoCatalog`
2208 Amplifier detector data.
2212 lsst.ip.isr.isrFunctions.makeThresholdMask
2216 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2217 This is intended to indicate pixels that may be affected by unknown systematics;
2218 for example if non-linearity corrections above a certain level are unstable
2219 then that would be a useful value for suspectLevel. A value of `nan` indicates
2220 that no such level exists and no pixels are to be masked as suspicious.
2222 suspectLevel = amp.getSuspectLevel()
2223 if math.isnan(suspectLevel):
2226 maskedImage = exposure.getMaskedImage()
2227 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2228 isrFunctions.makeThresholdMask(
2229 maskedImage=dataView,
2230 threshold=suspectLevel,
2232 maskName=self.config.suspectMaskName,
2236 """!Mask defects using mask plane "BAD", in place.
2240 exposure : `lsst.afw.image.Exposure`
2241 Exposure to process.
2242 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2243 `lsst.afw.image.DefectBase`.
2244 List of defects to mask.
2248 Call this after CCD assembly, since defects may cross amplifier boundaries.
2250 maskedImage = exposure.getMaskedImage()
2251 if not isinstance(defectBaseList, Defects):
2253 defectList = Defects(defectBaseList)
2255 defectList = defectBaseList
2256 defectList.maskPixels(maskedImage, maskName=
"BAD")
2258 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2259 """!Mask edge pixels with applicable mask plane.
2263 exposure : `lsst.afw.image.Exposure`
2264 Exposure to process.
2265 numEdgePixels : `int`, optional
2266 Number of edge pixels to mask.
2267 maskPlane : `str`, optional
2268 Mask plane name to use.
2269 level : `str`, optional
2270 Level at which to mask edges.
2272 maskedImage = exposure.getMaskedImage()
2273 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2275 if numEdgePixels > 0:
2276 if level ==
'DETECTOR':
2277 boxes = [maskedImage.getBBox()]
2278 elif level ==
'AMP':
2279 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2283 subImage = maskedImage[box]
2284 box.grow(-numEdgePixels)
2286 SourceDetectionTask.setEdgeBits(
2292 """Mask and interpolate defects using mask plane "BAD", in place.
2296 exposure : `lsst.afw.image.Exposure`
2297 Exposure to process.
2298 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2299 `lsst.afw.image.DefectBase`.
2300 List of defects to mask and interpolate.
2304 lsst.ip.isr.isrTask.maskDefect()
2307 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2308 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2309 isrFunctions.interpolateFromMask(
2310 maskedImage=exposure.getMaskedImage(),
2311 fwhm=self.config.fwhm,
2312 growSaturatedFootprints=0,
2313 maskNameList=[
"BAD"],
2317 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2321 exposure : `lsst.afw.image.Exposure`
2322 Exposure to process.
2326 We mask over all NaNs, including those that are masked with
2327 other bits (because those may or may not be interpolated over
2328 later, and we want to remove all NaNs). Despite this
2329 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2330 the historical name.
2332 maskedImage = exposure.getMaskedImage()
2335 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2336 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2337 numNans =
maskNans(maskedImage, maskVal)
2338 self.metadata.set(
"NUMNANS", numNans)
2340 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2343 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2347 exposure : `lsst.afw.image.Exposure`
2348 Exposure to process.
2352 lsst.ip.isr.isrTask.maskNan()
2355 isrFunctions.interpolateFromMask(
2356 maskedImage=exposure.getMaskedImage(),
2357 fwhm=self.config.fwhm,
2358 growSaturatedFootprints=0,
2359 maskNameList=[
"UNMASKEDNAN"],
2363 """Measure the image background in subgrids, for quality control purposes.
2367 exposure : `lsst.afw.image.Exposure`
2368 Exposure to process.
2369 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2370 Configuration object containing parameters on which background
2371 statistics and subgrids to use.
2373 if IsrQaConfig
is not None:
2374 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2375 IsrQaConfig.flatness.nIter)
2376 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2377 statsControl.setAndMask(maskVal)
2378 maskedImage = exposure.getMaskedImage()
2379 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2380 skyLevel = stats.getValue(afwMath.MEDIAN)
2381 skySigma = stats.getValue(afwMath.STDEVCLIP)
2382 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2383 metadata = exposure.getMetadata()
2384 metadata.set(
'SKYLEVEL', skyLevel)
2385 metadata.set(
'SKYSIGMA', skySigma)
2388 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2389 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2390 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2391 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2392 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2393 skyLevels = numpy.zeros((nX, nY))
2396 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2398 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2400 xLLC = xc - meshXHalf
2401 yLLC = yc - meshYHalf
2402 xURC = xc + meshXHalf - 1
2403 yURC = yc + meshYHalf - 1
2406 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2408 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2410 good = numpy.where(numpy.isfinite(skyLevels))
2411 skyMedian = numpy.median(skyLevels[good])
2412 flatness = (skyLevels[good] - skyMedian) / skyMedian
2413 flatness_rms = numpy.std(flatness)
2414 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2416 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2417 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2418 nX, nY, flatness_pp, flatness_rms)
2420 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2421 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2422 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2423 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2424 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2427 """Set an approximate magnitude zero point for the exposure.
2431 exposure : `lsst.afw.image.Exposure`
2432 Exposure to process.
2434 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2435 if filterName
in self.config.fluxMag0T1:
2436 fluxMag0 = self.config.fluxMag0T1[filterName]
2438 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2439 fluxMag0 = self.config.defaultFluxMag0T1
2441 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2443 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2446 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2447 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2450 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2454 ccdExposure : `lsst.afw.image.Exposure`
2455 Exposure to process.
2456 fpPolygon : `lsst.afw.geom.Polygon`
2457 Polygon in focal plane coordinates.
2460 ccd = ccdExposure.getDetector()
2461 fpCorners = ccd.getCorners(FOCAL_PLANE)
2462 ccdPolygon = Polygon(fpCorners)
2465 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2468 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2469 validPolygon = Polygon(ccdPoints)
2470 ccdExposure.getInfo().setValidPolygon(validPolygon)
2474 """Context manager that applies and removes flats and darks,
2475 if the task is configured to apply them.
2479 exp : `lsst.afw.image.Exposure`
2480 Exposure to process.
2481 flat : `lsst.afw.image.Exposure`
2482 Flat exposure the same size as ``exp``.
2483 dark : `lsst.afw.image.Exposure`, optional
2484 Dark exposure the same size as ``exp``.
2488 exp : `lsst.afw.image.Exposure`
2489 The flat and dark corrected exposure.
2491 if self.config.doDark
and dark
is not None:
2493 if self.config.doFlat:
2498 if self.config.doFlat:
2500 if self.config.doDark
and dark
is not None:
2504 """Utility function to examine ISR exposure at different stages.
2508 exposure : `lsst.afw.image.Exposure`
2511 State of processing to view.
2513 frame = getDebugFrame(self._display, stepname)
2515 display = getDisplay(frame)
2516 display.scale(
'asinh',
'zscale')
2517 display.mtv(exposure)
2518 prompt =
"Press Enter to continue [c]... "
2520 ans = input(prompt).lower()
2521 if ans
in (
"",
"c",):
2526 """A Detector-like object that supports returning gain and saturation level
2528 This is used when the input exposure does not have a detector.
2532 exposure : `lsst.afw.image.Exposure`
2533 Exposure to generate a fake amplifier for.
2534 config : `lsst.ip.isr.isrTaskConfig`
2535 Configuration to apply to the fake amplifier.
2539 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2541 self.
_gain = config.gain
2568 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2572 """Task to wrap the default IsrTask to allow it to be retargeted.
2574 The standard IsrTask can be called directly from a command line
2575 program, but doing so removes the ability of the task to be
2576 retargeted. As most cameras override some set of the IsrTask
2577 methods, this would remove those data-specific methods in the
2578 output post-ISR images. This wrapping class fixes the issue,
2579 allowing identical post-ISR images to be generated by both the
2580 processCcd and isrTask code.
2582 ConfigClass = RunIsrConfig
2583 _DefaultName =
"runIsr"
2587 self.makeSubtask(
"isr")
2593 dataRef : `lsst.daf.persistence.ButlerDataRef`
2594 data reference of the detector data to be processed
2598 result : `pipeBase.Struct`
2599 Result struct with component:
2601 - exposure : `lsst.afw.image.Exposure`
2602 Post-ISR processed exposure.