32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
134 bias = cT.PrerequisiteInput(
136 doc=
"Input bias calibration.",
137 storageClass=
"ExposureF",
138 dimensions=[
"instrument",
"detector"],
141 dark = cT.PrerequisiteInput(
143 doc=
"Input dark calibration.",
144 storageClass=
"ExposureF",
145 dimensions=[
"instrument",
"detector"],
148 flat = cT.PrerequisiteInput(
150 doc=
"Input flat calibration.",
151 storageClass=
"ExposureF",
152 dimensions=[
"instrument",
"physical_filter",
"detector"],
155 fringes = cT.PrerequisiteInput(
157 doc=
"Input fringe calibration.",
158 storageClass=
"ExposureF",
159 dimensions=[
"instrument",
"physical_filter",
"detector"],
162 strayLightData = cT.PrerequisiteInput(
164 doc=
"Input stray light calibration.",
165 storageClass=
"StrayLightData",
166 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 bfKernel = cT.PrerequisiteInput(
171 doc=
"Input brighter-fatter kernel.",
172 storageClass=
"NumpyArray",
173 dimensions=[
"instrument"],
176 newBFKernel = cT.PrerequisiteInput(
177 name=
'brighterFatterKernel',
178 doc=
"Newer complete kernel + gain solutions.",
179 storageClass=
"BrighterFatterKernel",
180 dimensions=[
"instrument",
"detector"],
183 defects = cT.PrerequisiteInput(
185 doc=
"Input defect tables.",
186 storageClass=
"Defects",
187 dimensions=[
"instrument",
"detector"],
190 opticsTransmission = cT.PrerequisiteInput(
191 name=
"transmission_optics",
192 storageClass=
"TransmissionCurve",
193 doc=
"Transmission curve due to the optics.",
194 dimensions=[
"instrument"],
197 filterTransmission = cT.PrerequisiteInput(
198 name=
"transmission_filter",
199 storageClass=
"TransmissionCurve",
200 doc=
"Transmission curve due to the filter.",
201 dimensions=[
"instrument",
"physical_filter"],
204 sensorTransmission = cT.PrerequisiteInput(
205 name=
"transmission_sensor",
206 storageClass=
"TransmissionCurve",
207 doc=
"Transmission curve due to the sensor.",
208 dimensions=[
"instrument",
"detector"],
211 atmosphereTransmission = cT.PrerequisiteInput(
212 name=
"transmission_atmosphere",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the atmosphere.",
215 dimensions=[
"instrument"],
218 illumMaskedImage = cT.PrerequisiteInput(
220 doc=
"Input illumination correction.",
221 storageClass=
"MaskedImageF",
222 dimensions=[
"instrument",
"physical_filter",
"detector"],
226 outputExposure = cT.Output(
228 doc=
"Output ISR processed exposure.",
229 storageClass=
"Exposure",
230 dimensions=[
"instrument",
"exposure",
"detector"],
232 preInterpExposure = cT.Output(
233 name=
'preInterpISRCCD',
234 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
235 storageClass=
"ExposureF",
236 dimensions=[
"instrument",
"exposure",
"detector"],
238 outputOssThumbnail = cT.Output(
240 doc=
"Output Overscan-subtracted thumbnail image.",
241 storageClass=
"Thumbnail",
242 dimensions=[
"instrument",
"exposure",
"detector"],
244 outputFlattenedThumbnail = cT.Output(
245 name=
"FlattenedThumb",
246 doc=
"Output flat-corrected thumbnail image.",
247 storageClass=
"Thumbnail",
248 dimensions=[
"instrument",
"exposure",
"detector"],
254 if config.doBias
is not True:
255 self.prerequisiteInputs.discard(
"bias")
256 if config.doLinearize
is not True:
257 self.prerequisiteInputs.discard(
"linearizer")
258 if config.doCrosstalk
is not True:
259 self.inputs.discard(
"crosstalkSources")
260 self.prerequisiteInputs.discard(
"crosstalk")
261 if config.doBrighterFatter
is not True:
262 self.prerequisiteInputs.discard(
"bfKernel")
263 self.prerequisiteInputs.discard(
"newBFKernel")
264 if config.doDefect
is not True:
265 self.prerequisiteInputs.discard(
"defects")
266 if config.doDark
is not True:
267 self.prerequisiteInputs.discard(
"dark")
268 if config.doFlat
is not True:
269 self.prerequisiteInputs.discard(
"flat")
270 if config.doAttachTransmissionCurve
is not True:
271 self.prerequisiteInputs.discard(
"opticsTransmission")
272 self.prerequisiteInputs.discard(
"filterTransmission")
273 self.prerequisiteInputs.discard(
"sensorTransmission")
274 self.prerequisiteInputs.discard(
"atmosphereTransmission")
275 if config.doUseOpticsTransmission
is not True:
276 self.prerequisiteInputs.discard(
"opticsTransmission")
277 if config.doUseFilterTransmission
is not True:
278 self.prerequisiteInputs.discard(
"filterTransmission")
279 if config.doUseSensorTransmission
is not True:
280 self.prerequisiteInputs.discard(
"sensorTransmission")
281 if config.doUseAtmosphereTransmission
is not True:
282 self.prerequisiteInputs.discard(
"atmosphereTransmission")
283 if config.doIlluminationCorrection
is not True:
284 self.prerequisiteInputs.discard(
"illumMaskedImage")
286 if config.doWrite
is not True:
287 self.outputs.discard(
"outputExposure")
288 self.outputs.discard(
"preInterpExposure")
289 self.outputs.discard(
"outputFlattenedThumbnail")
290 self.outputs.discard(
"outputOssThumbnail")
291 if config.doSaveInterpPixels
is not True:
292 self.outputs.discard(
"preInterpExposure")
293 if config.qa.doThumbnailOss
is not True:
294 self.outputs.discard(
"outputOssThumbnail")
295 if config.qa.doThumbnailFlattened
is not True:
296 self.outputs.discard(
"outputFlattenedThumbnail")
300 pipelineConnections=IsrTaskConnections):
301 """Configuration parameters for IsrTask.
303 Items are grouped in the order in which they are executed by the task.
305 datasetType = pexConfig.Field(
307 doc=
"Dataset type for input data; users will typically leave this alone, "
308 "but camera-specific ISR tasks will override it",
312 fallbackFilterName = pexConfig.Field(
314 doc=
"Fallback default filter name for calibrations.",
317 useFallbackDate = pexConfig.Field(
319 doc=
"Pass observation date when using fallback filter.",
322 expectWcs = pexConfig.Field(
325 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
327 fwhm = pexConfig.Field(
329 doc=
"FWHM of PSF in arcseconds.",
332 qa = pexConfig.ConfigField(
334 doc=
"QA related configuration options.",
338 doConvertIntToFloat = pexConfig.Field(
340 doc=
"Convert integer raw images to floating point values?",
345 doSaturation = pexConfig.Field(
347 doc=
"Mask saturated pixels? NB: this is totally independent of the"
348 " interpolation option - this is ONLY setting the bits in the mask."
349 " To have them interpolated make sure doSaturationInterpolation=True",
352 saturatedMaskName = pexConfig.Field(
354 doc=
"Name of mask plane to use in saturation detection and interpolation",
357 saturation = pexConfig.Field(
359 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
360 default=float(
"NaN"),
362 growSaturationFootprintSize = pexConfig.Field(
364 doc=
"Number of pixels by which to grow the saturation footprints",
369 doSuspect = pexConfig.Field(
371 doc=
"Mask suspect pixels?",
374 suspectMaskName = pexConfig.Field(
376 doc=
"Name of mask plane to use for suspect pixels",
379 numEdgeSuspect = pexConfig.Field(
381 doc=
"Number of edge pixels to be flagged as untrustworthy.",
384 edgeMaskLevel = pexConfig.ChoiceField(
386 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
389 'DETECTOR':
'Mask only the edges of the full detector.',
390 'AMP':
'Mask edges of each amplifier.',
395 doSetBadRegions = pexConfig.Field(
397 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
400 badStatistic = pexConfig.ChoiceField(
402 doc=
"How to estimate the average value for BAD regions.",
405 "MEANCLIP":
"Correct using the (clipped) mean of good data",
406 "MEDIAN":
"Correct using the median of the good data",
411 doOverscan = pexConfig.Field(
413 doc=
"Do overscan subtraction?",
416 overscan = pexConfig.ConfigurableField(
417 target=OverscanCorrectionTask,
418 doc=
"Overscan subtraction task for image segments.",
421 overscanFitType = pexConfig.ChoiceField(
423 doc=
"The method for fitting the overscan bias level.",
426 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
427 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
428 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
429 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
430 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
431 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
432 "MEAN":
"Correct using the mean of the overscan region",
433 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
434 "MEDIAN":
"Correct using the median of the overscan region",
435 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
437 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
438 " This option will no longer be used, and will be removed after v20.")
440 overscanOrder = pexConfig.Field(
442 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
443 "or number of spline knots if overscan fit type is a spline."),
445 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
446 " This option will no longer be used, and will be removed after v20.")
448 overscanNumSigmaClip = pexConfig.Field(
450 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
452 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
453 " This option will no longer be used, and will be removed after v20.")
455 overscanIsInt = pexConfig.Field(
457 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
458 " and overscan.FitType=MEDIAN_PER_ROW.",
460 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
461 " This option will no longer be used, and will be removed after v20.")
464 overscanNumLeadingColumnsToSkip = pexConfig.Field(
466 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
469 overscanNumTrailingColumnsToSkip = pexConfig.Field(
471 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
474 overscanMaxDev = pexConfig.Field(
476 doc=
"Maximum deviation from the median for overscan",
477 default=1000.0, check=
lambda x: x > 0
479 overscanBiasJump = pexConfig.Field(
481 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
484 overscanBiasJumpKeyword = pexConfig.Field(
486 doc=
"Header keyword containing information about devices.",
487 default=
"NO_SUCH_KEY",
489 overscanBiasJumpDevices = pexConfig.ListField(
491 doc=
"List of devices that need piecewise overscan correction.",
494 overscanBiasJumpLocation = pexConfig.Field(
496 doc=
"Location of bias jump along y-axis.",
501 doAssembleCcd = pexConfig.Field(
504 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
506 assembleCcd = pexConfig.ConfigurableField(
507 target=AssembleCcdTask,
508 doc=
"CCD assembly task",
512 doAssembleIsrExposures = pexConfig.Field(
515 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
517 doTrimToMatchCalib = pexConfig.Field(
520 doc=
"Trim raw data to match calibration bounding boxes?"
524 doBias = pexConfig.Field(
526 doc=
"Apply bias frame correction?",
529 biasDataProductName = pexConfig.Field(
531 doc=
"Name of the bias data product",
534 doBiasBeforeOverscan = pexConfig.Field(
536 doc=
"Reverse order of overscan and bias correction.",
541 doVariance = pexConfig.Field(
543 doc=
"Calculate variance?",
546 gain = pexConfig.Field(
548 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
549 default=float(
"NaN"),
551 readNoise = pexConfig.Field(
553 doc=
"The read noise to use if no Detector is present in the Exposure",
556 doEmpiricalReadNoise = pexConfig.Field(
559 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
563 doLinearize = pexConfig.Field(
565 doc=
"Correct for nonlinearity of the detector's response?",
570 doCrosstalk = pexConfig.Field(
572 doc=
"Apply intra-CCD crosstalk correction?",
575 doCrosstalkBeforeAssemble = pexConfig.Field(
577 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
580 crosstalk = pexConfig.ConfigurableField(
581 target=CrosstalkTask,
582 doc=
"Intra-CCD crosstalk correction",
586 doDefect = pexConfig.Field(
588 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
591 doNanMasking = pexConfig.Field(
593 doc=
"Mask NAN pixels?",
596 doWidenSaturationTrails = pexConfig.Field(
598 doc=
"Widen bleed trails based on their width?",
603 doBrighterFatter = pexConfig.Field(
606 doc=
"Apply the brighter fatter correction"
608 brighterFatterLevel = pexConfig.ChoiceField(
611 doc=
"The level at which to correct for brighter-fatter.",
613 "AMP":
"Every amplifier treated separately.",
614 "DETECTOR":
"One kernel per detector",
617 brighterFatterMaxIter = pexConfig.Field(
620 doc=
"Maximum number of iterations for the brighter fatter correction"
622 brighterFatterThreshold = pexConfig.Field(
625 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
626 " absolute value of the difference between the current corrected image and the one"
627 " from the previous iteration summed over all the pixels."
629 brighterFatterApplyGain = pexConfig.Field(
632 doc=
"Should the gain be applied when applying the brighter fatter correction?"
634 brighterFatterMaskGrowSize = pexConfig.Field(
637 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
638 " when brighter-fatter correction is applied."
642 doDark = pexConfig.Field(
644 doc=
"Apply dark frame correction?",
647 darkDataProductName = pexConfig.Field(
649 doc=
"Name of the dark data product",
654 doStrayLight = pexConfig.Field(
656 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
659 strayLight = pexConfig.ConfigurableField(
660 target=StrayLightTask,
661 doc=
"y-band stray light correction"
665 doFlat = pexConfig.Field(
667 doc=
"Apply flat field correction?",
670 flatDataProductName = pexConfig.Field(
672 doc=
"Name of the flat data product",
675 flatScalingType = pexConfig.ChoiceField(
677 doc=
"The method for scaling the flat on the fly.",
680 "USER":
"Scale by flatUserScale",
681 "MEAN":
"Scale by the inverse of the mean",
682 "MEDIAN":
"Scale by the inverse of the median",
685 flatUserScale = pexConfig.Field(
687 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
690 doTweakFlat = pexConfig.Field(
692 doc=
"Tweak flats to match observed amplifier ratios?",
697 doApplyGains = pexConfig.Field(
699 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
702 normalizeGains = pexConfig.Field(
704 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
709 doFringe = pexConfig.Field(
711 doc=
"Apply fringe correction?",
714 fringe = pexConfig.ConfigurableField(
716 doc=
"Fringe subtraction task",
718 fringeAfterFlat = pexConfig.Field(
720 doc=
"Do fringe subtraction after flat-fielding?",
725 doMeasureBackground = pexConfig.Field(
727 doc=
"Measure the background level on the reduced image?",
732 doCameraSpecificMasking = pexConfig.Field(
734 doc=
"Mask camera-specific bad regions?",
737 masking = pexConfig.ConfigurableField(
744 doInterpolate = pexConfig.Field(
746 doc=
"Interpolate masked pixels?",
749 doSaturationInterpolation = pexConfig.Field(
751 doc=
"Perform interpolation over pixels masked as saturated?"
752 " NB: This is independent of doSaturation; if that is False this plane"
753 " will likely be blank, resulting in a no-op here.",
756 doNanInterpolation = pexConfig.Field(
758 doc=
"Perform interpolation over pixels masked as NaN?"
759 " NB: This is independent of doNanMasking; if that is False this plane"
760 " will likely be blank, resulting in a no-op here.",
763 doNanInterpAfterFlat = pexConfig.Field(
765 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
766 "also have to interpolate them before flat-fielding."),
769 maskListToInterpolate = pexConfig.ListField(
771 doc=
"List of mask planes that should be interpolated.",
772 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
774 doSaveInterpPixels = pexConfig.Field(
776 doc=
"Save a copy of the pre-interpolated pixel values?",
781 fluxMag0T1 = pexConfig.DictField(
784 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
785 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
788 defaultFluxMag0T1 = pexConfig.Field(
790 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
791 default=pow(10.0, 0.4*28.0)
795 doVignette = pexConfig.Field(
797 doc=
"Apply vignetting parameters?",
800 vignette = pexConfig.ConfigurableField(
802 doc=
"Vignetting task.",
806 doAttachTransmissionCurve = pexConfig.Field(
809 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
811 doUseOpticsTransmission = pexConfig.Field(
814 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
816 doUseFilterTransmission = pexConfig.Field(
819 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
821 doUseSensorTransmission = pexConfig.Field(
824 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
826 doUseAtmosphereTransmission = pexConfig.Field(
829 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
833 doIlluminationCorrection = pexConfig.Field(
836 doc=
"Perform illumination correction?"
838 illuminationCorrectionDataProductName = pexConfig.Field(
840 doc=
"Name of the illumination correction data product.",
843 illumScale = pexConfig.Field(
845 doc=
"Scale factor for the illumination correction.",
848 illumFilters = pexConfig.ListField(
851 doc=
"Only perform illumination correction for these filters."
855 doWrite = pexConfig.Field(
857 doc=
"Persist postISRCCD?",
864 raise ValueError(
"You may not specify both doFlat and doApplyGains")
866 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
868 self.config.maskListToInterpolate.append(
"SAT")
870 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
873 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
874 """Apply common instrument signature correction algorithms to a raw frame.
876 The process for correcting imaging data is very similar from
877 camera to camera. This task provides a vanilla implementation of
878 doing these corrections, including the ability to turn certain
879 corrections off if they are not needed. The inputs to the primary
880 method, `run()`, are a raw exposure to be corrected and the
881 calibration data products. The raw input is a single chip sized
882 mosaic of all amps including overscans and other non-science
883 pixels. The method `runDataRef()` identifies and defines the
884 calibration data products, and is intended for use by a
885 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
886 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
887 subclassed for different camera, although the most camera specific
888 methods have been split into subtasks that can be redirected
891 The __init__ method sets up the subtasks for ISR processing, using
892 the defaults from `lsst.ip.isr`.
897 Positional arguments passed to the Task constructor. None used at this time.
898 kwargs : `dict`, optional
899 Keyword arguments passed on to the Task constructor. None used at this time.
901 ConfigClass = IsrTaskConfig
906 self.makeSubtask(
"assembleCcd")
907 self.makeSubtask(
"crosstalk")
908 self.makeSubtask(
"strayLight")
909 self.makeSubtask(
"fringe")
910 self.makeSubtask(
"masking")
911 self.makeSubtask(
"overscan")
912 self.makeSubtask(
"vignette")
915 inputs = butlerQC.get(inputRefs)
918 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
919 except Exception
as e:
920 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
923 inputs[
'isGen3'] =
True
925 detector = inputs[
'ccdExposure'].getDetector()
927 if self.config.doCrosstalk
is True:
930 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
931 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
932 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
934 coeffVector = (self.config.crosstalk.crosstalkValues
935 if self.config.crosstalk.useConfigCoefficients
else None)
936 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
937 inputs[
'crosstalk'] = crosstalkCalib
938 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
939 if 'crosstalkSources' not in inputs:
940 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
943 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
945 linearizer.fromYaml(inputs[
'linearizer'])
949 inputs[
'linearizer'] = linearizer
951 if self.config.doDefect
is True:
952 if "defects" in inputs
and inputs[
'defects']
is not None:
955 if not isinstance(inputs[
"defects"], Defects):
956 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
960 if self.config.doBrighterFatter:
961 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
962 if brighterFatterKernel
is None:
963 brighterFatterKernel = inputs.get(
'bfKernel',
None)
965 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
966 detId = detector.getId()
967 inputs[
'bfGains'] = brighterFatterKernel.gain
970 if self.config.brighterFatterLevel ==
'DETECTOR':
971 if brighterFatterKernel.detectorKernel:
972 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
973 elif brighterFatterKernel.detectorKernelFromAmpKernels:
974 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
976 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
979 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
981 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
982 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
983 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
985 assembler=self.assembleCcd
986 if self.config.doAssembleIsrExposures
else None)
988 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
990 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
991 if 'strayLightData' not in inputs:
992 inputs[
'strayLightData'] =
None
994 outputs = self.
run(**inputs)
995 butlerQC.put(outputs, outputRefs)
998 """Retrieve necessary frames for instrument signature removal.
1000 Pre-fetching all required ISR data products limits the IO
1001 required by the ISR. Any conflict between the calibration data
1002 available and that needed for ISR is also detected prior to
1003 doing processing, allowing it to fail quickly.
1007 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1008 Butler reference of the detector data to be processed
1009 rawExposure : `afw.image.Exposure`
1010 The raw exposure that will later be corrected with the
1011 retrieved calibration data; should not be modified in this
1016 result : `lsst.pipe.base.Struct`
1017 Result struct with components (which may be `None`):
1018 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1019 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1020 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1021 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1022 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1023 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1024 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
1025 - ``fringes``: `lsst.pipe.base.Struct` with components:
1026 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1027 - ``seed``: random seed derived from the ccdExposureId for random
1028 number generator (`uint32`).
1029 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1030 A ``TransmissionCurve`` that represents the throughput of the optics,
1031 to be evaluated in focal-plane coordinates.
1032 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1033 A ``TransmissionCurve`` that represents the throughput of the filter
1034 itself, to be evaluated in focal-plane coordinates.
1035 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1036 A ``TransmissionCurve`` that represents the throughput of the sensor
1037 itself, to be evaluated in post-assembly trimmed detector coordinates.
1038 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1039 A ``TransmissionCurve`` that represents the throughput of the
1040 atmosphere, assumed to be spatially constant.
1041 - ``strayLightData`` : `object`
1042 An opaque object containing calibration information for
1043 stray-light correction. If `None`, no correction will be
1045 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1049 NotImplementedError :
1050 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1053 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1054 dateObs = dateObs.toPython().isoformat()
1055 except RuntimeError:
1056 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1059 ccd = rawExposure.getDetector()
1060 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
1061 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1062 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
1063 if self.config.doBias
else None)
1065 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1067 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1068 linearizer.log = self.log
1069 if isinstance(linearizer, numpy.ndarray):
1072 crosstalkCalib =
None
1073 if self.config.doCrosstalk:
1075 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1077 coeffVector = (self.config.crosstalk.crosstalkValues
1078 if self.config.crosstalk.useConfigCoefficients
else None)
1079 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1080 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1081 if self.config.doCrosstalk
else None)
1083 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
1084 if self.config.doDark
else None)
1085 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
1087 if self.config.doFlat
else None)
1089 brighterFatterKernel =
None
1090 brighterFatterGains =
None
1091 if self.config.doBrighterFatter
is True:
1096 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1097 brighterFatterGains = brighterFatterKernel.gain
1098 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1101 brighterFatterKernel = dataRef.get(
"bfKernel")
1102 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
1104 brighterFatterKernel =
None
1105 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1108 if self.config.brighterFatterLevel ==
'DETECTOR':
1109 if brighterFatterKernel.detectorKernel:
1110 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1111 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1112 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1114 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1117 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1119 defectList = (dataRef.get(
"defects")
1120 if self.config.doDefect
else None)
1121 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1122 if self.config.doAssembleIsrExposures
else None)
1123 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1124 else pipeBase.Struct(fringes=
None))
1126 if self.config.doAttachTransmissionCurve:
1127 opticsTransmission = (dataRef.get(
"transmission_optics")
1128 if self.config.doUseOpticsTransmission
else None)
1129 filterTransmission = (dataRef.get(
"transmission_filter")
1130 if self.config.doUseFilterTransmission
else None)
1131 sensorTransmission = (dataRef.get(
"transmission_sensor")
1132 if self.config.doUseSensorTransmission
else None)
1133 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1134 if self.config.doUseAtmosphereTransmission
else None)
1136 opticsTransmission =
None
1137 filterTransmission =
None
1138 sensorTransmission =
None
1139 atmosphereTransmission =
None
1141 if self.config.doStrayLight:
1142 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1144 strayLightData =
None
1147 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1148 if (self.config.doIlluminationCorrection
1149 and filterName
in self.config.illumFilters)
1153 return pipeBase.Struct(bias=biasExposure,
1154 linearizer=linearizer,
1155 crosstalk=crosstalkCalib,
1156 crosstalkSources=crosstalkSources,
1159 bfKernel=brighterFatterKernel,
1160 bfGains=brighterFatterGains,
1162 fringes=fringeStruct,
1163 opticsTransmission=opticsTransmission,
1164 filterTransmission=filterTransmission,
1165 sensorTransmission=sensorTransmission,
1166 atmosphereTransmission=atmosphereTransmission,
1167 strayLightData=strayLightData,
1168 illumMaskedImage=illumMaskedImage
1171 @pipeBase.timeMethod
1172 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1173 crosstalk=None, crosstalkSources=None,
1174 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1175 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1176 sensorTransmission=
None, atmosphereTransmission=
None,
1177 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1180 """Perform instrument signature removal on an exposure.
1182 Steps included in the ISR processing, in order performed, are:
1183 - saturation and suspect pixel masking
1184 - overscan subtraction
1185 - CCD assembly of individual amplifiers
1187 - variance image construction
1188 - linearization of non-linear response
1190 - brighter-fatter correction
1193 - stray light subtraction
1195 - masking of known defects and camera specific features
1196 - vignette calculation
1197 - appending transmission curve and distortion model
1201 ccdExposure : `lsst.afw.image.Exposure`
1202 The raw exposure that is to be run through ISR. The
1203 exposure is modified by this method.
1204 camera : `lsst.afw.cameraGeom.Camera`, optional
1205 The camera geometry for this exposure. Required if ``isGen3`` is
1206 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1207 ``flat`` does not have an associated detector.
1208 bias : `lsst.afw.image.Exposure`, optional
1209 Bias calibration frame.
1210 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1211 Functor for linearization.
1212 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1213 Calibration for crosstalk.
1214 crosstalkSources : `list`, optional
1215 List of possible crosstalk sources.
1216 dark : `lsst.afw.image.Exposure`, optional
1217 Dark calibration frame.
1218 flat : `lsst.afw.image.Exposure`, optional
1219 Flat calibration frame.
1220 bfKernel : `numpy.ndarray`, optional
1221 Brighter-fatter kernel.
1222 bfGains : `dict` of `float`, optional
1223 Gains used to override the detector's nominal gains for the
1224 brighter-fatter correction. A dict keyed by amplifier name for
1225 the detector in question.
1226 defects : `lsst.meas.algorithms.Defects`, optional
1228 fringes : `lsst.pipe.base.Struct`, optional
1229 Struct containing the fringe correction data, with
1231 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1232 - ``seed``: random seed derived from the ccdExposureId for random
1233 number generator (`uint32`)
1234 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1235 A ``TransmissionCurve`` that represents the throughput of the optics,
1236 to be evaluated in focal-plane coordinates.
1237 filterTransmission : `lsst.afw.image.TransmissionCurve`
1238 A ``TransmissionCurve`` that represents the throughput of the filter
1239 itself, to be evaluated in focal-plane coordinates.
1240 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1241 A ``TransmissionCurve`` that represents the throughput of the sensor
1242 itself, to be evaluated in post-assembly trimmed detector coordinates.
1243 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1244 A ``TransmissionCurve`` that represents the throughput of the
1245 atmosphere, assumed to be spatially constant.
1246 detectorNum : `int`, optional
1247 The integer number for the detector to process.
1248 isGen3 : bool, optional
1249 Flag this call to run() as using the Gen3 butler environment.
1250 strayLightData : `object`, optional
1251 Opaque object containing calibration information for stray-light
1252 correction. If `None`, no correction will be performed.
1253 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1254 Illumination correction image.
1258 result : `lsst.pipe.base.Struct`
1259 Result struct with component:
1260 - ``exposure`` : `afw.image.Exposure`
1261 The fully ISR corrected exposure.
1262 - ``outputExposure`` : `afw.image.Exposure`
1263 An alias for `exposure`
1264 - ``ossThumb`` : `numpy.ndarray`
1265 Thumbnail image of the exposure after overscan subtraction.
1266 - ``flattenedThumb`` : `numpy.ndarray`
1267 Thumbnail image of the exposure after flat-field correction.
1272 Raised if a configuration option is set to True, but the
1273 required calibration data has not been specified.
1277 The current processed exposure can be viewed by setting the
1278 appropriate lsstDebug entries in the `debug.display`
1279 dictionary. The names of these entries correspond to some of
1280 the IsrTaskConfig Boolean options, with the value denoting the
1281 frame to use. The exposure is shown inside the matching
1282 option check and after the processing of that step has
1283 finished. The steps with debug points are:
1294 In addition, setting the "postISRCCD" entry displays the
1295 exposure after all ISR processing has finished.
1303 if detectorNum
is None:
1304 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1306 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1311 if isinstance(ccdExposure, ButlerDataRef):
1314 ccd = ccdExposure.getDetector()
1315 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1318 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1319 ccd = [
FakeAmp(ccdExposure, self.config)]
1322 if self.config.doBias
and bias
is None:
1323 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1325 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1326 if self.config.doBrighterFatter
and bfKernel
is None:
1327 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1328 if self.config.doDark
and dark
is None:
1329 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1330 if self.config.doFlat
and flat
is None:
1331 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1332 if self.config.doDefect
and defects
is None:
1333 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1334 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1335 and fringes.fringes
is None):
1340 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1341 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1342 and illumMaskedImage
is None):
1343 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1346 if self.config.doConvertIntToFloat:
1347 self.log.info(
"Converting exposure to floating point values.")
1350 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1351 self.log.info(
"Applying bias correction.")
1352 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1353 trimToFit=self.config.doTrimToMatchCalib)
1360 if ccdExposure.getBBox().contains(amp.getBBox()):
1364 if self.config.doOverscan
and not badAmp:
1367 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1368 if overscanResults
is not None and \
1369 self.config.qa
is not None and self.config.qa.saveStats
is True:
1370 if isinstance(overscanResults.overscanFit, float):
1371 qaMedian = overscanResults.overscanFit
1372 qaStdev = float(
"NaN")
1374 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1375 afwMath.MEDIAN | afwMath.STDEVCLIP)
1376 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1377 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1379 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1380 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1381 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1382 amp.getName(), qaMedian, qaStdev)
1383 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1386 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1387 overscanResults =
None
1389 overscans.append(overscanResults
if overscanResults
is not None else None)
1391 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1393 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1394 self.log.info(
"Applying crosstalk correction.")
1395 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1396 crosstalkSources=crosstalkSources)
1397 self.
debugView(ccdExposure,
"doCrosstalk")
1399 if self.config.doAssembleCcd:
1400 self.log.info(
"Assembling CCD from amplifiers.")
1401 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1403 if self.config.expectWcs
and not ccdExposure.getWcs():
1404 self.log.warn(
"No WCS found in input exposure.")
1405 self.
debugView(ccdExposure,
"doAssembleCcd")
1408 if self.config.qa.doThumbnailOss:
1409 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1411 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1412 self.log.info(
"Applying bias correction.")
1413 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1414 trimToFit=self.config.doTrimToMatchCalib)
1417 if self.config.doVariance:
1418 for amp, overscanResults
in zip(ccd, overscans):
1419 if ccdExposure.getBBox().contains(amp.getBBox()):
1420 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1421 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1422 if overscanResults
is not None:
1424 overscanImage=overscanResults.overscanImage)
1428 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1429 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1430 afwMath.MEDIAN | afwMath.STDEVCLIP)
1431 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1432 qaStats.getValue(afwMath.MEDIAN))
1433 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1434 qaStats.getValue(afwMath.STDEVCLIP))
1435 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1436 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1437 qaStats.getValue(afwMath.STDEVCLIP))
1440 self.log.info(
"Applying linearizer.")
1441 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1442 detector=ccd, log=self.log)
1444 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1445 self.log.info(
"Applying crosstalk correction.")
1446 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1447 crosstalkSources=crosstalkSources, isTrimmed=
True)
1448 self.
debugView(ccdExposure,
"doCrosstalk")
1452 if self.config.doDefect:
1453 self.log.info(
"Masking defects.")
1456 if self.config.numEdgeSuspect > 0:
1457 self.log.info(
"Masking edges as SUSPECT.")
1458 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1459 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1461 if self.config.doNanMasking:
1462 self.log.info(
"Masking NAN value pixels.")
1465 if self.config.doWidenSaturationTrails:
1466 self.log.info(
"Widening saturation trails.")
1467 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1469 if self.config.doCameraSpecificMasking:
1470 self.log.info(
"Masking regions for camera specific reasons.")
1471 self.masking.
run(ccdExposure)
1473 if self.config.doBrighterFatter:
1482 interpExp = ccdExposure.clone()
1484 isrFunctions.interpolateFromMask(
1485 maskedImage=interpExp.getMaskedImage(),
1486 fwhm=self.config.fwhm,
1487 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1488 maskNameList=self.config.maskListToInterpolate
1490 bfExp = interpExp.clone()
1492 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1493 type(bfKernel), type(bfGains))
1494 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1495 self.config.brighterFatterMaxIter,
1496 self.config.brighterFatterThreshold,
1497 self.config.brighterFatterApplyGain,
1499 if bfResults[1] == self.config.brighterFatterMaxIter:
1500 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1503 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1505 image = ccdExposure.getMaskedImage().getImage()
1506 bfCorr = bfExp.getMaskedImage().getImage()
1507 bfCorr -= interpExp.getMaskedImage().getImage()
1516 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1517 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1520 if self.config.brighterFatterMaskGrowSize > 0:
1521 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1522 for maskPlane
in self.config.maskListToInterpolate:
1523 isrFunctions.growMasks(ccdExposure.getMask(),
1524 radius=self.config.brighterFatterMaskGrowSize,
1525 maskNameList=maskPlane,
1526 maskValue=maskPlane)
1528 self.
debugView(ccdExposure,
"doBrighterFatter")
1530 if self.config.doDark:
1531 self.log.info(
"Applying dark correction.")
1535 if self.config.doFringe
and not self.config.fringeAfterFlat:
1536 self.log.info(
"Applying fringe correction before flat.")
1537 self.fringe.
run(ccdExposure, **fringes.getDict())
1540 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1541 self.log.info(
"Checking strayLight correction.")
1542 self.strayLight.
run(ccdExposure, strayLightData)
1543 self.
debugView(ccdExposure,
"doStrayLight")
1545 if self.config.doFlat:
1546 self.log.info(
"Applying flat correction.")
1550 if self.config.doApplyGains:
1551 self.log.info(
"Applying gain correction instead of flat.")
1552 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1554 if self.config.doFringe
and self.config.fringeAfterFlat:
1555 self.log.info(
"Applying fringe correction after flat.")
1556 self.fringe.
run(ccdExposure, **fringes.getDict())
1558 if self.config.doVignette:
1559 self.log.info(
"Constructing Vignette polygon.")
1562 if self.config.vignette.doWriteVignettePolygon:
1565 if self.config.doAttachTransmissionCurve:
1566 self.log.info(
"Adding transmission curves.")
1567 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1568 filterTransmission=filterTransmission,
1569 sensorTransmission=sensorTransmission,
1570 atmosphereTransmission=atmosphereTransmission)
1572 flattenedThumb =
None
1573 if self.config.qa.doThumbnailFlattened:
1574 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1576 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1577 self.log.info(
"Performing illumination correction.")
1578 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1579 illumMaskedImage, illumScale=self.config.illumScale,
1580 trimToFit=self.config.doTrimToMatchCalib)
1583 if self.config.doSaveInterpPixels:
1584 preInterpExp = ccdExposure.clone()
1599 if self.config.doSetBadRegions:
1600 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1601 if badPixelCount > 0:
1602 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1604 if self.config.doInterpolate:
1605 self.log.info(
"Interpolating masked pixels.")
1606 isrFunctions.interpolateFromMask(
1607 maskedImage=ccdExposure.getMaskedImage(),
1608 fwhm=self.config.fwhm,
1609 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1610 maskNameList=list(self.config.maskListToInterpolate)
1615 if self.config.doMeasureBackground:
1616 self.log.info(
"Measuring background level.")
1619 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1621 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1622 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1623 afwMath.MEDIAN | afwMath.STDEVCLIP)
1624 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1625 qaStats.getValue(afwMath.MEDIAN))
1626 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1627 qaStats.getValue(afwMath.STDEVCLIP))
1628 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1629 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1630 qaStats.getValue(afwMath.STDEVCLIP))
1632 self.
debugView(ccdExposure,
"postISRCCD")
1634 return pipeBase.Struct(
1635 exposure=ccdExposure,
1637 flattenedThumb=flattenedThumb,
1639 preInterpolatedExposure=preInterpExp,
1640 outputExposure=ccdExposure,
1641 outputOssThumbnail=ossThumb,
1642 outputFlattenedThumbnail=flattenedThumb,
1645 @pipeBase.timeMethod
1647 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1649 This method contains the `CmdLineTask` interface to the ISR
1650 processing. All IO is handled here, freeing the `run()` method
1651 to manage only pixel-level calculations. The steps performed
1653 - Read in necessary detrending/isr/calibration data.
1654 - Process raw exposure in `run()`.
1655 - Persist the ISR-corrected exposure as "postISRCCD" if
1656 config.doWrite=True.
1660 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1661 DataRef of the detector data to be processed
1665 result : `lsst.pipe.base.Struct`
1666 Result struct with component:
1667 - ``exposure`` : `afw.image.Exposure`
1668 The fully ISR corrected exposure.
1673 Raised if a configuration option is set to True, but the
1674 required calibration data does not exist.
1677 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1679 ccdExposure = sensorRef.get(self.config.datasetType)
1681 camera = sensorRef.get(
"camera")
1682 isrData = self.
readIsrData(sensorRef, ccdExposure)
1684 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1686 if self.config.doWrite:
1687 sensorRef.put(result.exposure,
"postISRCCD")
1688 if result.preInterpolatedExposure
is not None:
1689 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1690 if result.ossThumb
is not None:
1691 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1692 if result.flattenedThumb
is not None:
1693 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1698 """Retrieve a calibration dataset for removing instrument signature.
1703 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1704 DataRef of the detector data to find calibration datasets
1707 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1708 dateObs : `str`, optional
1709 Date of the observation. Used to correct butler failures
1710 when using fallback filters.
1712 If True, disable butler proxies to enable error handling
1713 within this routine.
1717 exposure : `lsst.afw.image.Exposure`
1718 Requested calibration frame.
1723 Raised if no matching calibration frame can be found.
1726 exp = dataRef.get(datasetType, immediate=immediate)
1727 except Exception
as exc1:
1728 if not self.config.fallbackFilterName:
1729 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1731 if self.config.useFallbackDate
and dateObs:
1732 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1733 dateObs=dateObs, immediate=immediate)
1735 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1736 except Exception
as exc2:
1737 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1738 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1739 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1741 if self.config.doAssembleIsrExposures:
1742 exp = self.assembleCcd.assembleCcd(exp)
1746 """Ensure that the data returned by Butler is a fully constructed exposure.
1748 ISR requires exposure-level image data for historical reasons, so if we did
1749 not recieve that from Butler, construct it from what we have, modifying the
1754 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1755 `lsst.afw.image.ImageF`
1756 The input data structure obtained from Butler.
1757 camera : `lsst.afw.cameraGeom.camera`
1758 The camera associated with the image. Used to find the appropriate
1761 The detector this exposure should match.
1765 inputExp : `lsst.afw.image.Exposure`
1766 The re-constructed exposure, with appropriate detector parameters.
1771 Raised if the input data cannot be used to construct an exposure.
1773 if isinstance(inputExp, afwImage.DecoratedImageU):
1774 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1775 elif isinstance(inputExp, afwImage.ImageF):
1776 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1777 elif isinstance(inputExp, afwImage.MaskedImageF):
1778 inputExp = afwImage.makeExposure(inputExp)
1779 elif isinstance(inputExp, afwImage.Exposure):
1781 elif inputExp
is None:
1785 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1788 if inputExp.getDetector()
is None:
1789 inputExp.setDetector(camera[detectorNum])
1794 """Convert exposure image from uint16 to float.
1796 If the exposure does not need to be converted, the input is
1797 immediately returned. For exposures that are converted to use
1798 floating point pixels, the variance is set to unity and the
1803 exposure : `lsst.afw.image.Exposure`
1804 The raw exposure to be converted.
1808 newexposure : `lsst.afw.image.Exposure`
1809 The input ``exposure``, converted to floating point pixels.
1814 Raised if the exposure type cannot be converted to float.
1817 if isinstance(exposure, afwImage.ExposureF):
1819 self.log.debug(
"Exposure already of type float.")
1821 if not hasattr(exposure,
"convertF"):
1822 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1824 newexposure = exposure.convertF()
1825 newexposure.variance[:] = 1
1826 newexposure.mask[:] = 0x0
1831 """Identify bad amplifiers, saturated and suspect pixels.
1835 ccdExposure : `lsst.afw.image.Exposure`
1836 Input exposure to be masked.
1837 amp : `lsst.afw.table.AmpInfoCatalog`
1838 Catalog of parameters defining the amplifier on this
1840 defects : `lsst.meas.algorithms.Defects`
1841 List of defects. Used to determine if the entire
1847 If this is true, the entire amplifier area is covered by
1848 defects and unusable.
1851 maskedImage = ccdExposure.getMaskedImage()
1857 if defects
is not None:
1858 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1863 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1865 maskView = dataView.getMask()
1866 maskView |= maskView.getPlaneBitMask(
"BAD")
1873 if self.config.doSaturation
and not badAmp:
1874 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1875 if self.config.doSuspect
and not badAmp:
1876 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1877 if math.isfinite(self.config.saturation):
1878 limits.update({self.config.saturatedMaskName: self.config.saturation})
1880 for maskName, maskThreshold
in limits.items():
1881 if not math.isnan(maskThreshold):
1882 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1883 isrFunctions.makeThresholdMask(
1884 maskedImage=dataView,
1885 threshold=maskThreshold,
1891 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1893 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1894 self.config.suspectMaskName])
1895 if numpy.all(maskView.getArray() & maskVal > 0):
1897 maskView |= maskView.getPlaneBitMask(
"BAD")
1902 """Apply overscan correction in place.
1904 This method does initial pixel rejection of the overscan
1905 region. The overscan can also be optionally segmented to
1906 allow for discontinuous overscan responses to be fit
1907 separately. The actual overscan subtraction is performed by
1908 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1909 which is called here after the amplifier is preprocessed.
1913 ccdExposure : `lsst.afw.image.Exposure`
1914 Exposure to have overscan correction performed.
1915 amp : `lsst.afw.cameraGeom.Amplifer`
1916 The amplifier to consider while correcting the overscan.
1920 overscanResults : `lsst.pipe.base.Struct`
1921 Result struct with components:
1922 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1923 Value or fit subtracted from the amplifier image data.
1924 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1925 Value or fit subtracted from the overscan image data.
1926 - ``overscanImage`` : `lsst.afw.image.Image`
1927 Image of the overscan region with the overscan
1928 correction applied. This quantity is used to estimate
1929 the amplifier read noise empirically.
1934 Raised if the ``amp`` does not contain raw pixel information.
1938 lsst.ip.isr.isrFunctions.overscanCorrection
1940 if amp.getRawHorizontalOverscanBBox().isEmpty():
1941 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1944 statControl = afwMath.StatisticsControl()
1945 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1948 dataBBox = amp.getRawDataBBox()
1949 oscanBBox = amp.getRawHorizontalOverscanBBox()
1953 prescanBBox = amp.getRawPrescanBBox()
1954 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1955 dx0 += self.config.overscanNumLeadingColumnsToSkip
1956 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1958 dx0 += self.config.overscanNumTrailingColumnsToSkip
1959 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1965 if ((self.config.overscanBiasJump
1966 and self.config.overscanBiasJumpLocation)
1967 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1968 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
1969 self.config.overscanBiasJumpDevices)):
1970 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1971 yLower = self.config.overscanBiasJumpLocation
1972 yUpper = dataBBox.getHeight() - yLower
1974 yUpper = self.config.overscanBiasJumpLocation
1975 yLower = dataBBox.getHeight() - yUpper
1993 oscanBBox.getHeight())))
1996 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1997 ampImage = ccdExposure.maskedImage[imageBBox]
1998 overscanImage = ccdExposure.maskedImage[overscanBBox]
2000 overscanArray = overscanImage.image.array
2001 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2002 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2003 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2005 statControl = afwMath.StatisticsControl()
2006 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2008 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2011 levelStat = afwMath.MEDIAN
2012 sigmaStat = afwMath.STDEVCLIP
2014 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2015 self.config.qa.flatness.nIter)
2016 metadata = ccdExposure.getMetadata()
2017 ampNum = amp.getName()
2019 if isinstance(overscanResults.overscanFit, float):
2020 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2021 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2023 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2024 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2025 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2027 return overscanResults
2030 """Set the variance plane using the amplifier gain and read noise
2032 The read noise is calculated from the ``overscanImage`` if the
2033 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2034 the value from the amplifier data is used.
2038 ampExposure : `lsst.afw.image.Exposure`
2039 Exposure to process.
2040 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2041 Amplifier detector data.
2042 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2043 Image of overscan, required only for empirical read noise.
2047 lsst.ip.isr.isrFunctions.updateVariance
2049 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2050 gain = amp.getGain()
2052 if math.isnan(gain):
2054 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2057 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2058 amp.getName(), gain, patchedGain)
2061 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2062 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
2064 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2065 stats = afwMath.StatisticsControl()
2066 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2067 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2068 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2069 amp.getName(), readNoise)
2071 readNoise = amp.getReadNoise()
2073 isrFunctions.updateVariance(
2074 maskedImage=ampExposure.getMaskedImage(),
2076 readNoise=readNoise,
2080 """Apply dark correction in place.
2084 exposure : `lsst.afw.image.Exposure`
2085 Exposure to process.
2086 darkExposure : `lsst.afw.image.Exposure`
2087 Dark exposure of the same size as ``exposure``.
2088 invert : `Bool`, optional
2089 If True, re-add the dark to an already corrected image.
2094 Raised if either ``exposure`` or ``darkExposure`` do not
2095 have their dark time defined.
2099 lsst.ip.isr.isrFunctions.darkCorrection
2101 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2102 if math.isnan(expScale):
2103 raise RuntimeError(
"Exposure darktime is NAN.")
2104 if darkExposure.getInfo().getVisitInfo()
is not None \
2105 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2106 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2110 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2113 isrFunctions.darkCorrection(
2114 maskedImage=exposure.getMaskedImage(),
2115 darkMaskedImage=darkExposure.getMaskedImage(),
2117 darkScale=darkScale,
2119 trimToFit=self.config.doTrimToMatchCalib
2123 """Check if linearization is needed for the detector cameraGeom.
2125 Checks config.doLinearize and the linearity type of the first
2130 detector : `lsst.afw.cameraGeom.Detector`
2131 Detector to get linearity type from.
2135 doLinearize : `Bool`
2136 If True, linearization should be performed.
2138 return self.config.doLinearize
and \
2139 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2142 """Apply flat correction in place.
2146 exposure : `lsst.afw.image.Exposure`
2147 Exposure to process.
2148 flatExposure : `lsst.afw.image.Exposure`
2149 Flat exposure of the same size as ``exposure``.
2150 invert : `Bool`, optional
2151 If True, unflatten an already flattened image.
2155 lsst.ip.isr.isrFunctions.flatCorrection
2157 isrFunctions.flatCorrection(
2158 maskedImage=exposure.getMaskedImage(),
2159 flatMaskedImage=flatExposure.getMaskedImage(),
2160 scalingType=self.config.flatScalingType,
2161 userScale=self.config.flatUserScale,
2163 trimToFit=self.config.doTrimToMatchCalib
2167 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2171 exposure : `lsst.afw.image.Exposure`
2172 Exposure to process. Only the amplifier DataSec is processed.
2173 amp : `lsst.afw.table.AmpInfoCatalog`
2174 Amplifier detector data.
2178 lsst.ip.isr.isrFunctions.makeThresholdMask
2180 if not math.isnan(amp.getSaturation()):
2181 maskedImage = exposure.getMaskedImage()
2182 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2183 isrFunctions.makeThresholdMask(
2184 maskedImage=dataView,
2185 threshold=amp.getSaturation(),
2187 maskName=self.config.saturatedMaskName,
2191 """Interpolate over saturated pixels, in place.
2193 This method should be called after `saturationDetection`, to
2194 ensure that the saturated pixels have been identified in the
2195 SAT mask. It should also be called after `assembleCcd`, since
2196 saturated regions may cross amplifier boundaries.
2200 exposure : `lsst.afw.image.Exposure`
2201 Exposure to process.
2205 lsst.ip.isr.isrTask.saturationDetection
2206 lsst.ip.isr.isrFunctions.interpolateFromMask
2208 isrFunctions.interpolateFromMask(
2209 maskedImage=exposure.getMaskedImage(),
2210 fwhm=self.config.fwhm,
2211 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2212 maskNameList=list(self.config.saturatedMaskName),
2216 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2220 exposure : `lsst.afw.image.Exposure`
2221 Exposure to process. Only the amplifier DataSec is processed.
2222 amp : `lsst.afw.table.AmpInfoCatalog`
2223 Amplifier detector data.
2227 lsst.ip.isr.isrFunctions.makeThresholdMask
2231 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2232 This is intended to indicate pixels that may be affected by unknown systematics;
2233 for example if non-linearity corrections above a certain level are unstable
2234 then that would be a useful value for suspectLevel. A value of `nan` indicates
2235 that no such level exists and no pixels are to be masked as suspicious.
2237 suspectLevel = amp.getSuspectLevel()
2238 if math.isnan(suspectLevel):
2241 maskedImage = exposure.getMaskedImage()
2242 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2243 isrFunctions.makeThresholdMask(
2244 maskedImage=dataView,
2245 threshold=suspectLevel,
2247 maskName=self.config.suspectMaskName,
2251 """Mask defects using mask plane "BAD", in place.
2255 exposure : `lsst.afw.image.Exposure`
2256 Exposure to process.
2257 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2258 `lsst.afw.image.DefectBase`.
2259 List of defects to mask.
2263 Call this after CCD assembly, since defects may cross amplifier boundaries.
2265 maskedImage = exposure.getMaskedImage()
2266 if not isinstance(defectBaseList, Defects):
2268 defectList = Defects(defectBaseList)
2270 defectList = defectBaseList
2271 defectList.maskPixels(maskedImage, maskName=
"BAD")
2273 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2274 """Mask edge pixels with applicable mask plane.
2278 exposure : `lsst.afw.image.Exposure`
2279 Exposure to process.
2280 numEdgePixels : `int`, optional
2281 Number of edge pixels to mask.
2282 maskPlane : `str`, optional
2283 Mask plane name to use.
2284 level : `str`, optional
2285 Level at which to mask edges.
2287 maskedImage = exposure.getMaskedImage()
2288 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2290 if numEdgePixels > 0:
2291 if level ==
'DETECTOR':
2292 boxes = [maskedImage.getBBox()]
2293 elif level ==
'AMP':
2294 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2298 subImage = maskedImage[box]
2299 box.grow(-numEdgePixels)
2301 SourceDetectionTask.setEdgeBits(
2307 """Mask and interpolate defects using mask plane "BAD", in place.
2311 exposure : `lsst.afw.image.Exposure`
2312 Exposure to process.
2313 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2314 `lsst.afw.image.DefectBase`.
2315 List of defects to mask and interpolate.
2319 lsst.ip.isr.isrTask.maskDefect
2322 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2323 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2324 isrFunctions.interpolateFromMask(
2325 maskedImage=exposure.getMaskedImage(),
2326 fwhm=self.config.fwhm,
2327 growSaturatedFootprints=0,
2328 maskNameList=[
"BAD"],
2332 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2336 exposure : `lsst.afw.image.Exposure`
2337 Exposure to process.
2341 We mask over all NaNs, including those that are masked with
2342 other bits (because those may or may not be interpolated over
2343 later, and we want to remove all NaNs). Despite this
2344 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2345 the historical name.
2347 maskedImage = exposure.getMaskedImage()
2350 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2351 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2352 numNans =
maskNans(maskedImage, maskVal)
2353 self.metadata.set(
"NUMNANS", numNans)
2355 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2358 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2362 exposure : `lsst.afw.image.Exposure`
2363 Exposure to process.
2367 lsst.ip.isr.isrTask.maskNan
2370 isrFunctions.interpolateFromMask(
2371 maskedImage=exposure.getMaskedImage(),
2372 fwhm=self.config.fwhm,
2373 growSaturatedFootprints=0,
2374 maskNameList=[
"UNMASKEDNAN"],
2378 """Measure the image background in subgrids, for quality control purposes.
2382 exposure : `lsst.afw.image.Exposure`
2383 Exposure to process.
2384 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2385 Configuration object containing parameters on which background
2386 statistics and subgrids to use.
2388 if IsrQaConfig
is not None:
2389 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2390 IsrQaConfig.flatness.nIter)
2391 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2392 statsControl.setAndMask(maskVal)
2393 maskedImage = exposure.getMaskedImage()
2394 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2395 skyLevel = stats.getValue(afwMath.MEDIAN)
2396 skySigma = stats.getValue(afwMath.STDEVCLIP)
2397 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2398 metadata = exposure.getMetadata()
2399 metadata.set(
'SKYLEVEL', skyLevel)
2400 metadata.set(
'SKYSIGMA', skySigma)
2403 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2404 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2405 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2406 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2407 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2408 skyLevels = numpy.zeros((nX, nY))
2411 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2413 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2415 xLLC = xc - meshXHalf
2416 yLLC = yc - meshYHalf
2417 xURC = xc + meshXHalf - 1
2418 yURC = yc + meshYHalf - 1
2421 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2423 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2425 good = numpy.where(numpy.isfinite(skyLevels))
2426 skyMedian = numpy.median(skyLevels[good])
2427 flatness = (skyLevels[good] - skyMedian) / skyMedian
2428 flatness_rms = numpy.std(flatness)
2429 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2431 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2432 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2433 nX, nY, flatness_pp, flatness_rms)
2435 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2436 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2437 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2438 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2439 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2442 """Set an approximate magnitude zero point for the exposure.
2446 exposure : `lsst.afw.image.Exposure`
2447 Exposure to process.
2449 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2450 if filterName
in self.config.fluxMag0T1:
2451 fluxMag0 = self.config.fluxMag0T1[filterName]
2453 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2454 fluxMag0 = self.config.defaultFluxMag0T1
2456 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2458 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2461 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2462 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2465 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2469 ccdExposure : `lsst.afw.image.Exposure`
2470 Exposure to process.
2471 fpPolygon : `lsst.afw.geom.Polygon`
2472 Polygon in focal plane coordinates.
2475 ccd = ccdExposure.getDetector()
2476 fpCorners = ccd.getCorners(FOCAL_PLANE)
2477 ccdPolygon = Polygon(fpCorners)
2480 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2483 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2484 validPolygon = Polygon(ccdPoints)
2485 ccdExposure.getInfo().setValidPolygon(validPolygon)
2489 """Context manager that applies and removes flats and darks,
2490 if the task is configured to apply them.
2494 exp : `lsst.afw.image.Exposure`
2495 Exposure to process.
2496 flat : `lsst.afw.image.Exposure`
2497 Flat exposure the same size as ``exp``.
2498 dark : `lsst.afw.image.Exposure`, optional
2499 Dark exposure the same size as ``exp``.
2503 exp : `lsst.afw.image.Exposure`
2504 The flat and dark corrected exposure.
2506 if self.config.doDark
and dark
is not None:
2508 if self.config.doFlat:
2513 if self.config.doFlat:
2515 if self.config.doDark
and dark
is not None:
2519 """Utility function to examine ISR exposure at different stages.
2523 exposure : `lsst.afw.image.Exposure`
2526 State of processing to view.
2528 frame = getDebugFrame(self._display, stepname)
2530 display = getDisplay(frame)
2531 display.scale(
'asinh',
'zscale')
2532 display.mtv(exposure)
2533 prompt =
"Press Enter to continue [c]... "
2535 ans = input(prompt).lower()
2536 if ans
in (
"",
"c",):
2541 """A Detector-like object that supports returning gain and saturation level
2543 This is used when the input exposure does not have a detector.
2547 exposure : `lsst.afw.image.Exposure`
2548 Exposure to generate a fake amplifier for.
2549 config : `lsst.ip.isr.isrTaskConfig`
2550 Configuration to apply to the fake amplifier.
2554 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2556 self.
_gain = config.gain
2583 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2587 """Task to wrap the default IsrTask to allow it to be retargeted.
2589 The standard IsrTask can be called directly from a command line
2590 program, but doing so removes the ability of the task to be
2591 retargeted. As most cameras override some set of the IsrTask
2592 methods, this would remove those data-specific methods in the
2593 output post-ISR images. This wrapping class fixes the issue,
2594 allowing identical post-ISR images to be generated by both the
2595 processCcd and isrTask code.
2597 ConfigClass = RunIsrConfig
2598 _DefaultName =
"runIsr"
2602 self.makeSubtask(
"isr")
2608 dataRef : `lsst.daf.persistence.ButlerDataRef`
2609 data reference of the detector data to be processed
2613 result : `pipeBase.Struct`
2614 Result struct with component:
2616 - exposure : `lsst.afw.image.Exposure`
2617 Post-ISR processed exposure.