32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
134 bias = cT.PrerequisiteInput(
136 doc=
"Input bias calibration.",
137 storageClass=
"ExposureF",
138 dimensions=[
"instrument",
"detector"],
141 dark = cT.PrerequisiteInput(
143 doc=
"Input dark calibration.",
144 storageClass=
"ExposureF",
145 dimensions=[
"instrument",
"detector"],
148 flat = cT.PrerequisiteInput(
150 doc=
"Input flat calibration.",
151 storageClass=
"ExposureF",
152 dimensions=[
"instrument",
"physical_filter",
"detector"],
155 fringes = cT.PrerequisiteInput(
157 doc=
"Input fringe calibration.",
158 storageClass=
"ExposureF",
159 dimensions=[
"instrument",
"physical_filter",
"detector"],
162 strayLightData = cT.PrerequisiteInput(
164 doc=
"Input stray light calibration.",
165 storageClass=
"StrayLightData",
166 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 bfKernel = cT.PrerequisiteInput(
171 doc=
"Input brighter-fatter kernel.",
172 storageClass=
"NumpyArray",
173 dimensions=[
"instrument"],
176 newBFKernel = cT.PrerequisiteInput(
177 name=
'brighterFatterKernel',
178 doc=
"Newer complete kernel + gain solutions.",
179 storageClass=
"BrighterFatterKernel",
180 dimensions=[
"instrument",
"detector"],
183 defects = cT.PrerequisiteInput(
185 doc=
"Input defect tables.",
186 storageClass=
"Defects",
187 dimensions=[
"instrument",
"detector"],
190 opticsTransmission = cT.PrerequisiteInput(
191 name=
"transmission_optics",
192 storageClass=
"TransmissionCurve",
193 doc=
"Transmission curve due to the optics.",
194 dimensions=[
"instrument"],
197 filterTransmission = cT.PrerequisiteInput(
198 name=
"transmission_filter",
199 storageClass=
"TransmissionCurve",
200 doc=
"Transmission curve due to the filter.",
201 dimensions=[
"instrument",
"physical_filter"],
204 sensorTransmission = cT.PrerequisiteInput(
205 name=
"transmission_sensor",
206 storageClass=
"TransmissionCurve",
207 doc=
"Transmission curve due to the sensor.",
208 dimensions=[
"instrument",
"detector"],
211 atmosphereTransmission = cT.PrerequisiteInput(
212 name=
"transmission_atmosphere",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the atmosphere.",
215 dimensions=[
"instrument"],
218 illumMaskedImage = cT.PrerequisiteInput(
220 doc=
"Input illumination correction.",
221 storageClass=
"MaskedImageF",
222 dimensions=[
"instrument",
"physical_filter",
"detector"],
226 outputExposure = cT.Output(
228 doc=
"Output ISR processed exposure.",
229 storageClass=
"Exposure",
230 dimensions=[
"instrument",
"exposure",
"detector"],
232 preInterpExposure = cT.Output(
233 name=
'preInterpISRCCD',
234 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
235 storageClass=
"ExposureF",
236 dimensions=[
"instrument",
"exposure",
"detector"],
238 outputOssThumbnail = cT.Output(
240 doc=
"Output Overscan-subtracted thumbnail image.",
241 storageClass=
"Thumbnail",
242 dimensions=[
"instrument",
"exposure",
"detector"],
244 outputFlattenedThumbnail = cT.Output(
245 name=
"FlattenedThumb",
246 doc=
"Output flat-corrected thumbnail image.",
247 storageClass=
"Thumbnail",
248 dimensions=[
"instrument",
"exposure",
"detector"],
254 if config.doBias
is not True:
255 self.prerequisiteInputs.discard(
"bias")
256 if config.doLinearize
is not True:
257 self.prerequisiteInputs.discard(
"linearizer")
258 if config.doCrosstalk
is not True:
259 self.inputs.discard(
"crosstalkSources")
260 self.prerequisiteInputs.discard(
"crosstalk")
261 if config.doBrighterFatter
is not True:
262 self.prerequisiteInputs.discard(
"bfKernel")
263 self.prerequisiteInputs.discard(
"newBFKernel")
264 if config.doDefect
is not True:
265 self.prerequisiteInputs.discard(
"defects")
266 if config.doDark
is not True:
267 self.prerequisiteInputs.discard(
"dark")
268 if config.doFlat
is not True:
269 self.prerequisiteInputs.discard(
"flat")
270 if config.doAttachTransmissionCurve
is not True:
271 self.prerequisiteInputs.discard(
"opticsTransmission")
272 self.prerequisiteInputs.discard(
"filterTransmission")
273 self.prerequisiteInputs.discard(
"sensorTransmission")
274 self.prerequisiteInputs.discard(
"atmosphereTransmission")
275 if config.doUseOpticsTransmission
is not True:
276 self.prerequisiteInputs.discard(
"opticsTransmission")
277 if config.doUseFilterTransmission
is not True:
278 self.prerequisiteInputs.discard(
"filterTransmission")
279 if config.doUseSensorTransmission
is not True:
280 self.prerequisiteInputs.discard(
"sensorTransmission")
281 if config.doUseAtmosphereTransmission
is not True:
282 self.prerequisiteInputs.discard(
"atmosphereTransmission")
283 if config.doIlluminationCorrection
is not True:
284 self.prerequisiteInputs.discard(
"illumMaskedImage")
286 if config.doWrite
is not True:
287 self.outputs.discard(
"outputExposure")
288 self.outputs.discard(
"preInterpExposure")
289 self.outputs.discard(
"outputFlattenedThumbnail")
290 self.outputs.discard(
"outputOssThumbnail")
291 if config.doSaveInterpPixels
is not True:
292 self.outputs.discard(
"preInterpExposure")
293 if config.qa.doThumbnailOss
is not True:
294 self.outputs.discard(
"outputOssThumbnail")
295 if config.qa.doThumbnailFlattened
is not True:
296 self.outputs.discard(
"outputFlattenedThumbnail")
300 pipelineConnections=IsrTaskConnections):
301 """Configuration parameters for IsrTask.
303 Items are grouped in the order in which they are executed by the task.
305 datasetType = pexConfig.Field(
307 doc=
"Dataset type for input data; users will typically leave this alone, "
308 "but camera-specific ISR tasks will override it",
312 fallbackFilterName = pexConfig.Field(
314 doc=
"Fallback default filter name for calibrations.",
317 useFallbackDate = pexConfig.Field(
319 doc=
"Pass observation date when using fallback filter.",
322 expectWcs = pexConfig.Field(
325 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
327 fwhm = pexConfig.Field(
329 doc=
"FWHM of PSF in arcseconds.",
332 qa = pexConfig.ConfigField(
334 doc=
"QA related configuration options.",
338 doConvertIntToFloat = pexConfig.Field(
340 doc=
"Convert integer raw images to floating point values?",
345 doSaturation = pexConfig.Field(
347 doc=
"Mask saturated pixels? NB: this is totally independent of the"
348 " interpolation option - this is ONLY setting the bits in the mask."
349 " To have them interpolated make sure doSaturationInterpolation=True",
352 saturatedMaskName = pexConfig.Field(
354 doc=
"Name of mask plane to use in saturation detection and interpolation",
357 saturation = pexConfig.Field(
359 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
360 default=float(
"NaN"),
362 growSaturationFootprintSize = pexConfig.Field(
364 doc=
"Number of pixels by which to grow the saturation footprints",
369 doSuspect = pexConfig.Field(
371 doc=
"Mask suspect pixels?",
374 suspectMaskName = pexConfig.Field(
376 doc=
"Name of mask plane to use for suspect pixels",
379 numEdgeSuspect = pexConfig.Field(
381 doc=
"Number of edge pixels to be flagged as untrustworthy.",
384 edgeMaskLevel = pexConfig.ChoiceField(
386 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
389 'DETECTOR':
'Mask only the edges of the full detector.',
390 'AMP':
'Mask edges of each amplifier.',
395 doSetBadRegions = pexConfig.Field(
397 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
400 badStatistic = pexConfig.ChoiceField(
402 doc=
"How to estimate the average value for BAD regions.",
405 "MEANCLIP":
"Correct using the (clipped) mean of good data",
406 "MEDIAN":
"Correct using the median of the good data",
411 doOverscan = pexConfig.Field(
413 doc=
"Do overscan subtraction?",
416 overscan = pexConfig.ConfigurableField(
417 target=OverscanCorrectionTask,
418 doc=
"Overscan subtraction task for image segments.",
421 overscanFitType = pexConfig.ChoiceField(
423 doc=
"The method for fitting the overscan bias level.",
426 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
427 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
428 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
429 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
430 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
431 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
432 "MEAN":
"Correct using the mean of the overscan region",
433 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
434 "MEDIAN":
"Correct using the median of the overscan region",
435 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
437 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
438 " This option will no longer be used, and will be removed after v20.")
440 overscanOrder = pexConfig.Field(
442 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
443 "or number of spline knots if overscan fit type is a spline."),
445 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
446 " This option will no longer be used, and will be removed after v20.")
448 overscanNumSigmaClip = pexConfig.Field(
450 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
452 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
453 " This option will no longer be used, and will be removed after v20.")
455 overscanIsInt = pexConfig.Field(
457 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
458 " and overscan.FitType=MEDIAN_PER_ROW.",
460 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
461 " This option will no longer be used, and will be removed after v20.")
464 overscanNumLeadingColumnsToSkip = pexConfig.Field(
466 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
469 overscanNumTrailingColumnsToSkip = pexConfig.Field(
471 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
474 overscanMaxDev = pexConfig.Field(
476 doc=
"Maximum deviation from the median for overscan",
477 default=1000.0, check=
lambda x: x > 0
479 overscanBiasJump = pexConfig.Field(
481 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
484 overscanBiasJumpKeyword = pexConfig.Field(
486 doc=
"Header keyword containing information about devices.",
487 default=
"NO_SUCH_KEY",
489 overscanBiasJumpDevices = pexConfig.ListField(
491 doc=
"List of devices that need piecewise overscan correction.",
494 overscanBiasJumpLocation = pexConfig.Field(
496 doc=
"Location of bias jump along y-axis.",
501 doAssembleCcd = pexConfig.Field(
504 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
506 assembleCcd = pexConfig.ConfigurableField(
507 target=AssembleCcdTask,
508 doc=
"CCD assembly task",
512 doAssembleIsrExposures = pexConfig.Field(
515 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
517 doTrimToMatchCalib = pexConfig.Field(
520 doc=
"Trim raw data to match calibration bounding boxes?"
524 doBias = pexConfig.Field(
526 doc=
"Apply bias frame correction?",
529 biasDataProductName = pexConfig.Field(
531 doc=
"Name of the bias data product",
534 doBiasBeforeOverscan = pexConfig.Field(
536 doc=
"Reverse order of overscan and bias correction.",
541 doVariance = pexConfig.Field(
543 doc=
"Calculate variance?",
546 gain = pexConfig.Field(
548 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
549 default=float(
"NaN"),
551 readNoise = pexConfig.Field(
553 doc=
"The read noise to use if no Detector is present in the Exposure",
556 doEmpiricalReadNoise = pexConfig.Field(
559 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
563 doLinearize = pexConfig.Field(
565 doc=
"Correct for nonlinearity of the detector's response?",
570 doCrosstalk = pexConfig.Field(
572 doc=
"Apply intra-CCD crosstalk correction?",
575 doCrosstalkBeforeAssemble = pexConfig.Field(
577 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
580 crosstalk = pexConfig.ConfigurableField(
581 target=CrosstalkTask,
582 doc=
"Intra-CCD crosstalk correction",
586 doDefect = pexConfig.Field(
588 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
591 doNanMasking = pexConfig.Field(
593 doc=
"Mask NAN pixels?",
596 doWidenSaturationTrails = pexConfig.Field(
598 doc=
"Widen bleed trails based on their width?",
603 doBrighterFatter = pexConfig.Field(
606 doc=
"Apply the brighter fatter correction"
608 brighterFatterLevel = pexConfig.ChoiceField(
611 doc=
"The level at which to correct for brighter-fatter.",
613 "AMP":
"Every amplifier treated separately.",
614 "DETECTOR":
"One kernel per detector",
617 brighterFatterMaxIter = pexConfig.Field(
620 doc=
"Maximum number of iterations for the brighter fatter correction"
622 brighterFatterThreshold = pexConfig.Field(
625 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
626 " absolute value of the difference between the current corrected image and the one"
627 " from the previous iteration summed over all the pixels."
629 brighterFatterApplyGain = pexConfig.Field(
632 doc=
"Should the gain be applied when applying the brighter fatter correction?"
634 brighterFatterMaskGrowSize = pexConfig.Field(
637 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
638 " when brighter-fatter correction is applied."
642 doDark = pexConfig.Field(
644 doc=
"Apply dark frame correction?",
647 darkDataProductName = pexConfig.Field(
649 doc=
"Name of the dark data product",
654 doStrayLight = pexConfig.Field(
656 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
659 strayLight = pexConfig.ConfigurableField(
660 target=StrayLightTask,
661 doc=
"y-band stray light correction"
665 doFlat = pexConfig.Field(
667 doc=
"Apply flat field correction?",
670 flatDataProductName = pexConfig.Field(
672 doc=
"Name of the flat data product",
675 flatScalingType = pexConfig.ChoiceField(
677 doc=
"The method for scaling the flat on the fly.",
680 "USER":
"Scale by flatUserScale",
681 "MEAN":
"Scale by the inverse of the mean",
682 "MEDIAN":
"Scale by the inverse of the median",
685 flatUserScale = pexConfig.Field(
687 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
690 doTweakFlat = pexConfig.Field(
692 doc=
"Tweak flats to match observed amplifier ratios?",
697 doApplyGains = pexConfig.Field(
699 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
702 normalizeGains = pexConfig.Field(
704 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
709 doFringe = pexConfig.Field(
711 doc=
"Apply fringe correction?",
714 fringe = pexConfig.ConfigurableField(
716 doc=
"Fringe subtraction task",
718 fringeAfterFlat = pexConfig.Field(
720 doc=
"Do fringe subtraction after flat-fielding?",
725 doMeasureBackground = pexConfig.Field(
727 doc=
"Measure the background level on the reduced image?",
732 doCameraSpecificMasking = pexConfig.Field(
734 doc=
"Mask camera-specific bad regions?",
737 masking = pexConfig.ConfigurableField(
744 doInterpolate = pexConfig.Field(
746 doc=
"Interpolate masked pixels?",
749 doSaturationInterpolation = pexConfig.Field(
751 doc=
"Perform interpolation over pixels masked as saturated?"
752 " NB: This is independent of doSaturation; if that is False this plane"
753 " will likely be blank, resulting in a no-op here.",
756 doNanInterpolation = pexConfig.Field(
758 doc=
"Perform interpolation over pixels masked as NaN?"
759 " NB: This is independent of doNanMasking; if that is False this plane"
760 " will likely be blank, resulting in a no-op here.",
763 doNanInterpAfterFlat = pexConfig.Field(
765 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
766 "also have to interpolate them before flat-fielding."),
769 maskListToInterpolate = pexConfig.ListField(
771 doc=
"List of mask planes that should be interpolated.",
772 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
774 doSaveInterpPixels = pexConfig.Field(
776 doc=
"Save a copy of the pre-interpolated pixel values?",
781 fluxMag0T1 = pexConfig.DictField(
784 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
785 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
788 defaultFluxMag0T1 = pexConfig.Field(
790 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
791 default=pow(10.0, 0.4*28.0)
795 doVignette = pexConfig.Field(
797 doc=
"Apply vignetting parameters?",
800 vignette = pexConfig.ConfigurableField(
802 doc=
"Vignetting task.",
806 doAttachTransmissionCurve = pexConfig.Field(
809 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
811 doUseOpticsTransmission = pexConfig.Field(
814 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
816 doUseFilterTransmission = pexConfig.Field(
819 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
821 doUseSensorTransmission = pexConfig.Field(
824 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
826 doUseAtmosphereTransmission = pexConfig.Field(
829 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
833 doIlluminationCorrection = pexConfig.Field(
836 doc=
"Perform illumination correction?"
838 illuminationCorrectionDataProductName = pexConfig.Field(
840 doc=
"Name of the illumination correction data product.",
843 illumScale = pexConfig.Field(
845 doc=
"Scale factor for the illumination correction.",
848 illumFilters = pexConfig.ListField(
851 doc=
"Only perform illumination correction for these filters."
855 doWrite = pexConfig.Field(
857 doc=
"Persist postISRCCD?",
864 raise ValueError(
"You may not specify both doFlat and doApplyGains")
866 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
875 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
876 """Apply common instrument signature correction algorithms to a raw frame.
878 The process for correcting imaging data is very similar from
879 camera to camera. This task provides a vanilla implementation of
880 doing these corrections, including the ability to turn certain
881 corrections off if they are not needed. The inputs to the primary
882 method, `run()`, are a raw exposure to be corrected and the
883 calibration data products. The raw input is a single chip sized
884 mosaic of all amps including overscans and other non-science
885 pixels. The method `runDataRef()` identifies and defines the
886 calibration data products, and is intended for use by a
887 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
888 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
889 subclassed for different camera, although the most camera specific
890 methods have been split into subtasks that can be redirected
893 The __init__ method sets up the subtasks for ISR processing, using
894 the defaults from `lsst.ip.isr`.
899 Positional arguments passed to the Task constructor. None used at this time.
900 kwargs : `dict`, optional
901 Keyword arguments passed on to the Task constructor. None used at this time.
903 ConfigClass = IsrTaskConfig
908 self.makeSubtask(
"assembleCcd")
909 self.makeSubtask(
"crosstalk")
910 self.makeSubtask(
"strayLight")
911 self.makeSubtask(
"fringe")
912 self.makeSubtask(
"masking")
913 self.makeSubtask(
"overscan")
914 self.makeSubtask(
"vignette")
917 inputs = butlerQC.get(inputRefs)
920 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
921 except Exception
as e:
922 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
925 inputs[
'isGen3'] =
True
927 detector = inputs[
'ccdExposure'].getDetector()
929 if self.config.doCrosstalk
is True:
932 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
933 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
934 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
936 coeffVector = (self.config.crosstalk.crosstalkValues
937 if self.config.crosstalk.useConfigCoefficients
else None)
938 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
939 inputs[
'crosstalk'] = crosstalkCalib
940 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
941 if 'crosstalkSources' not in inputs:
942 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
945 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
947 linearizer.fromYaml(inputs[
'linearizer'])
951 inputs[
'linearizer'] = linearizer
953 if self.config.doDefect
is True:
954 if "defects" in inputs
and inputs[
'defects']
is not None:
957 if not isinstance(inputs[
"defects"], Defects):
958 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
962 if self.config.doBrighterFatter:
963 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
964 if brighterFatterKernel
is None:
965 brighterFatterKernel = inputs.get(
'bfKernel',
None)
967 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
968 detId = detector.getId()
969 inputs[
'bfGains'] = brighterFatterKernel.gain
972 if self.config.brighterFatterLevel ==
'DETECTOR':
973 if brighterFatterKernel.detectorKernel:
974 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
975 elif brighterFatterKernel.detectorKernelFromAmpKernels:
976 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
978 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
981 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
983 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
984 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
985 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
987 assembler=self.assembleCcd
988 if self.config.doAssembleIsrExposures
else None)
990 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
992 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
993 if 'strayLightData' not in inputs:
994 inputs[
'strayLightData'] =
None
996 outputs = self.
runrun(**inputs)
997 butlerQC.put(outputs, outputRefs)
1000 """Retrieve necessary frames for instrument signature removal.
1002 Pre-fetching all required ISR data products limits the IO
1003 required by the ISR. Any conflict between the calibration data
1004 available and that needed for ISR is also detected prior to
1005 doing processing, allowing it to fail quickly.
1009 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1010 Butler reference of the detector data to be processed
1011 rawExposure : `afw.image.Exposure`
1012 The raw exposure that will later be corrected with the
1013 retrieved calibration data; should not be modified in this
1018 result : `lsst.pipe.base.Struct`
1019 Result struct with components (which may be `None`):
1020 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1021 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1022 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1023 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1024 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1025 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1026 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1027 - ``fringes``: `lsst.pipe.base.Struct` with components:
1028 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1029 - ``seed``: random seed derived from the ccdExposureId for random
1030 number generator (`uint32`).
1031 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1032 A ``TransmissionCurve`` that represents the throughput of the optics,
1033 to be evaluated in focal-plane coordinates.
1034 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1035 A ``TransmissionCurve`` that represents the throughput of the filter
1036 itself, to be evaluated in focal-plane coordinates.
1037 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1038 A ``TransmissionCurve`` that represents the throughput of the sensor
1039 itself, to be evaluated in post-assembly trimmed detector coordinates.
1040 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1041 A ``TransmissionCurve`` that represents the throughput of the
1042 atmosphere, assumed to be spatially constant.
1043 - ``strayLightData`` : `object`
1044 An opaque object containing calibration information for
1045 stray-light correction. If `None`, no correction will be
1047 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1051 NotImplementedError :
1052 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1055 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1056 dateObs = dateObs.toPython().isoformat()
1057 except RuntimeError:
1058 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1061 ccd = rawExposure.getDetector()
1063 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
1064 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1065 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1066 if self.config.doBias
else None)
1068 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1070 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1071 linearizer.log = self.log
1072 if isinstance(linearizer, numpy.ndarray):
1075 crosstalkCalib =
None
1076 if self.config.doCrosstalk:
1078 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1080 coeffVector = (self.config.crosstalk.crosstalkValues
1081 if self.config.crosstalk.useConfigCoefficients
else None)
1082 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1083 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1084 if self.config.doCrosstalk
else None)
1086 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1087 if self.config.doDark
else None)
1088 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1090 if self.config.doFlat
else None)
1092 brighterFatterKernel =
None
1093 brighterFatterGains =
None
1094 if self.config.doBrighterFatter
is True:
1099 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1100 brighterFatterGains = brighterFatterKernel.gain
1101 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1104 brighterFatterKernel = dataRef.get(
"bfKernel")
1105 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
1107 brighterFatterKernel =
None
1108 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1111 if self.config.brighterFatterLevel ==
'DETECTOR':
1112 if brighterFatterKernel.detectorKernel:
1113 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1114 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1115 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1117 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1120 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1122 defectList = (dataRef.get(
"defects")
1123 if self.config.doDefect
else None)
1124 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1125 if self.config.doAssembleIsrExposures
else None)
1126 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1127 else pipeBase.Struct(fringes=
None))
1129 if self.config.doAttachTransmissionCurve:
1130 opticsTransmission = (dataRef.get(
"transmission_optics")
1131 if self.config.doUseOpticsTransmission
else None)
1132 filterTransmission = (dataRef.get(
"transmission_filter")
1133 if self.config.doUseFilterTransmission
else None)
1134 sensorTransmission = (dataRef.get(
"transmission_sensor")
1135 if self.config.doUseSensorTransmission
else None)
1136 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1137 if self.config.doUseAtmosphereTransmission
else None)
1139 opticsTransmission =
None
1140 filterTransmission =
None
1141 sensorTransmission =
None
1142 atmosphereTransmission =
None
1144 if self.config.doStrayLight:
1145 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1147 strayLightData =
None
1150 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1151 if (self.config.doIlluminationCorrection
1152 and filterName
in self.config.illumFilters)
1156 return pipeBase.Struct(bias=biasExposure,
1157 linearizer=linearizer,
1158 crosstalk=crosstalkCalib,
1159 crosstalkSources=crosstalkSources,
1162 bfKernel=brighterFatterKernel,
1163 bfGains=brighterFatterGains,
1165 fringes=fringeStruct,
1166 opticsTransmission=opticsTransmission,
1167 filterTransmission=filterTransmission,
1168 sensorTransmission=sensorTransmission,
1169 atmosphereTransmission=atmosphereTransmission,
1170 strayLightData=strayLightData,
1171 illumMaskedImage=illumMaskedImage
1174 @pipeBase.timeMethod
1175 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1176 crosstalk=None, crosstalkSources=None,
1177 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1178 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1179 sensorTransmission=
None, atmosphereTransmission=
None,
1180 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1183 """Perform instrument signature removal on an exposure.
1185 Steps included in the ISR processing, in order performed, are:
1186 - saturation and suspect pixel masking
1187 - overscan subtraction
1188 - CCD assembly of individual amplifiers
1190 - variance image construction
1191 - linearization of non-linear response
1193 - brighter-fatter correction
1196 - stray light subtraction
1198 - masking of known defects and camera specific features
1199 - vignette calculation
1200 - appending transmission curve and distortion model
1204 ccdExposure : `lsst.afw.image.Exposure`
1205 The raw exposure that is to be run through ISR. The
1206 exposure is modified by this method.
1207 camera : `lsst.afw.cameraGeom.Camera`, optional
1208 The camera geometry for this exposure. Required if ``isGen3`` is
1209 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1210 ``flat`` does not have an associated detector.
1211 bias : `lsst.afw.image.Exposure`, optional
1212 Bias calibration frame.
1213 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1214 Functor for linearization.
1215 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1216 Calibration for crosstalk.
1217 crosstalkSources : `list`, optional
1218 List of possible crosstalk sources.
1219 dark : `lsst.afw.image.Exposure`, optional
1220 Dark calibration frame.
1221 flat : `lsst.afw.image.Exposure`, optional
1222 Flat calibration frame.
1223 bfKernel : `numpy.ndarray`, optional
1224 Brighter-fatter kernel.
1225 bfGains : `dict` of `float`, optional
1226 Gains used to override the detector's nominal gains for the
1227 brighter-fatter correction. A dict keyed by amplifier name for
1228 the detector in question.
1229 defects : `lsst.ip.isr.Defects`, optional
1231 fringes : `lsst.pipe.base.Struct`, optional
1232 Struct containing the fringe correction data, with
1234 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1235 - ``seed``: random seed derived from the ccdExposureId for random
1236 number generator (`uint32`)
1237 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1238 A ``TransmissionCurve`` that represents the throughput of the optics,
1239 to be evaluated in focal-plane coordinates.
1240 filterTransmission : `lsst.afw.image.TransmissionCurve`
1241 A ``TransmissionCurve`` that represents the throughput of the filter
1242 itself, to be evaluated in focal-plane coordinates.
1243 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1244 A ``TransmissionCurve`` that represents the throughput of the sensor
1245 itself, to be evaluated in post-assembly trimmed detector coordinates.
1246 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1247 A ``TransmissionCurve`` that represents the throughput of the
1248 atmosphere, assumed to be spatially constant.
1249 detectorNum : `int`, optional
1250 The integer number for the detector to process.
1251 isGen3 : bool, optional
1252 Flag this call to run() as using the Gen3 butler environment.
1253 strayLightData : `object`, optional
1254 Opaque object containing calibration information for stray-light
1255 correction. If `None`, no correction will be performed.
1256 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1257 Illumination correction image.
1261 result : `lsst.pipe.base.Struct`
1262 Result struct with component:
1263 - ``exposure`` : `afw.image.Exposure`
1264 The fully ISR corrected exposure.
1265 - ``outputExposure`` : `afw.image.Exposure`
1266 An alias for `exposure`
1267 - ``ossThumb`` : `numpy.ndarray`
1268 Thumbnail image of the exposure after overscan subtraction.
1269 - ``flattenedThumb`` : `numpy.ndarray`
1270 Thumbnail image of the exposure after flat-field correction.
1275 Raised if a configuration option is set to True, but the
1276 required calibration data has not been specified.
1280 The current processed exposure can be viewed by setting the
1281 appropriate lsstDebug entries in the `debug.display`
1282 dictionary. The names of these entries correspond to some of
1283 the IsrTaskConfig Boolean options, with the value denoting the
1284 frame to use. The exposure is shown inside the matching
1285 option check and after the processing of that step has
1286 finished. The steps with debug points are:
1297 In addition, setting the "postISRCCD" entry displays the
1298 exposure after all ISR processing has finished.
1306 if detectorNum
is None:
1307 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1309 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1310 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1311 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1312 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1314 if isinstance(ccdExposure, ButlerDataRef):
1315 return self.
runDataRefrunDataRef(ccdExposure)
1317 ccd = ccdExposure.getDetector()
1319 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1322 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1323 ccd = [
FakeAmp(ccdExposure, self.config)]
1326 if self.config.doBias
and bias
is None:
1327 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1328 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1329 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1330 if self.config.doBrighterFatter
and bfKernel
is None:
1331 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1332 if self.config.doDark
and dark
is None:
1333 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1334 if self.config.doFlat
and flat
is None:
1335 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1336 if self.config.doDefect
and defects
is None:
1337 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1338 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1339 and fringes.fringes
is None):
1344 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1345 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1346 and illumMaskedImage
is None):
1347 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1350 if self.config.doConvertIntToFloat:
1351 self.log.info(
"Converting exposure to floating point values.")
1354 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1355 self.log.info(
"Applying bias correction.")
1356 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1357 trimToFit=self.config.doTrimToMatchCalib)
1358 self.
debugViewdebugView(ccdExposure,
"doBias")
1364 if ccdExposure.getBBox().contains(amp.getBBox()):
1366 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1368 if self.config.doOverscan
and not badAmp:
1371 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1372 if overscanResults
is not None and \
1373 self.config.qa
is not None and self.config.qa.saveStats
is True:
1374 if isinstance(overscanResults.overscanFit, float):
1375 qaMedian = overscanResults.overscanFit
1376 qaStdev = float(
"NaN")
1378 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1379 afwMath.MEDIAN | afwMath.STDEVCLIP)
1380 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1381 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1383 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1384 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1385 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1386 amp.getName(), qaMedian, qaStdev)
1389 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1390 afwMath.MEDIAN | afwMath.STDEVCLIP)
1391 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1392 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1394 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1395 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1396 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1397 amp.getName(), qaMedianAfter, qaStdevAfter)
1399 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1402 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1403 overscanResults =
None
1405 overscans.append(overscanResults
if overscanResults
is not None else None)
1407 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1409 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1410 self.log.info(
"Applying crosstalk correction.")
1411 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1412 crosstalkSources=crosstalkSources)
1413 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1415 if self.config.doAssembleCcd:
1416 self.log.info(
"Assembling CCD from amplifiers.")
1417 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1419 if self.config.expectWcs
and not ccdExposure.getWcs():
1420 self.log.warn(
"No WCS found in input exposure.")
1421 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1424 if self.config.qa.doThumbnailOss:
1425 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1427 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1428 self.log.info(
"Applying bias correction.")
1429 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1430 trimToFit=self.config.doTrimToMatchCalib)
1431 self.
debugViewdebugView(ccdExposure,
"doBias")
1433 if self.config.doVariance:
1434 for amp, overscanResults
in zip(ccd, overscans):
1435 if ccdExposure.getBBox().contains(amp.getBBox()):
1436 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1437 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1438 if overscanResults
is not None:
1440 overscanImage=overscanResults.overscanImage)
1444 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1445 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1446 afwMath.MEDIAN | afwMath.STDEVCLIP)
1447 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1448 qaStats.getValue(afwMath.MEDIAN))
1449 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1450 qaStats.getValue(afwMath.STDEVCLIP))
1451 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1452 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1453 qaStats.getValue(afwMath.STDEVCLIP))
1456 self.log.info(
"Applying linearizer.")
1457 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1458 detector=ccd, log=self.log)
1460 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1461 self.log.info(
"Applying crosstalk correction.")
1462 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1463 crosstalkSources=crosstalkSources, isTrimmed=
True)
1464 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1468 if self.config.doDefect:
1469 self.log.info(
"Masking defects.")
1470 self.
maskDefectmaskDefect(ccdExposure, defects)
1472 if self.config.numEdgeSuspect > 0:
1473 self.log.info(
"Masking edges as SUSPECT.")
1474 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1475 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1477 if self.config.doNanMasking:
1478 self.log.info(
"Masking NAN value pixels.")
1479 self.
maskNanmaskNan(ccdExposure)
1481 if self.config.doWidenSaturationTrails:
1482 self.log.info(
"Widening saturation trails.")
1483 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1485 if self.config.doCameraSpecificMasking:
1486 self.log.info(
"Masking regions for camera specific reasons.")
1487 self.masking.
run(ccdExposure)
1489 if self.config.doBrighterFatter:
1498 interpExp = ccdExposure.clone()
1499 with self.
flatContextflatContext(interpExp, flat, dark):
1500 isrFunctions.interpolateFromMask(
1501 maskedImage=interpExp.getMaskedImage(),
1502 fwhm=self.config.fwhm,
1503 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1504 maskNameList=self.config.maskListToInterpolate
1506 bfExp = interpExp.clone()
1508 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1509 type(bfKernel), type(bfGains))
1510 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1511 self.config.brighterFatterMaxIter,
1512 self.config.brighterFatterThreshold,
1513 self.config.brighterFatterApplyGain,
1515 if bfResults[1] == self.config.brighterFatterMaxIter:
1516 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1519 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1521 image = ccdExposure.getMaskedImage().getImage()
1522 bfCorr = bfExp.getMaskedImage().getImage()
1523 bfCorr -= interpExp.getMaskedImage().getImage()
1532 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1533 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1536 if self.config.brighterFatterMaskGrowSize > 0:
1537 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1538 for maskPlane
in self.config.maskListToInterpolate:
1539 isrFunctions.growMasks(ccdExposure.getMask(),
1540 radius=self.config.brighterFatterMaskGrowSize,
1541 maskNameList=maskPlane,
1542 maskValue=maskPlane)
1544 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1546 if self.config.doDark:
1547 self.log.info(
"Applying dark correction.")
1549 self.
debugViewdebugView(ccdExposure,
"doDark")
1551 if self.config.doFringe
and not self.config.fringeAfterFlat:
1552 self.log.info(
"Applying fringe correction before flat.")
1553 self.fringe.
run(ccdExposure, **fringes.getDict())
1554 self.
debugViewdebugView(ccdExposure,
"doFringe")
1556 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1557 self.log.info(
"Checking strayLight correction.")
1558 self.strayLight.
run(ccdExposure, strayLightData)
1559 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1561 if self.config.doFlat:
1562 self.log.info(
"Applying flat correction.")
1564 self.
debugViewdebugView(ccdExposure,
"doFlat")
1566 if self.config.doApplyGains:
1567 self.log.info(
"Applying gain correction instead of flat.")
1568 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1570 if self.config.doFringe
and self.config.fringeAfterFlat:
1571 self.log.info(
"Applying fringe correction after flat.")
1572 self.fringe.
run(ccdExposure, **fringes.getDict())
1574 if self.config.doVignette:
1575 self.log.info(
"Constructing Vignette polygon.")
1578 if self.config.vignette.doWriteVignettePolygon:
1581 if self.config.doAttachTransmissionCurve:
1582 self.log.info(
"Adding transmission curves.")
1583 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1584 filterTransmission=filterTransmission,
1585 sensorTransmission=sensorTransmission,
1586 atmosphereTransmission=atmosphereTransmission)
1588 flattenedThumb =
None
1589 if self.config.qa.doThumbnailFlattened:
1590 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1592 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1593 self.log.info(
"Performing illumination correction.")
1594 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1595 illumMaskedImage, illumScale=self.config.illumScale,
1596 trimToFit=self.config.doTrimToMatchCalib)
1599 if self.config.doSaveInterpPixels:
1600 preInterpExp = ccdExposure.clone()
1615 if self.config.doSetBadRegions:
1616 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1617 if badPixelCount > 0:
1618 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1620 if self.config.doInterpolate:
1621 self.log.info(
"Interpolating masked pixels.")
1622 isrFunctions.interpolateFromMask(
1623 maskedImage=ccdExposure.getMaskedImage(),
1624 fwhm=self.config.fwhm,
1625 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1626 maskNameList=list(self.config.maskListToInterpolate)
1631 if self.config.doMeasureBackground:
1632 self.log.info(
"Measuring background level.")
1635 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1637 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1638 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1639 afwMath.MEDIAN | afwMath.STDEVCLIP)
1640 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1641 qaStats.getValue(afwMath.MEDIAN))
1642 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1643 qaStats.getValue(afwMath.STDEVCLIP))
1644 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1645 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1646 qaStats.getValue(afwMath.STDEVCLIP))
1648 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1650 return pipeBase.Struct(
1651 exposure=ccdExposure,
1653 flattenedThumb=flattenedThumb,
1655 preInterpolatedExposure=preInterpExp,
1656 outputExposure=ccdExposure,
1657 outputOssThumbnail=ossThumb,
1658 outputFlattenedThumbnail=flattenedThumb,
1661 @pipeBase.timeMethod
1663 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1665 This method contains the `CmdLineTask` interface to the ISR
1666 processing. All IO is handled here, freeing the `run()` method
1667 to manage only pixel-level calculations. The steps performed
1669 - Read in necessary detrending/isr/calibration data.
1670 - Process raw exposure in `run()`.
1671 - Persist the ISR-corrected exposure as "postISRCCD" if
1672 config.doWrite=True.
1676 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1677 DataRef of the detector data to be processed
1681 result : `lsst.pipe.base.Struct`
1682 Result struct with component:
1683 - ``exposure`` : `afw.image.Exposure`
1684 The fully ISR corrected exposure.
1689 Raised if a configuration option is set to True, but the
1690 required calibration data does not exist.
1693 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1695 ccdExposure = sensorRef.get(self.config.datasetType)
1697 camera = sensorRef.get(
"camera")
1698 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1700 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1702 if self.config.doWrite:
1703 sensorRef.put(result.exposure,
"postISRCCD")
1704 if result.preInterpolatedExposure
is not None:
1705 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1706 if result.ossThumb
is not None:
1707 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1708 if result.flattenedThumb
is not None:
1709 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1714 """Retrieve a calibration dataset for removing instrument signature.
1719 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1720 DataRef of the detector data to find calibration datasets
1723 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1724 dateObs : `str`, optional
1725 Date of the observation. Used to correct butler failures
1726 when using fallback filters.
1728 If True, disable butler proxies to enable error handling
1729 within this routine.
1733 exposure : `lsst.afw.image.Exposure`
1734 Requested calibration frame.
1739 Raised if no matching calibration frame can be found.
1742 exp = dataRef.get(datasetType, immediate=immediate)
1743 except Exception
as exc1:
1744 if not self.config.fallbackFilterName:
1745 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1747 if self.config.useFallbackDate
and dateObs:
1748 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1749 dateObs=dateObs, immediate=immediate)
1751 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1752 except Exception
as exc2:
1753 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1754 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1755 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1757 if self.config.doAssembleIsrExposures:
1758 exp = self.assembleCcd.assembleCcd(exp)
1762 """Ensure that the data returned by Butler is a fully constructed exposure.
1764 ISR requires exposure-level image data for historical reasons, so if we did
1765 not recieve that from Butler, construct it from what we have, modifying the
1770 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1771 `lsst.afw.image.ImageF`
1772 The input data structure obtained from Butler.
1773 camera : `lsst.afw.cameraGeom.camera`
1774 The camera associated with the image. Used to find the appropriate
1777 The detector this exposure should match.
1781 inputExp : `lsst.afw.image.Exposure`
1782 The re-constructed exposure, with appropriate detector parameters.
1787 Raised if the input data cannot be used to construct an exposure.
1789 if isinstance(inputExp, afwImage.DecoratedImageU):
1790 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1791 elif isinstance(inputExp, afwImage.ImageF):
1792 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1793 elif isinstance(inputExp, afwImage.MaskedImageF):
1794 inputExp = afwImage.makeExposure(inputExp)
1795 elif isinstance(inputExp, afwImage.Exposure):
1797 elif inputExp
is None:
1801 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1804 if inputExp.getDetector()
is None:
1805 inputExp.setDetector(camera[detectorNum])
1810 """Convert exposure image from uint16 to float.
1812 If the exposure does not need to be converted, the input is
1813 immediately returned. For exposures that are converted to use
1814 floating point pixels, the variance is set to unity and the
1819 exposure : `lsst.afw.image.Exposure`
1820 The raw exposure to be converted.
1824 newexposure : `lsst.afw.image.Exposure`
1825 The input ``exposure``, converted to floating point pixels.
1830 Raised if the exposure type cannot be converted to float.
1833 if isinstance(exposure, afwImage.ExposureF):
1835 self.log.debug(
"Exposure already of type float.")
1837 if not hasattr(exposure,
"convertF"):
1838 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1840 newexposure = exposure.convertF()
1841 newexposure.variance[:] = 1
1842 newexposure.mask[:] = 0x0
1847 """Identify bad amplifiers, saturated and suspect pixels.
1851 ccdExposure : `lsst.afw.image.Exposure`
1852 Input exposure to be masked.
1853 amp : `lsst.afw.table.AmpInfoCatalog`
1854 Catalog of parameters defining the amplifier on this
1856 defects : `lsst.ip.isr.Defects`
1857 List of defects. Used to determine if the entire
1863 If this is true, the entire amplifier area is covered by
1864 defects and unusable.
1867 maskedImage = ccdExposure.getMaskedImage()
1873 if defects
is not None:
1874 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1879 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1881 maskView = dataView.getMask()
1882 maskView |= maskView.getPlaneBitMask(
"BAD")
1889 if self.config.doSaturation
and not badAmp:
1890 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1891 if self.config.doSuspect
and not badAmp:
1892 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1893 if math.isfinite(self.config.saturation):
1894 limits.update({self.config.saturatedMaskName: self.config.saturation})
1896 for maskName, maskThreshold
in limits.items():
1897 if not math.isnan(maskThreshold):
1898 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1899 isrFunctions.makeThresholdMask(
1900 maskedImage=dataView,
1901 threshold=maskThreshold,
1907 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1909 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1910 self.config.suspectMaskName])
1911 if numpy.all(maskView.getArray() & maskVal > 0):
1913 maskView |= maskView.getPlaneBitMask(
"BAD")
1918 """Apply overscan correction in place.
1920 This method does initial pixel rejection of the overscan
1921 region. The overscan can also be optionally segmented to
1922 allow for discontinuous overscan responses to be fit
1923 separately. The actual overscan subtraction is performed by
1924 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1925 which is called here after the amplifier is preprocessed.
1929 ccdExposure : `lsst.afw.image.Exposure`
1930 Exposure to have overscan correction performed.
1931 amp : `lsst.afw.cameraGeom.Amplifer`
1932 The amplifier to consider while correcting the overscan.
1936 overscanResults : `lsst.pipe.base.Struct`
1937 Result struct with components:
1938 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1939 Value or fit subtracted from the amplifier image data.
1940 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1941 Value or fit subtracted from the overscan image data.
1942 - ``overscanImage`` : `lsst.afw.image.Image`
1943 Image of the overscan region with the overscan
1944 correction applied. This quantity is used to estimate
1945 the amplifier read noise empirically.
1950 Raised if the ``amp`` does not contain raw pixel information.
1954 lsst.ip.isr.isrFunctions.overscanCorrection
1956 if amp.getRawHorizontalOverscanBBox().isEmpty():
1957 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1960 statControl = afwMath.StatisticsControl()
1961 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1964 dataBBox = amp.getRawDataBBox()
1965 oscanBBox = amp.getRawHorizontalOverscanBBox()
1969 prescanBBox = amp.getRawPrescanBBox()
1970 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1971 dx0 += self.config.overscanNumLeadingColumnsToSkip
1972 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1974 dx0 += self.config.overscanNumTrailingColumnsToSkip
1975 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1981 if ((self.config.overscanBiasJump
1982 and self.config.overscanBiasJumpLocation)
1983 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1984 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
1985 self.config.overscanBiasJumpDevices)):
1986 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1987 yLower = self.config.overscanBiasJumpLocation
1988 yUpper = dataBBox.getHeight() - yLower
1990 yUpper = self.config.overscanBiasJumpLocation
1991 yLower = dataBBox.getHeight() - yUpper
2009 oscanBBox.getHeight())))
2012 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2013 ampImage = ccdExposure.maskedImage[imageBBox]
2014 overscanImage = ccdExposure.maskedImage[overscanBBox]
2016 overscanArray = overscanImage.image.array
2017 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2018 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2019 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2021 statControl = afwMath.StatisticsControl()
2022 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2024 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2027 levelStat = afwMath.MEDIAN
2028 sigmaStat = afwMath.STDEVCLIP
2030 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2031 self.config.qa.flatness.nIter)
2032 metadata = ccdExposure.getMetadata()
2033 ampNum = amp.getName()
2035 if isinstance(overscanResults.overscanFit, float):
2036 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2037 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2039 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2040 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2041 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2043 return overscanResults
2046 """Set the variance plane using the amplifier gain and read noise
2048 The read noise is calculated from the ``overscanImage`` if the
2049 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2050 the value from the amplifier data is used.
2054 ampExposure : `lsst.afw.image.Exposure`
2055 Exposure to process.
2056 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2057 Amplifier detector data.
2058 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2059 Image of overscan, required only for empirical read noise.
2063 lsst.ip.isr.isrFunctions.updateVariance
2065 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2066 gain = amp.getGain()
2068 if math.isnan(gain):
2070 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2073 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2074 amp.getName(), gain, patchedGain)
2077 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2078 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
2080 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2081 stats = afwMath.StatisticsControl()
2082 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2083 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2084 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2085 amp.getName(), readNoise)
2087 readNoise = amp.getReadNoise()
2089 isrFunctions.updateVariance(
2090 maskedImage=ampExposure.getMaskedImage(),
2092 readNoise=readNoise,
2096 """Apply dark correction in place.
2100 exposure : `lsst.afw.image.Exposure`
2101 Exposure to process.
2102 darkExposure : `lsst.afw.image.Exposure`
2103 Dark exposure of the same size as ``exposure``.
2104 invert : `Bool`, optional
2105 If True, re-add the dark to an already corrected image.
2110 Raised if either ``exposure`` or ``darkExposure`` do not
2111 have their dark time defined.
2115 lsst.ip.isr.isrFunctions.darkCorrection
2117 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2118 if math.isnan(expScale):
2119 raise RuntimeError(
"Exposure darktime is NAN.")
2120 if darkExposure.getInfo().getVisitInfo()
is not None \
2121 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2122 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2126 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2129 isrFunctions.darkCorrection(
2130 maskedImage=exposure.getMaskedImage(),
2131 darkMaskedImage=darkExposure.getMaskedImage(),
2133 darkScale=darkScale,
2135 trimToFit=self.config.doTrimToMatchCalib
2139 """Check if linearization is needed for the detector cameraGeom.
2141 Checks config.doLinearize and the linearity type of the first
2146 detector : `lsst.afw.cameraGeom.Detector`
2147 Detector to get linearity type from.
2151 doLinearize : `Bool`
2152 If True, linearization should be performed.
2154 return self.config.doLinearize
and \
2155 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2158 """Apply flat correction in place.
2162 exposure : `lsst.afw.image.Exposure`
2163 Exposure to process.
2164 flatExposure : `lsst.afw.image.Exposure`
2165 Flat exposure of the same size as ``exposure``.
2166 invert : `Bool`, optional
2167 If True, unflatten an already flattened image.
2171 lsst.ip.isr.isrFunctions.flatCorrection
2173 isrFunctions.flatCorrection(
2174 maskedImage=exposure.getMaskedImage(),
2175 flatMaskedImage=flatExposure.getMaskedImage(),
2176 scalingType=self.config.flatScalingType,
2177 userScale=self.config.flatUserScale,
2179 trimToFit=self.config.doTrimToMatchCalib
2183 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2187 exposure : `lsst.afw.image.Exposure`
2188 Exposure to process. Only the amplifier DataSec is processed.
2189 amp : `lsst.afw.table.AmpInfoCatalog`
2190 Amplifier detector data.
2194 lsst.ip.isr.isrFunctions.makeThresholdMask
2196 if not math.isnan(amp.getSaturation()):
2197 maskedImage = exposure.getMaskedImage()
2198 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2199 isrFunctions.makeThresholdMask(
2200 maskedImage=dataView,
2201 threshold=amp.getSaturation(),
2203 maskName=self.config.saturatedMaskName,
2207 """Interpolate over saturated pixels, in place.
2209 This method should be called after `saturationDetection`, to
2210 ensure that the saturated pixels have been identified in the
2211 SAT mask. It should also be called after `assembleCcd`, since
2212 saturated regions may cross amplifier boundaries.
2216 exposure : `lsst.afw.image.Exposure`
2217 Exposure to process.
2221 lsst.ip.isr.isrTask.saturationDetection
2222 lsst.ip.isr.isrFunctions.interpolateFromMask
2224 isrFunctions.interpolateFromMask(
2225 maskedImage=exposure.getMaskedImage(),
2226 fwhm=self.config.fwhm,
2227 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2228 maskNameList=list(self.config.saturatedMaskName),
2232 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2236 exposure : `lsst.afw.image.Exposure`
2237 Exposure to process. Only the amplifier DataSec is processed.
2238 amp : `lsst.afw.table.AmpInfoCatalog`
2239 Amplifier detector data.
2243 lsst.ip.isr.isrFunctions.makeThresholdMask
2247 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2248 This is intended to indicate pixels that may be affected by unknown systematics;
2249 for example if non-linearity corrections above a certain level are unstable
2250 then that would be a useful value for suspectLevel. A value of `nan` indicates
2251 that no such level exists and no pixels are to be masked as suspicious.
2253 suspectLevel = amp.getSuspectLevel()
2254 if math.isnan(suspectLevel):
2257 maskedImage = exposure.getMaskedImage()
2258 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2259 isrFunctions.makeThresholdMask(
2260 maskedImage=dataView,
2261 threshold=suspectLevel,
2263 maskName=self.config.suspectMaskName,
2267 """Mask defects using mask plane "BAD", in place.
2271 exposure : `lsst.afw.image.Exposure`
2272 Exposure to process.
2273 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2274 `lsst.afw.image.DefectBase`.
2275 List of defects to mask.
2279 Call this after CCD assembly, since defects may cross amplifier boundaries.
2281 maskedImage = exposure.getMaskedImage()
2282 if not isinstance(defectBaseList, Defects):
2284 defectList =
Defects(defectBaseList)
2286 defectList = defectBaseList
2287 defectList.maskPixels(maskedImage, maskName=
"BAD")
2289 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2290 """Mask edge pixels with applicable mask plane.
2294 exposure : `lsst.afw.image.Exposure`
2295 Exposure to process.
2296 numEdgePixels : `int`, optional
2297 Number of edge pixels to mask.
2298 maskPlane : `str`, optional
2299 Mask plane name to use.
2300 level : `str`, optional
2301 Level at which to mask edges.
2303 maskedImage = exposure.getMaskedImage()
2304 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2306 if numEdgePixels > 0:
2307 if level ==
'DETECTOR':
2308 boxes = [maskedImage.getBBox()]
2309 elif level ==
'AMP':
2310 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2314 subImage = maskedImage[box]
2315 box.grow(-numEdgePixels)
2317 SourceDetectionTask.setEdgeBits(
2323 """Mask and interpolate defects using mask plane "BAD", in place.
2327 exposure : `lsst.afw.image.Exposure`
2328 Exposure to process.
2329 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2330 `lsst.afw.image.DefectBase`.
2331 List of defects to mask and interpolate.
2335 lsst.ip.isr.isrTask.maskDefect
2337 self.
maskDefectmaskDefect(exposure, defectBaseList)
2338 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2339 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2340 isrFunctions.interpolateFromMask(
2341 maskedImage=exposure.getMaskedImage(),
2342 fwhm=self.config.fwhm,
2343 growSaturatedFootprints=0,
2344 maskNameList=[
"BAD"],
2348 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2352 exposure : `lsst.afw.image.Exposure`
2353 Exposure to process.
2357 We mask over all NaNs, including those that are masked with
2358 other bits (because those may or may not be interpolated over
2359 later, and we want to remove all NaNs). Despite this
2360 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2361 the historical name.
2363 maskedImage = exposure.getMaskedImage()
2366 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2367 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2368 numNans =
maskNans(maskedImage, maskVal)
2369 self.metadata.set(
"NUMNANS", numNans)
2371 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2374 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2378 exposure : `lsst.afw.image.Exposure`
2379 Exposure to process.
2383 lsst.ip.isr.isrTask.maskNan
2386 isrFunctions.interpolateFromMask(
2387 maskedImage=exposure.getMaskedImage(),
2388 fwhm=self.config.fwhm,
2389 growSaturatedFootprints=0,
2390 maskNameList=[
"UNMASKEDNAN"],
2394 """Measure the image background in subgrids, for quality control purposes.
2398 exposure : `lsst.afw.image.Exposure`
2399 Exposure to process.
2400 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2401 Configuration object containing parameters on which background
2402 statistics and subgrids to use.
2404 if IsrQaConfig
is not None:
2405 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2406 IsrQaConfig.flatness.nIter)
2407 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2408 statsControl.setAndMask(maskVal)
2409 maskedImage = exposure.getMaskedImage()
2410 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2411 skyLevel = stats.getValue(afwMath.MEDIAN)
2412 skySigma = stats.getValue(afwMath.STDEVCLIP)
2413 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2414 metadata = exposure.getMetadata()
2415 metadata.set(
'SKYLEVEL', skyLevel)
2416 metadata.set(
'SKYSIGMA', skySigma)
2419 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2420 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2421 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2422 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2423 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2424 skyLevels = numpy.zeros((nX, nY))
2427 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2429 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2431 xLLC = xc - meshXHalf
2432 yLLC = yc - meshYHalf
2433 xURC = xc + meshXHalf - 1
2434 yURC = yc + meshYHalf - 1
2437 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2439 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2441 good = numpy.where(numpy.isfinite(skyLevels))
2442 skyMedian = numpy.median(skyLevels[good])
2443 flatness = (skyLevels[good] - skyMedian) / skyMedian
2444 flatness_rms = numpy.std(flatness)
2445 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2447 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2448 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2449 nX, nY, flatness_pp, flatness_rms)
2451 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2452 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2453 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2454 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2455 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2458 """Set an approximate magnitude zero point for the exposure.
2462 exposure : `lsst.afw.image.Exposure`
2463 Exposure to process.
2466 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2467 if filterName
in self.config.fluxMag0T1:
2468 fluxMag0 = self.config.fluxMag0T1[filterName]
2470 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2471 fluxMag0 = self.config.defaultFluxMag0T1
2473 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2475 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2478 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2479 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2482 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2486 ccdExposure : `lsst.afw.image.Exposure`
2487 Exposure to process.
2488 fpPolygon : `lsst.afw.geom.Polygon`
2489 Polygon in focal plane coordinates.
2492 ccd = ccdExposure.getDetector()
2493 fpCorners = ccd.getCorners(FOCAL_PLANE)
2494 ccdPolygon = Polygon(fpCorners)
2497 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2500 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2501 validPolygon = Polygon(ccdPoints)
2502 ccdExposure.getInfo().setValidPolygon(validPolygon)
2506 """Context manager that applies and removes flats and darks,
2507 if the task is configured to apply them.
2511 exp : `lsst.afw.image.Exposure`
2512 Exposure to process.
2513 flat : `lsst.afw.image.Exposure`
2514 Flat exposure the same size as ``exp``.
2515 dark : `lsst.afw.image.Exposure`, optional
2516 Dark exposure the same size as ``exp``.
2520 exp : `lsst.afw.image.Exposure`
2521 The flat and dark corrected exposure.
2523 if self.config.doDark
and dark
is not None:
2525 if self.config.doFlat:
2530 if self.config.doFlat:
2532 if self.config.doDark
and dark
is not None:
2536 """Utility function to examine ISR exposure at different stages.
2540 exposure : `lsst.afw.image.Exposure`
2543 State of processing to view.
2545 frame = getDebugFrame(self._display, stepname)
2547 display = getDisplay(frame)
2548 display.scale(
'asinh',
'zscale')
2549 display.mtv(exposure)
2550 prompt =
"Press Enter to continue [c]... "
2552 ans = input(prompt).lower()
2553 if ans
in (
"",
"c",):
2558 """A Detector-like object that supports returning gain and saturation level
2560 This is used when the input exposure does not have a detector.
2564 exposure : `lsst.afw.image.Exposure`
2565 Exposure to generate a fake amplifier for.
2566 config : `lsst.ip.isr.isrTaskConfig`
2567 Configuration to apply to the fake amplifier.
2571 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2573 self.
_gain_gain = config.gain
2578 return self.
_bbox_bbox
2581 return self.
_bbox_bbox
2587 return self.
_gain_gain
2600 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2604 """Task to wrap the default IsrTask to allow it to be retargeted.
2606 The standard IsrTask can be called directly from a command line
2607 program, but doing so removes the ability of the task to be
2608 retargeted. As most cameras override some set of the IsrTask
2609 methods, this would remove those data-specific methods in the
2610 output post-ISR images. This wrapping class fixes the issue,
2611 allowing identical post-ISR images to be generated by both the
2612 processCcd and isrTask code.
2614 ConfigClass = RunIsrConfig
2615 _DefaultName =
"runIsr"
2619 self.makeSubtask(
"isr")
2625 dataRef : `lsst.daf.persistence.ButlerDataRef`
2626 data reference of the detector data to be processed
2630 result : `pipeBase.Struct`
2631 Result struct with component:
2633 - exposure : `lsst.afw.image.Exposure`
2634 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def ensureExposure(self, inputExp, camera, detectorNum)
def updateVariance(self, ampExposure, amp, overscanImage=None)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.