32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
134 bias = cT.PrerequisiteInput(
136 doc=
"Input bias calibration.",
137 storageClass=
"ExposureF",
138 dimensions=[
"instrument",
"detector"],
141 dark = cT.PrerequisiteInput(
143 doc=
"Input dark calibration.",
144 storageClass=
"ExposureF",
145 dimensions=[
"instrument",
"detector"],
148 flat = cT.PrerequisiteInput(
150 doc=
"Input flat calibration.",
151 storageClass=
"ExposureF",
152 dimensions=[
"instrument",
"physical_filter",
"detector"],
155 fringes = cT.PrerequisiteInput(
157 doc=
"Input fringe calibration.",
158 storageClass=
"ExposureF",
159 dimensions=[
"instrument",
"physical_filter",
"detector"],
162 strayLightData = cT.PrerequisiteInput(
164 doc=
"Input stray light calibration.",
165 storageClass=
"StrayLightData",
166 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 bfKernel = cT.PrerequisiteInput(
171 doc=
"Input brighter-fatter kernel.",
172 storageClass=
"NumpyArray",
173 dimensions=[
"instrument"],
176 newBFKernel = cT.PrerequisiteInput(
177 name=
'brighterFatterKernel',
178 doc=
"Newer complete kernel + gain solutions.",
179 storageClass=
"BrighterFatterKernel",
180 dimensions=[
"instrument",
"detector"],
183 defects = cT.PrerequisiteInput(
185 doc=
"Input defect tables.",
186 storageClass=
"Defects",
187 dimensions=[
"instrument",
"detector"],
190 linearizer = cT.PrerequisiteInput(
192 storageClass=
"Linearizer",
193 doc=
"Linearity correction calibration.",
194 dimensions=[
"instrument",
"detector"],
197 opticsTransmission = cT.PrerequisiteInput(
198 name=
"transmission_optics",
199 storageClass=
"TransmissionCurve",
200 doc=
"Transmission curve due to the optics.",
201 dimensions=[
"instrument"],
204 filterTransmission = cT.PrerequisiteInput(
205 name=
"transmission_filter",
206 storageClass=
"TransmissionCurve",
207 doc=
"Transmission curve due to the filter.",
208 dimensions=[
"instrument",
"physical_filter"],
211 sensorTransmission = cT.PrerequisiteInput(
212 name=
"transmission_sensor",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the sensor.",
215 dimensions=[
"instrument",
"detector"],
218 atmosphereTransmission = cT.PrerequisiteInput(
219 name=
"transmission_atmosphere",
220 storageClass=
"TransmissionCurve",
221 doc=
"Transmission curve due to the atmosphere.",
222 dimensions=[
"instrument"],
225 illumMaskedImage = cT.PrerequisiteInput(
227 doc=
"Input illumination correction.",
228 storageClass=
"MaskedImageF",
229 dimensions=[
"instrument",
"physical_filter",
"detector"],
233 outputExposure = cT.Output(
235 doc=
"Output ISR processed exposure.",
236 storageClass=
"Exposure",
237 dimensions=[
"instrument",
"exposure",
"detector"],
239 preInterpExposure = cT.Output(
240 name=
'preInterpISRCCD',
241 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
242 storageClass=
"ExposureF",
243 dimensions=[
"instrument",
"exposure",
"detector"],
245 outputOssThumbnail = cT.Output(
247 doc=
"Output Overscan-subtracted thumbnail image.",
248 storageClass=
"Thumbnail",
249 dimensions=[
"instrument",
"exposure",
"detector"],
251 outputFlattenedThumbnail = cT.Output(
252 name=
"FlattenedThumb",
253 doc=
"Output flat-corrected thumbnail image.",
254 storageClass=
"Thumbnail",
255 dimensions=[
"instrument",
"exposure",
"detector"],
261 if config.doBias
is not True:
262 self.prerequisiteInputs.discard(
"bias")
263 if config.doLinearize
is not True:
264 self.prerequisiteInputs.discard(
"linearizer")
265 if config.doCrosstalk
is not True:
266 self.inputs.discard(
"crosstalkSources")
267 self.prerequisiteInputs.discard(
"crosstalk")
268 if config.doBrighterFatter
is not True:
269 self.prerequisiteInputs.discard(
"bfKernel")
270 self.prerequisiteInputs.discard(
"newBFKernel")
271 if config.doDefect
is not True:
272 self.prerequisiteInputs.discard(
"defects")
273 if config.doDark
is not True:
274 self.prerequisiteInputs.discard(
"dark")
275 if config.doFlat
is not True:
276 self.prerequisiteInputs.discard(
"flat")
277 if config.doAttachTransmissionCurve
is not True:
278 self.prerequisiteInputs.discard(
"opticsTransmission")
279 self.prerequisiteInputs.discard(
"filterTransmission")
280 self.prerequisiteInputs.discard(
"sensorTransmission")
281 self.prerequisiteInputs.discard(
"atmosphereTransmission")
282 if config.doUseOpticsTransmission
is not True:
283 self.prerequisiteInputs.discard(
"opticsTransmission")
284 if config.doUseFilterTransmission
is not True:
285 self.prerequisiteInputs.discard(
"filterTransmission")
286 if config.doUseSensorTransmission
is not True:
287 self.prerequisiteInputs.discard(
"sensorTransmission")
288 if config.doUseAtmosphereTransmission
is not True:
289 self.prerequisiteInputs.discard(
"atmosphereTransmission")
290 if config.doIlluminationCorrection
is not True:
291 self.prerequisiteInputs.discard(
"illumMaskedImage")
293 if config.doWrite
is not True:
294 self.outputs.discard(
"outputExposure")
295 self.outputs.discard(
"preInterpExposure")
296 self.outputs.discard(
"outputFlattenedThumbnail")
297 self.outputs.discard(
"outputOssThumbnail")
298 if config.doSaveInterpPixels
is not True:
299 self.outputs.discard(
"preInterpExposure")
300 if config.qa.doThumbnailOss
is not True:
301 self.outputs.discard(
"outputOssThumbnail")
302 if config.qa.doThumbnailFlattened
is not True:
303 self.outputs.discard(
"outputFlattenedThumbnail")
307 pipelineConnections=IsrTaskConnections):
308 """Configuration parameters for IsrTask.
310 Items are grouped in the order in which they are executed by the task.
312 datasetType = pexConfig.Field(
314 doc=
"Dataset type for input data; users will typically leave this alone, "
315 "but camera-specific ISR tasks will override it",
319 fallbackFilterName = pexConfig.Field(
321 doc=
"Fallback default filter name for calibrations.",
324 useFallbackDate = pexConfig.Field(
326 doc=
"Pass observation date when using fallback filter.",
329 expectWcs = pexConfig.Field(
332 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
334 fwhm = pexConfig.Field(
336 doc=
"FWHM of PSF in arcseconds.",
339 qa = pexConfig.ConfigField(
341 doc=
"QA related configuration options.",
345 doConvertIntToFloat = pexConfig.Field(
347 doc=
"Convert integer raw images to floating point values?",
352 doSaturation = pexConfig.Field(
354 doc=
"Mask saturated pixels? NB: this is totally independent of the"
355 " interpolation option - this is ONLY setting the bits in the mask."
356 " To have them interpolated make sure doSaturationInterpolation=True",
359 saturatedMaskName = pexConfig.Field(
361 doc=
"Name of mask plane to use in saturation detection and interpolation",
364 saturation = pexConfig.Field(
366 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
367 default=float(
"NaN"),
369 growSaturationFootprintSize = pexConfig.Field(
371 doc=
"Number of pixels by which to grow the saturation footprints",
376 doSuspect = pexConfig.Field(
378 doc=
"Mask suspect pixels?",
381 suspectMaskName = pexConfig.Field(
383 doc=
"Name of mask plane to use for suspect pixels",
386 numEdgeSuspect = pexConfig.Field(
388 doc=
"Number of edge pixels to be flagged as untrustworthy.",
391 edgeMaskLevel = pexConfig.ChoiceField(
393 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
396 'DETECTOR':
'Mask only the edges of the full detector.',
397 'AMP':
'Mask edges of each amplifier.',
402 doSetBadRegions = pexConfig.Field(
404 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
407 badStatistic = pexConfig.ChoiceField(
409 doc=
"How to estimate the average value for BAD regions.",
412 "MEANCLIP":
"Correct using the (clipped) mean of good data",
413 "MEDIAN":
"Correct using the median of the good data",
418 doOverscan = pexConfig.Field(
420 doc=
"Do overscan subtraction?",
423 overscan = pexConfig.ConfigurableField(
424 target=OverscanCorrectionTask,
425 doc=
"Overscan subtraction task for image segments.",
428 overscanFitType = pexConfig.ChoiceField(
430 doc=
"The method for fitting the overscan bias level.",
433 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
434 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
435 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
436 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
437 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
438 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
439 "MEAN":
"Correct using the mean of the overscan region",
440 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
441 "MEDIAN":
"Correct using the median of the overscan region",
442 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
444 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
445 " This option will no longer be used, and will be removed after v20.")
447 overscanOrder = pexConfig.Field(
449 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
450 "or number of spline knots if overscan fit type is a spline."),
452 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
453 " This option will no longer be used, and will be removed after v20.")
455 overscanNumSigmaClip = pexConfig.Field(
457 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
459 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
460 " This option will no longer be used, and will be removed after v20.")
462 overscanIsInt = pexConfig.Field(
464 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
465 " and overscan.FitType=MEDIAN_PER_ROW.",
467 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
468 " This option will no longer be used, and will be removed after v20.")
471 overscanNumLeadingColumnsToSkip = pexConfig.Field(
473 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
476 overscanNumTrailingColumnsToSkip = pexConfig.Field(
478 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
481 overscanMaxDev = pexConfig.Field(
483 doc=
"Maximum deviation from the median for overscan",
484 default=1000.0, check=
lambda x: x > 0
486 overscanBiasJump = pexConfig.Field(
488 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
491 overscanBiasJumpKeyword = pexConfig.Field(
493 doc=
"Header keyword containing information about devices.",
494 default=
"NO_SUCH_KEY",
496 overscanBiasJumpDevices = pexConfig.ListField(
498 doc=
"List of devices that need piecewise overscan correction.",
501 overscanBiasJumpLocation = pexConfig.Field(
503 doc=
"Location of bias jump along y-axis.",
508 doAssembleCcd = pexConfig.Field(
511 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
513 assembleCcd = pexConfig.ConfigurableField(
514 target=AssembleCcdTask,
515 doc=
"CCD assembly task",
519 doAssembleIsrExposures = pexConfig.Field(
522 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
524 doTrimToMatchCalib = pexConfig.Field(
527 doc=
"Trim raw data to match calibration bounding boxes?"
531 doBias = pexConfig.Field(
533 doc=
"Apply bias frame correction?",
536 biasDataProductName = pexConfig.Field(
538 doc=
"Name of the bias data product",
541 doBiasBeforeOverscan = pexConfig.Field(
543 doc=
"Reverse order of overscan and bias correction.",
548 doVariance = pexConfig.Field(
550 doc=
"Calculate variance?",
553 gain = pexConfig.Field(
555 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
556 default=float(
"NaN"),
558 readNoise = pexConfig.Field(
560 doc=
"The read noise to use if no Detector is present in the Exposure",
563 doEmpiricalReadNoise = pexConfig.Field(
566 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
570 doLinearize = pexConfig.Field(
572 doc=
"Correct for nonlinearity of the detector's response?",
577 doCrosstalk = pexConfig.Field(
579 doc=
"Apply intra-CCD crosstalk correction?",
582 doCrosstalkBeforeAssemble = pexConfig.Field(
584 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
587 crosstalk = pexConfig.ConfigurableField(
588 target=CrosstalkTask,
589 doc=
"Intra-CCD crosstalk correction",
593 doDefect = pexConfig.Field(
595 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
598 doNanMasking = pexConfig.Field(
600 doc=
"Mask NAN pixels?",
603 doWidenSaturationTrails = pexConfig.Field(
605 doc=
"Widen bleed trails based on their width?",
610 doBrighterFatter = pexConfig.Field(
613 doc=
"Apply the brighter fatter correction"
615 brighterFatterLevel = pexConfig.ChoiceField(
618 doc=
"The level at which to correct for brighter-fatter.",
620 "AMP":
"Every amplifier treated separately.",
621 "DETECTOR":
"One kernel per detector",
624 brighterFatterMaxIter = pexConfig.Field(
627 doc=
"Maximum number of iterations for the brighter fatter correction"
629 brighterFatterThreshold = pexConfig.Field(
632 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
633 " absolute value of the difference between the current corrected image and the one"
634 " from the previous iteration summed over all the pixels."
636 brighterFatterApplyGain = pexConfig.Field(
639 doc=
"Should the gain be applied when applying the brighter fatter correction?"
641 brighterFatterMaskGrowSize = pexConfig.Field(
644 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
645 " when brighter-fatter correction is applied."
649 doDark = pexConfig.Field(
651 doc=
"Apply dark frame correction?",
654 darkDataProductName = pexConfig.Field(
656 doc=
"Name of the dark data product",
661 doStrayLight = pexConfig.Field(
663 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
666 strayLight = pexConfig.ConfigurableField(
667 target=StrayLightTask,
668 doc=
"y-band stray light correction"
672 doFlat = pexConfig.Field(
674 doc=
"Apply flat field correction?",
677 flatDataProductName = pexConfig.Field(
679 doc=
"Name of the flat data product",
682 flatScalingType = pexConfig.ChoiceField(
684 doc=
"The method for scaling the flat on the fly.",
687 "USER":
"Scale by flatUserScale",
688 "MEAN":
"Scale by the inverse of the mean",
689 "MEDIAN":
"Scale by the inverse of the median",
692 flatUserScale = pexConfig.Field(
694 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
697 doTweakFlat = pexConfig.Field(
699 doc=
"Tweak flats to match observed amplifier ratios?",
704 doApplyGains = pexConfig.Field(
706 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
709 normalizeGains = pexConfig.Field(
711 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
716 doFringe = pexConfig.Field(
718 doc=
"Apply fringe correction?",
721 fringe = pexConfig.ConfigurableField(
723 doc=
"Fringe subtraction task",
725 fringeAfterFlat = pexConfig.Field(
727 doc=
"Do fringe subtraction after flat-fielding?",
732 doMeasureBackground = pexConfig.Field(
734 doc=
"Measure the background level on the reduced image?",
739 doCameraSpecificMasking = pexConfig.Field(
741 doc=
"Mask camera-specific bad regions?",
744 masking = pexConfig.ConfigurableField(
751 doInterpolate = pexConfig.Field(
753 doc=
"Interpolate masked pixels?",
756 doSaturationInterpolation = pexConfig.Field(
758 doc=
"Perform interpolation over pixels masked as saturated?"
759 " NB: This is independent of doSaturation; if that is False this plane"
760 " will likely be blank, resulting in a no-op here.",
763 doNanInterpolation = pexConfig.Field(
765 doc=
"Perform interpolation over pixels masked as NaN?"
766 " NB: This is independent of doNanMasking; if that is False this plane"
767 " will likely be blank, resulting in a no-op here.",
770 doNanInterpAfterFlat = pexConfig.Field(
772 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
773 "also have to interpolate them before flat-fielding."),
776 maskListToInterpolate = pexConfig.ListField(
778 doc=
"List of mask planes that should be interpolated.",
779 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
781 doSaveInterpPixels = pexConfig.Field(
783 doc=
"Save a copy of the pre-interpolated pixel values?",
788 fluxMag0T1 = pexConfig.DictField(
791 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
792 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
795 defaultFluxMag0T1 = pexConfig.Field(
797 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
798 default=pow(10.0, 0.4*28.0)
802 doVignette = pexConfig.Field(
804 doc=
"Apply vignetting parameters?",
807 vignette = pexConfig.ConfigurableField(
809 doc=
"Vignetting task.",
813 doAttachTransmissionCurve = pexConfig.Field(
816 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
818 doUseOpticsTransmission = pexConfig.Field(
821 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
823 doUseFilterTransmission = pexConfig.Field(
826 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
828 doUseSensorTransmission = pexConfig.Field(
831 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
833 doUseAtmosphereTransmission = pexConfig.Field(
836 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
840 doIlluminationCorrection = pexConfig.Field(
843 doc=
"Perform illumination correction?"
845 illuminationCorrectionDataProductName = pexConfig.Field(
847 doc=
"Name of the illumination correction data product.",
850 illumScale = pexConfig.Field(
852 doc=
"Scale factor for the illumination correction.",
855 illumFilters = pexConfig.ListField(
858 doc=
"Only perform illumination correction for these filters."
862 doWrite = pexConfig.Field(
864 doc=
"Persist postISRCCD?",
871 raise ValueError(
"You may not specify both doFlat and doApplyGains")
873 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
882 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
883 """Apply common instrument signature correction algorithms to a raw frame.
885 The process for correcting imaging data is very similar from
886 camera to camera. This task provides a vanilla implementation of
887 doing these corrections, including the ability to turn certain
888 corrections off if they are not needed. The inputs to the primary
889 method, `run()`, are a raw exposure to be corrected and the
890 calibration data products. The raw input is a single chip sized
891 mosaic of all amps including overscans and other non-science
892 pixels. The method `runDataRef()` identifies and defines the
893 calibration data products, and is intended for use by a
894 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
895 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
896 subclassed for different camera, although the most camera specific
897 methods have been split into subtasks that can be redirected
900 The __init__ method sets up the subtasks for ISR processing, using
901 the defaults from `lsst.ip.isr`.
906 Positional arguments passed to the Task constructor. None used at this time.
907 kwargs : `dict`, optional
908 Keyword arguments passed on to the Task constructor. None used at this time.
910 ConfigClass = IsrTaskConfig
915 self.makeSubtask(
"assembleCcd")
916 self.makeSubtask(
"crosstalk")
917 self.makeSubtask(
"strayLight")
918 self.makeSubtask(
"fringe")
919 self.makeSubtask(
"masking")
920 self.makeSubtask(
"overscan")
921 self.makeSubtask(
"vignette")
924 inputs = butlerQC.get(inputRefs)
927 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
928 except Exception
as e:
929 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
932 inputs[
'isGen3'] =
True
934 detector = inputs[
'ccdExposure'].getDetector()
936 if self.config.doCrosstalk
is True:
939 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
940 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
941 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
943 coeffVector = (self.config.crosstalk.crosstalkValues
944 if self.config.crosstalk.useConfigCoefficients
else None)
945 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
946 inputs[
'crosstalk'] = crosstalkCalib
947 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
948 if 'crosstalkSources' not in inputs:
949 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
952 if 'linearizer' in inputs:
953 if isinstance(inputs[
'linearizer'], dict):
955 linearizer.fromYaml(inputs[
'linearizer'])
956 self.log.warn(
"Dictionary linearizers will be deprecated in DM-28741.")
957 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
961 self.log.warn(
"Bare lookup table linearizers will be deprecated in DM-28741.")
963 linearizer = inputs[
'linearizer']
964 linearizer.log = self.log
965 inputs[
'linearizer'] = linearizer
968 self.log.warn(
"Constructing linearizer from cameraGeom information.")
970 if self.config.doDefect
is True:
971 if "defects" in inputs
and inputs[
'defects']
is not None:
974 if not isinstance(inputs[
"defects"], Defects):
975 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
979 if self.config.doBrighterFatter:
980 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
981 if brighterFatterKernel
is None:
982 brighterFatterKernel = inputs.get(
'bfKernel',
None)
984 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
985 detId = detector.getId()
986 inputs[
'bfGains'] = brighterFatterKernel.gain
989 if self.config.brighterFatterLevel ==
'DETECTOR':
990 if brighterFatterKernel.detectorKernel:
991 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
992 elif brighterFatterKernel.detectorKernelFromAmpKernels:
993 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
995 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
998 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1000 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
1001 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
1002 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1004 assembler=self.assembleCcd
1005 if self.config.doAssembleIsrExposures
else None)
1007 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1009 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
1010 if 'strayLightData' not in inputs:
1011 inputs[
'strayLightData'] =
None
1013 outputs = self.
runrun(**inputs)
1014 butlerQC.put(outputs, outputRefs)
1017 """Retrieve necessary frames for instrument signature removal.
1019 Pre-fetching all required ISR data products limits the IO
1020 required by the ISR. Any conflict between the calibration data
1021 available and that needed for ISR is also detected prior to
1022 doing processing, allowing it to fail quickly.
1026 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1027 Butler reference of the detector data to be processed
1028 rawExposure : `afw.image.Exposure`
1029 The raw exposure that will later be corrected with the
1030 retrieved calibration data; should not be modified in this
1035 result : `lsst.pipe.base.Struct`
1036 Result struct with components (which may be `None`):
1037 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1038 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1039 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1040 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1041 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1042 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1043 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1044 - ``fringes``: `lsst.pipe.base.Struct` with components:
1045 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1046 - ``seed``: random seed derived from the ccdExposureId for random
1047 number generator (`uint32`).
1048 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1049 A ``TransmissionCurve`` that represents the throughput of the optics,
1050 to be evaluated in focal-plane coordinates.
1051 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1052 A ``TransmissionCurve`` that represents the throughput of the filter
1053 itself, to be evaluated in focal-plane coordinates.
1054 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1055 A ``TransmissionCurve`` that represents the throughput of the sensor
1056 itself, to be evaluated in post-assembly trimmed detector coordinates.
1057 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1058 A ``TransmissionCurve`` that represents the throughput of the
1059 atmosphere, assumed to be spatially constant.
1060 - ``strayLightData`` : `object`
1061 An opaque object containing calibration information for
1062 stray-light correction. If `None`, no correction will be
1064 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1068 NotImplementedError :
1069 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1072 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1073 dateObs = dateObs.toPython().isoformat()
1074 except RuntimeError:
1075 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1078 ccd = rawExposure.getDetector()
1080 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
1081 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1082 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1083 if self.config.doBias
else None)
1085 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1087 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1088 linearizer.log = self.log
1089 if isinstance(linearizer, numpy.ndarray):
1092 crosstalkCalib =
None
1093 if self.config.doCrosstalk:
1095 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1097 coeffVector = (self.config.crosstalk.crosstalkValues
1098 if self.config.crosstalk.useConfigCoefficients
else None)
1099 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1100 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1101 if self.config.doCrosstalk
else None)
1103 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1104 if self.config.doDark
else None)
1105 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1107 if self.config.doFlat
else None)
1109 brighterFatterKernel =
None
1110 brighterFatterGains =
None
1111 if self.config.doBrighterFatter
is True:
1116 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1117 brighterFatterGains = brighterFatterKernel.gain
1118 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1121 brighterFatterKernel = dataRef.get(
"bfKernel")
1122 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
1124 brighterFatterKernel =
None
1125 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1128 if self.config.brighterFatterLevel ==
'DETECTOR':
1129 if brighterFatterKernel.detectorKernel:
1130 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1131 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1132 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1134 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1137 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1139 defectList = (dataRef.get(
"defects")
1140 if self.config.doDefect
else None)
1141 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1142 if self.config.doAssembleIsrExposures
else None)
1143 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1144 else pipeBase.Struct(fringes=
None))
1146 if self.config.doAttachTransmissionCurve:
1147 opticsTransmission = (dataRef.get(
"transmission_optics")
1148 if self.config.doUseOpticsTransmission
else None)
1149 filterTransmission = (dataRef.get(
"transmission_filter")
1150 if self.config.doUseFilterTransmission
else None)
1151 sensorTransmission = (dataRef.get(
"transmission_sensor")
1152 if self.config.doUseSensorTransmission
else None)
1153 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1154 if self.config.doUseAtmosphereTransmission
else None)
1156 opticsTransmission =
None
1157 filterTransmission =
None
1158 sensorTransmission =
None
1159 atmosphereTransmission =
None
1161 if self.config.doStrayLight:
1162 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1164 strayLightData =
None
1167 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1168 if (self.config.doIlluminationCorrection
1169 and filterName
in self.config.illumFilters)
1173 return pipeBase.Struct(bias=biasExposure,
1174 linearizer=linearizer,
1175 crosstalk=crosstalkCalib,
1176 crosstalkSources=crosstalkSources,
1179 bfKernel=brighterFatterKernel,
1180 bfGains=brighterFatterGains,
1182 fringes=fringeStruct,
1183 opticsTransmission=opticsTransmission,
1184 filterTransmission=filterTransmission,
1185 sensorTransmission=sensorTransmission,
1186 atmosphereTransmission=atmosphereTransmission,
1187 strayLightData=strayLightData,
1188 illumMaskedImage=illumMaskedImage
1191 @pipeBase.timeMethod
1192 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1193 crosstalk=None, crosstalkSources=None,
1194 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1195 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1196 sensorTransmission=
None, atmosphereTransmission=
None,
1197 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1200 """Perform instrument signature removal on an exposure.
1202 Steps included in the ISR processing, in order performed, are:
1203 - saturation and suspect pixel masking
1204 - overscan subtraction
1205 - CCD assembly of individual amplifiers
1207 - variance image construction
1208 - linearization of non-linear response
1210 - brighter-fatter correction
1213 - stray light subtraction
1215 - masking of known defects and camera specific features
1216 - vignette calculation
1217 - appending transmission curve and distortion model
1221 ccdExposure : `lsst.afw.image.Exposure`
1222 The raw exposure that is to be run through ISR. The
1223 exposure is modified by this method.
1224 camera : `lsst.afw.cameraGeom.Camera`, optional
1225 The camera geometry for this exposure. Required if ``isGen3`` is
1226 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1227 ``flat`` does not have an associated detector.
1228 bias : `lsst.afw.image.Exposure`, optional
1229 Bias calibration frame.
1230 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1231 Functor for linearization.
1232 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1233 Calibration for crosstalk.
1234 crosstalkSources : `list`, optional
1235 List of possible crosstalk sources.
1236 dark : `lsst.afw.image.Exposure`, optional
1237 Dark calibration frame.
1238 flat : `lsst.afw.image.Exposure`, optional
1239 Flat calibration frame.
1240 bfKernel : `numpy.ndarray`, optional
1241 Brighter-fatter kernel.
1242 bfGains : `dict` of `float`, optional
1243 Gains used to override the detector's nominal gains for the
1244 brighter-fatter correction. A dict keyed by amplifier name for
1245 the detector in question.
1246 defects : `lsst.ip.isr.Defects`, optional
1248 fringes : `lsst.pipe.base.Struct`, optional
1249 Struct containing the fringe correction data, with
1251 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1252 - ``seed``: random seed derived from the ccdExposureId for random
1253 number generator (`uint32`)
1254 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1255 A ``TransmissionCurve`` that represents the throughput of the optics,
1256 to be evaluated in focal-plane coordinates.
1257 filterTransmission : `lsst.afw.image.TransmissionCurve`
1258 A ``TransmissionCurve`` that represents the throughput of the filter
1259 itself, to be evaluated in focal-plane coordinates.
1260 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1261 A ``TransmissionCurve`` that represents the throughput of the sensor
1262 itself, to be evaluated in post-assembly trimmed detector coordinates.
1263 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1264 A ``TransmissionCurve`` that represents the throughput of the
1265 atmosphere, assumed to be spatially constant.
1266 detectorNum : `int`, optional
1267 The integer number for the detector to process.
1268 isGen3 : bool, optional
1269 Flag this call to run() as using the Gen3 butler environment.
1270 strayLightData : `object`, optional
1271 Opaque object containing calibration information for stray-light
1272 correction. If `None`, no correction will be performed.
1273 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1274 Illumination correction image.
1278 result : `lsst.pipe.base.Struct`
1279 Result struct with component:
1280 - ``exposure`` : `afw.image.Exposure`
1281 The fully ISR corrected exposure.
1282 - ``outputExposure`` : `afw.image.Exposure`
1283 An alias for `exposure`
1284 - ``ossThumb`` : `numpy.ndarray`
1285 Thumbnail image of the exposure after overscan subtraction.
1286 - ``flattenedThumb`` : `numpy.ndarray`
1287 Thumbnail image of the exposure after flat-field correction.
1292 Raised if a configuration option is set to True, but the
1293 required calibration data has not been specified.
1297 The current processed exposure can be viewed by setting the
1298 appropriate lsstDebug entries in the `debug.display`
1299 dictionary. The names of these entries correspond to some of
1300 the IsrTaskConfig Boolean options, with the value denoting the
1301 frame to use. The exposure is shown inside the matching
1302 option check and after the processing of that step has
1303 finished. The steps with debug points are:
1314 In addition, setting the "postISRCCD" entry displays the
1315 exposure after all ISR processing has finished.
1323 if detectorNum
is None:
1324 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1326 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1327 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1328 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1329 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1331 if isinstance(ccdExposure, ButlerDataRef):
1332 return self.
runDataRefrunDataRef(ccdExposure)
1334 ccd = ccdExposure.getDetector()
1336 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1339 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1340 ccd = [
FakeAmp(ccdExposure, self.config)]
1343 if self.config.doBias
and bias
is None:
1344 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1345 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1346 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1347 if self.config.doBrighterFatter
and bfKernel
is None:
1348 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1349 if self.config.doDark
and dark
is None:
1350 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1351 if self.config.doFlat
and flat
is None:
1352 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1353 if self.config.doDefect
and defects
is None:
1354 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1355 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1356 and fringes.fringes
is None):
1361 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1362 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1363 and illumMaskedImage
is None):
1364 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1367 if self.config.doConvertIntToFloat:
1368 self.log.info(
"Converting exposure to floating point values.")
1371 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1372 self.log.info(
"Applying bias correction.")
1373 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1374 trimToFit=self.config.doTrimToMatchCalib)
1375 self.
debugViewdebugView(ccdExposure,
"doBias")
1381 if ccdExposure.getBBox().contains(amp.getBBox()):
1383 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1385 if self.config.doOverscan
and not badAmp:
1388 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1389 if overscanResults
is not None and \
1390 self.config.qa
is not None and self.config.qa.saveStats
is True:
1391 if isinstance(overscanResults.overscanFit, float):
1392 qaMedian = overscanResults.overscanFit
1393 qaStdev = float(
"NaN")
1395 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1396 afwMath.MEDIAN | afwMath.STDEVCLIP)
1397 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1398 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1400 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1401 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1402 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1403 amp.getName(), qaMedian, qaStdev)
1406 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1407 afwMath.MEDIAN | afwMath.STDEVCLIP)
1408 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1409 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1411 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1412 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1413 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1414 amp.getName(), qaMedianAfter, qaStdevAfter)
1416 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1419 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1420 overscanResults =
None
1422 overscans.append(overscanResults
if overscanResults
is not None else None)
1424 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1426 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1427 self.log.info(
"Applying crosstalk correction.")
1428 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1429 crosstalkSources=crosstalkSources)
1430 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1432 if self.config.doAssembleCcd:
1433 self.log.info(
"Assembling CCD from amplifiers.")
1434 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1436 if self.config.expectWcs
and not ccdExposure.getWcs():
1437 self.log.warn(
"No WCS found in input exposure.")
1438 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1441 if self.config.qa.doThumbnailOss:
1442 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1444 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1445 self.log.info(
"Applying bias correction.")
1446 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1447 trimToFit=self.config.doTrimToMatchCalib)
1448 self.
debugViewdebugView(ccdExposure,
"doBias")
1450 if self.config.doVariance:
1451 for amp, overscanResults
in zip(ccd, overscans):
1452 if ccdExposure.getBBox().contains(amp.getBBox()):
1453 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1454 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1455 if overscanResults
is not None:
1457 overscanImage=overscanResults.overscanImage)
1461 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1462 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1463 afwMath.MEDIAN | afwMath.STDEVCLIP)
1464 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1465 qaStats.getValue(afwMath.MEDIAN))
1466 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1467 qaStats.getValue(afwMath.STDEVCLIP))
1468 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1469 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1470 qaStats.getValue(afwMath.STDEVCLIP))
1473 self.log.info(
"Applying linearizer.")
1474 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1475 detector=ccd, log=self.log)
1477 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1478 self.log.info(
"Applying crosstalk correction.")
1479 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1480 crosstalkSources=crosstalkSources, isTrimmed=
True)
1481 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1485 if self.config.doDefect:
1486 self.log.info(
"Masking defects.")
1487 self.
maskDefectmaskDefect(ccdExposure, defects)
1489 if self.config.numEdgeSuspect > 0:
1490 self.log.info(
"Masking edges as SUSPECT.")
1491 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1492 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1494 if self.config.doNanMasking:
1495 self.log.info(
"Masking NAN value pixels.")
1496 self.
maskNanmaskNan(ccdExposure)
1498 if self.config.doWidenSaturationTrails:
1499 self.log.info(
"Widening saturation trails.")
1500 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1502 if self.config.doCameraSpecificMasking:
1503 self.log.info(
"Masking regions for camera specific reasons.")
1504 self.masking.
run(ccdExposure)
1506 if self.config.doBrighterFatter:
1515 interpExp = ccdExposure.clone()
1516 with self.
flatContextflatContext(interpExp, flat, dark):
1517 isrFunctions.interpolateFromMask(
1518 maskedImage=interpExp.getMaskedImage(),
1519 fwhm=self.config.fwhm,
1520 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1521 maskNameList=self.config.maskListToInterpolate
1523 bfExp = interpExp.clone()
1525 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1526 type(bfKernel), type(bfGains))
1527 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1528 self.config.brighterFatterMaxIter,
1529 self.config.brighterFatterThreshold,
1530 self.config.brighterFatterApplyGain,
1532 if bfResults[1] == self.config.brighterFatterMaxIter:
1533 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1536 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1538 image = ccdExposure.getMaskedImage().getImage()
1539 bfCorr = bfExp.getMaskedImage().getImage()
1540 bfCorr -= interpExp.getMaskedImage().getImage()
1549 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1550 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1553 if self.config.brighterFatterMaskGrowSize > 0:
1554 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1555 for maskPlane
in self.config.maskListToInterpolate:
1556 isrFunctions.growMasks(ccdExposure.getMask(),
1557 radius=self.config.brighterFatterMaskGrowSize,
1558 maskNameList=maskPlane,
1559 maskValue=maskPlane)
1561 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1563 if self.config.doDark:
1564 self.log.info(
"Applying dark correction.")
1566 self.
debugViewdebugView(ccdExposure,
"doDark")
1568 if self.config.doFringe
and not self.config.fringeAfterFlat:
1569 self.log.info(
"Applying fringe correction before flat.")
1570 self.fringe.
run(ccdExposure, **fringes.getDict())
1571 self.
debugViewdebugView(ccdExposure,
"doFringe")
1573 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1574 self.log.info(
"Checking strayLight correction.")
1575 self.strayLight.
run(ccdExposure, strayLightData)
1576 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1578 if self.config.doFlat:
1579 self.log.info(
"Applying flat correction.")
1581 self.
debugViewdebugView(ccdExposure,
"doFlat")
1583 if self.config.doApplyGains:
1584 self.log.info(
"Applying gain correction instead of flat.")
1585 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1587 if self.config.doFringe
and self.config.fringeAfterFlat:
1588 self.log.info(
"Applying fringe correction after flat.")
1589 self.fringe.
run(ccdExposure, **fringes.getDict())
1591 if self.config.doVignette:
1592 self.log.info(
"Constructing Vignette polygon.")
1595 if self.config.vignette.doWriteVignettePolygon:
1598 if self.config.doAttachTransmissionCurve:
1599 self.log.info(
"Adding transmission curves.")
1600 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1601 filterTransmission=filterTransmission,
1602 sensorTransmission=sensorTransmission,
1603 atmosphereTransmission=atmosphereTransmission)
1605 flattenedThumb =
None
1606 if self.config.qa.doThumbnailFlattened:
1607 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1609 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1610 self.log.info(
"Performing illumination correction.")
1611 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1612 illumMaskedImage, illumScale=self.config.illumScale,
1613 trimToFit=self.config.doTrimToMatchCalib)
1616 if self.config.doSaveInterpPixels:
1617 preInterpExp = ccdExposure.clone()
1632 if self.config.doSetBadRegions:
1633 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1634 if badPixelCount > 0:
1635 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1637 if self.config.doInterpolate:
1638 self.log.info(
"Interpolating masked pixels.")
1639 isrFunctions.interpolateFromMask(
1640 maskedImage=ccdExposure.getMaskedImage(),
1641 fwhm=self.config.fwhm,
1642 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1643 maskNameList=list(self.config.maskListToInterpolate)
1648 if self.config.doMeasureBackground:
1649 self.log.info(
"Measuring background level.")
1652 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1654 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1655 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1656 afwMath.MEDIAN | afwMath.STDEVCLIP)
1657 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1658 qaStats.getValue(afwMath.MEDIAN))
1659 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1660 qaStats.getValue(afwMath.STDEVCLIP))
1661 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1662 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1663 qaStats.getValue(afwMath.STDEVCLIP))
1665 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1667 return pipeBase.Struct(
1668 exposure=ccdExposure,
1670 flattenedThumb=flattenedThumb,
1672 preInterpolatedExposure=preInterpExp,
1673 outputExposure=ccdExposure,
1674 outputOssThumbnail=ossThumb,
1675 outputFlattenedThumbnail=flattenedThumb,
1678 @pipeBase.timeMethod
1680 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1682 This method contains the `CmdLineTask` interface to the ISR
1683 processing. All IO is handled here, freeing the `run()` method
1684 to manage only pixel-level calculations. The steps performed
1686 - Read in necessary detrending/isr/calibration data.
1687 - Process raw exposure in `run()`.
1688 - Persist the ISR-corrected exposure as "postISRCCD" if
1689 config.doWrite=True.
1693 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1694 DataRef of the detector data to be processed
1698 result : `lsst.pipe.base.Struct`
1699 Result struct with component:
1700 - ``exposure`` : `afw.image.Exposure`
1701 The fully ISR corrected exposure.
1706 Raised if a configuration option is set to True, but the
1707 required calibration data does not exist.
1710 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1712 ccdExposure = sensorRef.get(self.config.datasetType)
1714 camera = sensorRef.get(
"camera")
1715 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1717 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1719 if self.config.doWrite:
1720 sensorRef.put(result.exposure,
"postISRCCD")
1721 if result.preInterpolatedExposure
is not None:
1722 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1723 if result.ossThumb
is not None:
1724 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1725 if result.flattenedThumb
is not None:
1726 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1731 """Retrieve a calibration dataset for removing instrument signature.
1736 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1737 DataRef of the detector data to find calibration datasets
1740 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1741 dateObs : `str`, optional
1742 Date of the observation. Used to correct butler failures
1743 when using fallback filters.
1745 If True, disable butler proxies to enable error handling
1746 within this routine.
1750 exposure : `lsst.afw.image.Exposure`
1751 Requested calibration frame.
1756 Raised if no matching calibration frame can be found.
1759 exp = dataRef.get(datasetType, immediate=immediate)
1760 except Exception
as exc1:
1761 if not self.config.fallbackFilterName:
1762 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1764 if self.config.useFallbackDate
and dateObs:
1765 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1766 dateObs=dateObs, immediate=immediate)
1768 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1769 except Exception
as exc2:
1770 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1771 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1772 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1774 if self.config.doAssembleIsrExposures:
1775 exp = self.assembleCcd.assembleCcd(exp)
1779 """Ensure that the data returned by Butler is a fully constructed exposure.
1781 ISR requires exposure-level image data for historical reasons, so if we did
1782 not recieve that from Butler, construct it from what we have, modifying the
1787 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1788 `lsst.afw.image.ImageF`
1789 The input data structure obtained from Butler.
1790 camera : `lsst.afw.cameraGeom.camera`
1791 The camera associated with the image. Used to find the appropriate
1794 The detector this exposure should match.
1798 inputExp : `lsst.afw.image.Exposure`
1799 The re-constructed exposure, with appropriate detector parameters.
1804 Raised if the input data cannot be used to construct an exposure.
1806 if isinstance(inputExp, afwImage.DecoratedImageU):
1807 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1808 elif isinstance(inputExp, afwImage.ImageF):
1809 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1810 elif isinstance(inputExp, afwImage.MaskedImageF):
1811 inputExp = afwImage.makeExposure(inputExp)
1812 elif isinstance(inputExp, afwImage.Exposure):
1814 elif inputExp
is None:
1818 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1821 if inputExp.getDetector()
is None:
1822 inputExp.setDetector(camera[detectorNum])
1827 """Convert exposure image from uint16 to float.
1829 If the exposure does not need to be converted, the input is
1830 immediately returned. For exposures that are converted to use
1831 floating point pixels, the variance is set to unity and the
1836 exposure : `lsst.afw.image.Exposure`
1837 The raw exposure to be converted.
1841 newexposure : `lsst.afw.image.Exposure`
1842 The input ``exposure``, converted to floating point pixels.
1847 Raised if the exposure type cannot be converted to float.
1850 if isinstance(exposure, afwImage.ExposureF):
1852 self.log.debug(
"Exposure already of type float.")
1854 if not hasattr(exposure,
"convertF"):
1855 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1857 newexposure = exposure.convertF()
1858 newexposure.variance[:] = 1
1859 newexposure.mask[:] = 0x0
1864 """Identify bad amplifiers, saturated and suspect pixels.
1868 ccdExposure : `lsst.afw.image.Exposure`
1869 Input exposure to be masked.
1870 amp : `lsst.afw.table.AmpInfoCatalog`
1871 Catalog of parameters defining the amplifier on this
1873 defects : `lsst.ip.isr.Defects`
1874 List of defects. Used to determine if the entire
1880 If this is true, the entire amplifier area is covered by
1881 defects and unusable.
1884 maskedImage = ccdExposure.getMaskedImage()
1890 if defects
is not None:
1891 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1896 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1898 maskView = dataView.getMask()
1899 maskView |= maskView.getPlaneBitMask(
"BAD")
1906 if self.config.doSaturation
and not badAmp:
1907 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1908 if self.config.doSuspect
and not badAmp:
1909 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1910 if math.isfinite(self.config.saturation):
1911 limits.update({self.config.saturatedMaskName: self.config.saturation})
1913 for maskName, maskThreshold
in limits.items():
1914 if not math.isnan(maskThreshold):
1915 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1916 isrFunctions.makeThresholdMask(
1917 maskedImage=dataView,
1918 threshold=maskThreshold,
1924 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1926 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1927 self.config.suspectMaskName])
1928 if numpy.all(maskView.getArray() & maskVal > 0):
1930 maskView |= maskView.getPlaneBitMask(
"BAD")
1935 """Apply overscan correction in place.
1937 This method does initial pixel rejection of the overscan
1938 region. The overscan can also be optionally segmented to
1939 allow for discontinuous overscan responses to be fit
1940 separately. The actual overscan subtraction is performed by
1941 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1942 which is called here after the amplifier is preprocessed.
1946 ccdExposure : `lsst.afw.image.Exposure`
1947 Exposure to have overscan correction performed.
1948 amp : `lsst.afw.cameraGeom.Amplifer`
1949 The amplifier to consider while correcting the overscan.
1953 overscanResults : `lsst.pipe.base.Struct`
1954 Result struct with components:
1955 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1956 Value or fit subtracted from the amplifier image data.
1957 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1958 Value or fit subtracted from the overscan image data.
1959 - ``overscanImage`` : `lsst.afw.image.Image`
1960 Image of the overscan region with the overscan
1961 correction applied. This quantity is used to estimate
1962 the amplifier read noise empirically.
1967 Raised if the ``amp`` does not contain raw pixel information.
1971 lsst.ip.isr.isrFunctions.overscanCorrection
1973 if amp.getRawHorizontalOverscanBBox().isEmpty():
1974 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1977 statControl = afwMath.StatisticsControl()
1978 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1981 dataBBox = amp.getRawDataBBox()
1982 oscanBBox = amp.getRawHorizontalOverscanBBox()
1986 prescanBBox = amp.getRawPrescanBBox()
1987 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1988 dx0 += self.config.overscanNumLeadingColumnsToSkip
1989 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1991 dx0 += self.config.overscanNumTrailingColumnsToSkip
1992 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1998 if ((self.config.overscanBiasJump
1999 and self.config.overscanBiasJumpLocation)
2000 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2001 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2002 self.config.overscanBiasJumpDevices)):
2003 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2004 yLower = self.config.overscanBiasJumpLocation
2005 yUpper = dataBBox.getHeight() - yLower
2007 yUpper = self.config.overscanBiasJumpLocation
2008 yLower = dataBBox.getHeight() - yUpper
2026 oscanBBox.getHeight())))
2029 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2030 ampImage = ccdExposure.maskedImage[imageBBox]
2031 overscanImage = ccdExposure.maskedImage[overscanBBox]
2033 overscanArray = overscanImage.image.array
2034 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2035 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2036 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2038 statControl = afwMath.StatisticsControl()
2039 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2041 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2044 levelStat = afwMath.MEDIAN
2045 sigmaStat = afwMath.STDEVCLIP
2047 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2048 self.config.qa.flatness.nIter)
2049 metadata = ccdExposure.getMetadata()
2050 ampNum = amp.getName()
2052 if isinstance(overscanResults.overscanFit, float):
2053 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2054 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2056 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2057 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2058 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2060 return overscanResults
2063 """Set the variance plane using the amplifier gain and read noise
2065 The read noise is calculated from the ``overscanImage`` if the
2066 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2067 the value from the amplifier data is used.
2071 ampExposure : `lsst.afw.image.Exposure`
2072 Exposure to process.
2073 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2074 Amplifier detector data.
2075 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2076 Image of overscan, required only for empirical read noise.
2080 lsst.ip.isr.isrFunctions.updateVariance
2082 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2083 gain = amp.getGain()
2085 if math.isnan(gain):
2087 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2090 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2091 amp.getName(), gain, patchedGain)
2094 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2095 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
2097 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2098 stats = afwMath.StatisticsControl()
2099 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2100 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2101 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2102 amp.getName(), readNoise)
2104 readNoise = amp.getReadNoise()
2106 isrFunctions.updateVariance(
2107 maskedImage=ampExposure.getMaskedImage(),
2109 readNoise=readNoise,
2113 """Apply dark correction in place.
2117 exposure : `lsst.afw.image.Exposure`
2118 Exposure to process.
2119 darkExposure : `lsst.afw.image.Exposure`
2120 Dark exposure of the same size as ``exposure``.
2121 invert : `Bool`, optional
2122 If True, re-add the dark to an already corrected image.
2127 Raised if either ``exposure`` or ``darkExposure`` do not
2128 have their dark time defined.
2132 lsst.ip.isr.isrFunctions.darkCorrection
2134 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2135 if math.isnan(expScale):
2136 raise RuntimeError(
"Exposure darktime is NAN.")
2137 if darkExposure.getInfo().getVisitInfo()
is not None \
2138 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2139 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2143 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2146 isrFunctions.darkCorrection(
2147 maskedImage=exposure.getMaskedImage(),
2148 darkMaskedImage=darkExposure.getMaskedImage(),
2150 darkScale=darkScale,
2152 trimToFit=self.config.doTrimToMatchCalib
2156 """Check if linearization is needed for the detector cameraGeom.
2158 Checks config.doLinearize and the linearity type of the first
2163 detector : `lsst.afw.cameraGeom.Detector`
2164 Detector to get linearity type from.
2168 doLinearize : `Bool`
2169 If True, linearization should be performed.
2171 return self.config.doLinearize
and \
2172 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2175 """Apply flat correction in place.
2179 exposure : `lsst.afw.image.Exposure`
2180 Exposure to process.
2181 flatExposure : `lsst.afw.image.Exposure`
2182 Flat exposure of the same size as ``exposure``.
2183 invert : `Bool`, optional
2184 If True, unflatten an already flattened image.
2188 lsst.ip.isr.isrFunctions.flatCorrection
2190 isrFunctions.flatCorrection(
2191 maskedImage=exposure.getMaskedImage(),
2192 flatMaskedImage=flatExposure.getMaskedImage(),
2193 scalingType=self.config.flatScalingType,
2194 userScale=self.config.flatUserScale,
2196 trimToFit=self.config.doTrimToMatchCalib
2200 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2204 exposure : `lsst.afw.image.Exposure`
2205 Exposure to process. Only the amplifier DataSec is processed.
2206 amp : `lsst.afw.table.AmpInfoCatalog`
2207 Amplifier detector data.
2211 lsst.ip.isr.isrFunctions.makeThresholdMask
2213 if not math.isnan(amp.getSaturation()):
2214 maskedImage = exposure.getMaskedImage()
2215 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2216 isrFunctions.makeThresholdMask(
2217 maskedImage=dataView,
2218 threshold=amp.getSaturation(),
2220 maskName=self.config.saturatedMaskName,
2224 """Interpolate over saturated pixels, in place.
2226 This method should be called after `saturationDetection`, to
2227 ensure that the saturated pixels have been identified in the
2228 SAT mask. It should also be called after `assembleCcd`, since
2229 saturated regions may cross amplifier boundaries.
2233 exposure : `lsst.afw.image.Exposure`
2234 Exposure to process.
2238 lsst.ip.isr.isrTask.saturationDetection
2239 lsst.ip.isr.isrFunctions.interpolateFromMask
2241 isrFunctions.interpolateFromMask(
2242 maskedImage=exposure.getMaskedImage(),
2243 fwhm=self.config.fwhm,
2244 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2245 maskNameList=list(self.config.saturatedMaskName),
2249 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2253 exposure : `lsst.afw.image.Exposure`
2254 Exposure to process. Only the amplifier DataSec is processed.
2255 amp : `lsst.afw.table.AmpInfoCatalog`
2256 Amplifier detector data.
2260 lsst.ip.isr.isrFunctions.makeThresholdMask
2264 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2265 This is intended to indicate pixels that may be affected by unknown systematics;
2266 for example if non-linearity corrections above a certain level are unstable
2267 then that would be a useful value for suspectLevel. A value of `nan` indicates
2268 that no such level exists and no pixels are to be masked as suspicious.
2270 suspectLevel = amp.getSuspectLevel()
2271 if math.isnan(suspectLevel):
2274 maskedImage = exposure.getMaskedImage()
2275 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2276 isrFunctions.makeThresholdMask(
2277 maskedImage=dataView,
2278 threshold=suspectLevel,
2280 maskName=self.config.suspectMaskName,
2284 """Mask defects using mask plane "BAD", in place.
2288 exposure : `lsst.afw.image.Exposure`
2289 Exposure to process.
2290 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2291 `lsst.afw.image.DefectBase`.
2292 List of defects to mask.
2296 Call this after CCD assembly, since defects may cross amplifier boundaries.
2298 maskedImage = exposure.getMaskedImage()
2299 if not isinstance(defectBaseList, Defects):
2301 defectList =
Defects(defectBaseList)
2303 defectList = defectBaseList
2304 defectList.maskPixels(maskedImage, maskName=
"BAD")
2306 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2307 """Mask edge pixels with applicable mask plane.
2311 exposure : `lsst.afw.image.Exposure`
2312 Exposure to process.
2313 numEdgePixels : `int`, optional
2314 Number of edge pixels to mask.
2315 maskPlane : `str`, optional
2316 Mask plane name to use.
2317 level : `str`, optional
2318 Level at which to mask edges.
2320 maskedImage = exposure.getMaskedImage()
2321 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2323 if numEdgePixels > 0:
2324 if level ==
'DETECTOR':
2325 boxes = [maskedImage.getBBox()]
2326 elif level ==
'AMP':
2327 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2331 subImage = maskedImage[box]
2332 box.grow(-numEdgePixels)
2334 SourceDetectionTask.setEdgeBits(
2340 """Mask and interpolate defects using mask plane "BAD", in place.
2344 exposure : `lsst.afw.image.Exposure`
2345 Exposure to process.
2346 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2347 `lsst.afw.image.DefectBase`.
2348 List of defects to mask and interpolate.
2352 lsst.ip.isr.isrTask.maskDefect
2354 self.
maskDefectmaskDefect(exposure, defectBaseList)
2355 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2356 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2357 isrFunctions.interpolateFromMask(
2358 maskedImage=exposure.getMaskedImage(),
2359 fwhm=self.config.fwhm,
2360 growSaturatedFootprints=0,
2361 maskNameList=[
"BAD"],
2365 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2369 exposure : `lsst.afw.image.Exposure`
2370 Exposure to process.
2374 We mask over all NaNs, including those that are masked with
2375 other bits (because those may or may not be interpolated over
2376 later, and we want to remove all NaNs). Despite this
2377 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2378 the historical name.
2380 maskedImage = exposure.getMaskedImage()
2383 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2384 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2385 numNans =
maskNans(maskedImage, maskVal)
2386 self.metadata.set(
"NUMNANS", numNans)
2388 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2391 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2395 exposure : `lsst.afw.image.Exposure`
2396 Exposure to process.
2400 lsst.ip.isr.isrTask.maskNan
2403 isrFunctions.interpolateFromMask(
2404 maskedImage=exposure.getMaskedImage(),
2405 fwhm=self.config.fwhm,
2406 growSaturatedFootprints=0,
2407 maskNameList=[
"UNMASKEDNAN"],
2411 """Measure the image background in subgrids, for quality control purposes.
2415 exposure : `lsst.afw.image.Exposure`
2416 Exposure to process.
2417 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2418 Configuration object containing parameters on which background
2419 statistics and subgrids to use.
2421 if IsrQaConfig
is not None:
2422 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2423 IsrQaConfig.flatness.nIter)
2424 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2425 statsControl.setAndMask(maskVal)
2426 maskedImage = exposure.getMaskedImage()
2427 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2428 skyLevel = stats.getValue(afwMath.MEDIAN)
2429 skySigma = stats.getValue(afwMath.STDEVCLIP)
2430 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2431 metadata = exposure.getMetadata()
2432 metadata.set(
'SKYLEVEL', skyLevel)
2433 metadata.set(
'SKYSIGMA', skySigma)
2436 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2437 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2438 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2439 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2440 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2441 skyLevels = numpy.zeros((nX, nY))
2444 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2446 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2448 xLLC = xc - meshXHalf
2449 yLLC = yc - meshYHalf
2450 xURC = xc + meshXHalf - 1
2451 yURC = yc + meshYHalf - 1
2454 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2456 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2458 good = numpy.where(numpy.isfinite(skyLevels))
2459 skyMedian = numpy.median(skyLevels[good])
2460 flatness = (skyLevels[good] - skyMedian) / skyMedian
2461 flatness_rms = numpy.std(flatness)
2462 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2464 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2465 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2466 nX, nY, flatness_pp, flatness_rms)
2468 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2469 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2470 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2471 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2472 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2475 """Set an approximate magnitude zero point for the exposure.
2479 exposure : `lsst.afw.image.Exposure`
2480 Exposure to process.
2483 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2484 if filterName
in self.config.fluxMag0T1:
2485 fluxMag0 = self.config.fluxMag0T1[filterName]
2487 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2488 fluxMag0 = self.config.defaultFluxMag0T1
2490 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2492 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2495 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2496 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2499 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2503 ccdExposure : `lsst.afw.image.Exposure`
2504 Exposure to process.
2505 fpPolygon : `lsst.afw.geom.Polygon`
2506 Polygon in focal plane coordinates.
2509 ccd = ccdExposure.getDetector()
2510 fpCorners = ccd.getCorners(FOCAL_PLANE)
2511 ccdPolygon = Polygon(fpCorners)
2514 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2517 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2518 validPolygon = Polygon(ccdPoints)
2519 ccdExposure.getInfo().setValidPolygon(validPolygon)
2523 """Context manager that applies and removes flats and darks,
2524 if the task is configured to apply them.
2528 exp : `lsst.afw.image.Exposure`
2529 Exposure to process.
2530 flat : `lsst.afw.image.Exposure`
2531 Flat exposure the same size as ``exp``.
2532 dark : `lsst.afw.image.Exposure`, optional
2533 Dark exposure the same size as ``exp``.
2537 exp : `lsst.afw.image.Exposure`
2538 The flat and dark corrected exposure.
2540 if self.config.doDark
and dark
is not None:
2542 if self.config.doFlat:
2547 if self.config.doFlat:
2549 if self.config.doDark
and dark
is not None:
2553 """Utility function to examine ISR exposure at different stages.
2557 exposure : `lsst.afw.image.Exposure`
2560 State of processing to view.
2562 frame = getDebugFrame(self._display, stepname)
2564 display = getDisplay(frame)
2565 display.scale(
'asinh',
'zscale')
2566 display.mtv(exposure)
2567 prompt =
"Press Enter to continue [c]... "
2569 ans = input(prompt).lower()
2570 if ans
in (
"",
"c",):
2575 """A Detector-like object that supports returning gain and saturation level
2577 This is used when the input exposure does not have a detector.
2581 exposure : `lsst.afw.image.Exposure`
2582 Exposure to generate a fake amplifier for.
2583 config : `lsst.ip.isr.isrTaskConfig`
2584 Configuration to apply to the fake amplifier.
2588 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2590 self.
_gain_gain = config.gain
2595 return self.
_bbox_bbox
2598 return self.
_bbox_bbox
2604 return self.
_gain_gain
2617 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2621 """Task to wrap the default IsrTask to allow it to be retargeted.
2623 The standard IsrTask can be called directly from a command line
2624 program, but doing so removes the ability of the task to be
2625 retargeted. As most cameras override some set of the IsrTask
2626 methods, this would remove those data-specific methods in the
2627 output post-ISR images. This wrapping class fixes the issue,
2628 allowing identical post-ISR images to be generated by both the
2629 processCcd and isrTask code.
2631 ConfigClass = RunIsrConfig
2632 _DefaultName =
"runIsr"
2636 self.makeSubtask(
"isr")
2642 dataRef : `lsst.daf.persistence.ButlerDataRef`
2643 data reference of the detector data to be processed
2647 result : `pipeBase.Struct`
2648 Result struct with component:
2650 - exposure : `lsst.afw.image.Exposure`
2651 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def ensureExposure(self, inputExp, camera, detectorNum)
def updateVariance(self, ampExposure, amp, overscanImage=None)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.