29 import lsst.pipe.base
as pipeBase
30 import lsst.pipe.base.connectionTypes
as cT
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from .ampOffset
import AmpOffsetTask
57 from lsst.daf.butler
import DimensionGraph
60 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
64 """Lookup function to identify crosstalkSource entries.
66 This should return an empty list under most circumstances. Only
67 when inter-chip crosstalk has been identified should this be
70 This will be unused until DM-25348 resolves the quantum graph
77 registry : `lsst.daf.butler.Registry`
78 Butler registry to query.
79 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
80 Data id to transform to identify crosstalkSources. The
81 ``detector`` entry will be stripped.
82 collections : `lsst.daf.butler.CollectionSearch`
83 Collections to search through.
87 results : `list` [`lsst.daf.butler.DatasetRef`]
88 List of datasets that match the query that will be used as
91 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
92 results = list(registry.queryDatasets(datasetType,
93 collections=collections,
101 dimensions={
"instrument",
"exposure",
"detector"},
102 defaultTemplates={}):
103 ccdExposure = cT.Input(
105 doc=
"Input exposure to process.",
106 storageClass=
"Exposure",
107 dimensions=[
"instrument",
"exposure",
"detector"],
109 camera = cT.PrerequisiteInput(
111 storageClass=
"Camera",
112 doc=
"Input camera to construct complete exposures.",
113 dimensions=[
"instrument"],
117 crosstalk = cT.PrerequisiteInput(
119 doc=
"Input crosstalk object",
120 storageClass=
"CrosstalkCalib",
121 dimensions=[
"instrument",
"detector"],
127 crosstalkSources = cT.PrerequisiteInput(
128 name=
"isrOverscanCorrected",
129 doc=
"Overscan corrected input images.",
130 storageClass=
"Exposure",
131 dimensions=[
"instrument",
"exposure",
"detector"],
134 lookupFunction=crosstalkSourceLookup,
137 bias = cT.PrerequisiteInput(
139 doc=
"Input bias calibration.",
140 storageClass=
"ExposureF",
141 dimensions=[
"instrument",
"detector"],
144 dark = cT.PrerequisiteInput(
146 doc=
"Input dark calibration.",
147 storageClass=
"ExposureF",
148 dimensions=[
"instrument",
"detector"],
151 flat = cT.PrerequisiteInput(
153 doc=
"Input flat calibration.",
154 storageClass=
"ExposureF",
155 dimensions=[
"instrument",
"physical_filter",
"detector"],
158 ptc = cT.PrerequisiteInput(
160 doc=
"Input Photon Transfer Curve dataset",
161 storageClass=
"PhotonTransferCurveDataset",
162 dimensions=[
"instrument",
"detector"],
165 fringes = cT.PrerequisiteInput(
167 doc=
"Input fringe calibration.",
168 storageClass=
"ExposureF",
169 dimensions=[
"instrument",
"physical_filter",
"detector"],
173 strayLightData = cT.PrerequisiteInput(
175 doc=
"Input stray light calibration.",
176 storageClass=
"StrayLightData",
177 dimensions=[
"instrument",
"physical_filter",
"detector"],
182 bfKernel = cT.PrerequisiteInput(
184 doc=
"Input brighter-fatter kernel.",
185 storageClass=
"NumpyArray",
186 dimensions=[
"instrument"],
190 newBFKernel = cT.PrerequisiteInput(
191 name=
'brighterFatterKernel',
192 doc=
"Newer complete kernel + gain solutions.",
193 storageClass=
"BrighterFatterKernel",
194 dimensions=[
"instrument",
"detector"],
198 defects = cT.PrerequisiteInput(
200 doc=
"Input defect tables.",
201 storageClass=
"Defects",
202 dimensions=[
"instrument",
"detector"],
205 linearizer = cT.PrerequisiteInput(
207 storageClass=
"Linearizer",
208 doc=
"Linearity correction calibration.",
209 dimensions=[
"instrument",
"detector"],
213 opticsTransmission = cT.PrerequisiteInput(
214 name=
"transmission_optics",
215 storageClass=
"TransmissionCurve",
216 doc=
"Transmission curve due to the optics.",
217 dimensions=[
"instrument"],
220 filterTransmission = cT.PrerequisiteInput(
221 name=
"transmission_filter",
222 storageClass=
"TransmissionCurve",
223 doc=
"Transmission curve due to the filter.",
224 dimensions=[
"instrument",
"physical_filter"],
227 sensorTransmission = cT.PrerequisiteInput(
228 name=
"transmission_sensor",
229 storageClass=
"TransmissionCurve",
230 doc=
"Transmission curve due to the sensor.",
231 dimensions=[
"instrument",
"detector"],
234 atmosphereTransmission = cT.PrerequisiteInput(
235 name=
"transmission_atmosphere",
236 storageClass=
"TransmissionCurve",
237 doc=
"Transmission curve due to the atmosphere.",
238 dimensions=[
"instrument"],
241 illumMaskedImage = cT.PrerequisiteInput(
243 doc=
"Input illumination correction.",
244 storageClass=
"MaskedImageF",
245 dimensions=[
"instrument",
"physical_filter",
"detector"],
249 outputExposure = cT.Output(
251 doc=
"Output ISR processed exposure.",
252 storageClass=
"Exposure",
253 dimensions=[
"instrument",
"exposure",
"detector"],
255 preInterpExposure = cT.Output(
256 name=
'preInterpISRCCD',
257 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
258 storageClass=
"ExposureF",
259 dimensions=[
"instrument",
"exposure",
"detector"],
261 outputOssThumbnail = cT.Output(
263 doc=
"Output Overscan-subtracted thumbnail image.",
264 storageClass=
"Thumbnail",
265 dimensions=[
"instrument",
"exposure",
"detector"],
267 outputFlattenedThumbnail = cT.Output(
268 name=
"FlattenedThumb",
269 doc=
"Output flat-corrected thumbnail image.",
270 storageClass=
"Thumbnail",
271 dimensions=[
"instrument",
"exposure",
"detector"],
277 if config.doBias
is not True:
278 self.prerequisiteInputs.discard(
"bias")
279 if config.doLinearize
is not True:
280 self.prerequisiteInputs.discard(
"linearizer")
281 if config.doCrosstalk
is not True:
282 self.inputs.discard(
"crosstalkSources")
283 self.prerequisiteInputs.discard(
"crosstalk")
284 if config.doBrighterFatter
is not True:
285 self.prerequisiteInputs.discard(
"bfKernel")
286 self.prerequisiteInputs.discard(
"newBFKernel")
287 if config.doDefect
is not True:
288 self.prerequisiteInputs.discard(
"defects")
289 if config.doDark
is not True:
290 self.prerequisiteInputs.discard(
"dark")
291 if config.doFlat
is not True:
292 self.prerequisiteInputs.discard(
"flat")
293 if config.doFringe
is not True:
294 self.prerequisiteInputs.discard(
"fringe")
295 if config.doStrayLight
is not True:
296 self.prerequisiteInputs.discard(
"strayLightData")
297 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
298 self.prerequisiteInputs.discard(
"ptc")
299 if config.doAttachTransmissionCurve
is not True:
300 self.prerequisiteInputs.discard(
"opticsTransmission")
301 self.prerequisiteInputs.discard(
"filterTransmission")
302 self.prerequisiteInputs.discard(
"sensorTransmission")
303 self.prerequisiteInputs.discard(
"atmosphereTransmission")
304 if config.doUseOpticsTransmission
is not True:
305 self.prerequisiteInputs.discard(
"opticsTransmission")
306 if config.doUseFilterTransmission
is not True:
307 self.prerequisiteInputs.discard(
"filterTransmission")
308 if config.doUseSensorTransmission
is not True:
309 self.prerequisiteInputs.discard(
"sensorTransmission")
310 if config.doUseAtmosphereTransmission
is not True:
311 self.prerequisiteInputs.discard(
"atmosphereTransmission")
312 if config.doIlluminationCorrection
is not True:
313 self.prerequisiteInputs.discard(
"illumMaskedImage")
315 if config.doWrite
is not True:
316 self.outputs.discard(
"outputExposure")
317 self.outputs.discard(
"preInterpExposure")
318 self.outputs.discard(
"outputFlattenedThumbnail")
319 self.outputs.discard(
"outputOssThumbnail")
320 if config.doSaveInterpPixels
is not True:
321 self.outputs.discard(
"preInterpExposure")
322 if config.qa.doThumbnailOss
is not True:
323 self.outputs.discard(
"outputOssThumbnail")
324 if config.qa.doThumbnailFlattened
is not True:
325 self.outputs.discard(
"outputFlattenedThumbnail")
329 pipelineConnections=IsrTaskConnections):
330 """Configuration parameters for IsrTask.
332 Items are grouped in the order in which they are executed by the task.
334 datasetType = pexConfig.Field(
336 doc=
"Dataset type for input data; users will typically leave this alone, "
337 "but camera-specific ISR tasks will override it",
341 fallbackFilterName = pexConfig.Field(
343 doc=
"Fallback default filter name for calibrations.",
346 useFallbackDate = pexConfig.Field(
348 doc=
"Pass observation date when using fallback filter.",
351 expectWcs = pexConfig.Field(
354 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
356 fwhm = pexConfig.Field(
358 doc=
"FWHM of PSF in arcseconds.",
361 qa = pexConfig.ConfigField(
363 doc=
"QA related configuration options.",
367 doConvertIntToFloat = pexConfig.Field(
369 doc=
"Convert integer raw images to floating point values?",
374 doSaturation = pexConfig.Field(
376 doc=
"Mask saturated pixels? NB: this is totally independent of the"
377 " interpolation option - this is ONLY setting the bits in the mask."
378 " To have them interpolated make sure doSaturationInterpolation=True",
381 saturatedMaskName = pexConfig.Field(
383 doc=
"Name of mask plane to use in saturation detection and interpolation",
386 saturation = pexConfig.Field(
388 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
389 default=float(
"NaN"),
391 growSaturationFootprintSize = pexConfig.Field(
393 doc=
"Number of pixels by which to grow the saturation footprints",
398 doSuspect = pexConfig.Field(
400 doc=
"Mask suspect pixels?",
403 suspectMaskName = pexConfig.Field(
405 doc=
"Name of mask plane to use for suspect pixels",
408 numEdgeSuspect = pexConfig.Field(
410 doc=
"Number of edge pixels to be flagged as untrustworthy.",
413 edgeMaskLevel = pexConfig.ChoiceField(
415 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
418 'DETECTOR':
'Mask only the edges of the full detector.',
419 'AMP':
'Mask edges of each amplifier.',
424 doSetBadRegions = pexConfig.Field(
426 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
429 badStatistic = pexConfig.ChoiceField(
431 doc=
"How to estimate the average value for BAD regions.",
434 "MEANCLIP":
"Correct using the (clipped) mean of good data",
435 "MEDIAN":
"Correct using the median of the good data",
440 doOverscan = pexConfig.Field(
442 doc=
"Do overscan subtraction?",
445 overscan = pexConfig.ConfigurableField(
446 target=OverscanCorrectionTask,
447 doc=
"Overscan subtraction task for image segments.",
449 overscanFitType = pexConfig.ChoiceField(
451 doc=
"The method for fitting the overscan bias level.",
454 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
455 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
456 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
457 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
458 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
459 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
460 "MEAN":
"Correct using the mean of the overscan region",
461 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
462 "MEDIAN":
"Correct using the median of the overscan region",
463 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
465 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
466 " This option will no longer be used, and will be removed after v20.")
468 overscanOrder = pexConfig.Field(
470 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
471 "or number of spline knots if overscan fit type is a spline."),
473 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
474 " This option will no longer be used, and will be removed after v20.")
476 overscanNumSigmaClip = pexConfig.Field(
478 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
480 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
481 " This option will no longer be used, and will be removed after v20.")
483 overscanIsInt = pexConfig.Field(
485 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
486 " and overscan.FitType=MEDIAN_PER_ROW.",
488 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
489 " This option will no longer be used, and will be removed after v20.")
493 overscanNumLeadingColumnsToSkip = pexConfig.Field(
495 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
498 overscanNumTrailingColumnsToSkip = pexConfig.Field(
500 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
503 overscanMaxDev = pexConfig.Field(
505 doc=
"Maximum deviation from the median for overscan",
506 default=1000.0, check=
lambda x: x > 0
508 overscanBiasJump = pexConfig.Field(
510 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
513 overscanBiasJumpKeyword = pexConfig.Field(
515 doc=
"Header keyword containing information about devices.",
516 default=
"NO_SUCH_KEY",
518 overscanBiasJumpDevices = pexConfig.ListField(
520 doc=
"List of devices that need piecewise overscan correction.",
523 overscanBiasJumpLocation = pexConfig.Field(
525 doc=
"Location of bias jump along y-axis.",
530 doAssembleCcd = pexConfig.Field(
533 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
535 assembleCcd = pexConfig.ConfigurableField(
536 target=AssembleCcdTask,
537 doc=
"CCD assembly task",
541 doAssembleIsrExposures = pexConfig.Field(
544 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
546 doTrimToMatchCalib = pexConfig.Field(
549 doc=
"Trim raw data to match calibration bounding boxes?"
553 doBias = pexConfig.Field(
555 doc=
"Apply bias frame correction?",
558 biasDataProductName = pexConfig.Field(
560 doc=
"Name of the bias data product",
563 doBiasBeforeOverscan = pexConfig.Field(
565 doc=
"Reverse order of overscan and bias correction.",
570 doVariance = pexConfig.Field(
572 doc=
"Calculate variance?",
575 gain = pexConfig.Field(
577 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
578 default=float(
"NaN"),
580 readNoise = pexConfig.Field(
582 doc=
"The read noise to use if no Detector is present in the Exposure",
585 doEmpiricalReadNoise = pexConfig.Field(
588 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
590 usePtcReadNoise = pexConfig.Field(
593 doc=
"Use readnoise values from the Photon Transfer Curve?"
595 maskNegativeVariance = pexConfig.Field(
598 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
599 "in the measurement of the overscan at an edge due to the data falling off faster "
600 "than the overscan model can account for it."
602 negativeVarianceMaskName = pexConfig.Field(
605 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
608 doLinearize = pexConfig.Field(
610 doc=
"Correct for nonlinearity of the detector's response?",
615 doCrosstalk = pexConfig.Field(
617 doc=
"Apply intra-CCD crosstalk correction?",
620 doCrosstalkBeforeAssemble = pexConfig.Field(
622 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
625 crosstalk = pexConfig.ConfigurableField(
626 target=CrosstalkTask,
627 doc=
"Intra-CCD crosstalk correction",
631 doDefect = pexConfig.Field(
633 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
636 doNanMasking = pexConfig.Field(
638 doc=
"Mask non-finite (NAN, inf) pixels?",
641 doWidenSaturationTrails = pexConfig.Field(
643 doc=
"Widen bleed trails based on their width?",
648 doBrighterFatter = pexConfig.Field(
651 doc=
"Apply the brighter-fatter correction?"
653 brighterFatterLevel = pexConfig.ChoiceField(
656 doc=
"The level at which to correct for brighter-fatter.",
658 "AMP":
"Every amplifier treated separately.",
659 "DETECTOR":
"One kernel per detector",
662 brighterFatterMaxIter = pexConfig.Field(
665 doc=
"Maximum number of iterations for the brighter-fatter correction"
667 brighterFatterThreshold = pexConfig.Field(
670 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
671 "absolute value of the difference between the current corrected image and the one "
672 "from the previous iteration summed over all the pixels."
674 brighterFatterApplyGain = pexConfig.Field(
677 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
679 brighterFatterMaskListToInterpolate = pexConfig.ListField(
681 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
683 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
685 brighterFatterMaskGrowSize = pexConfig.Field(
688 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
689 "when brighter-fatter correction is applied."
693 doDark = pexConfig.Field(
695 doc=
"Apply dark frame correction?",
698 darkDataProductName = pexConfig.Field(
700 doc=
"Name of the dark data product",
705 doStrayLight = pexConfig.Field(
707 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
710 strayLight = pexConfig.ConfigurableField(
711 target=StrayLightTask,
712 doc=
"y-band stray light correction"
716 doFlat = pexConfig.Field(
718 doc=
"Apply flat field correction?",
721 flatDataProductName = pexConfig.Field(
723 doc=
"Name of the flat data product",
726 flatScalingType = pexConfig.ChoiceField(
728 doc=
"The method for scaling the flat on the fly.",
731 "USER":
"Scale by flatUserScale",
732 "MEAN":
"Scale by the inverse of the mean",
733 "MEDIAN":
"Scale by the inverse of the median",
736 flatUserScale = pexConfig.Field(
738 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
741 doTweakFlat = pexConfig.Field(
743 doc=
"Tweak flats to match observed amplifier ratios?",
749 doApplyGains = pexConfig.Field(
751 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
754 usePtcGains = pexConfig.Field(
756 doc=
"Use the gain values from the Photon Transfer Curve?",
759 normalizeGains = pexConfig.Field(
761 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
766 doFringe = pexConfig.Field(
768 doc=
"Apply fringe correction?",
771 fringe = pexConfig.ConfigurableField(
773 doc=
"Fringe subtraction task",
775 fringeAfterFlat = pexConfig.Field(
777 doc=
"Do fringe subtraction after flat-fielding?",
782 doAmpOffset = pexConfig.Field(
783 doc=
"Calculate and apply amp offset corrections?",
787 ampOffset = pexConfig.ConfigurableField(
788 doc=
"Amp offset correction task.",
789 target=AmpOffsetTask,
793 doMeasureBackground = pexConfig.Field(
795 doc=
"Measure the background level on the reduced image?",
800 doCameraSpecificMasking = pexConfig.Field(
802 doc=
"Mask camera-specific bad regions?",
805 masking = pexConfig.ConfigurableField(
811 doInterpolate = pexConfig.Field(
813 doc=
"Interpolate masked pixels?",
816 doSaturationInterpolation = pexConfig.Field(
818 doc=
"Perform interpolation over pixels masked as saturated?"
819 " NB: This is independent of doSaturation; if that is False this plane"
820 " will likely be blank, resulting in a no-op here.",
823 doNanInterpolation = pexConfig.Field(
825 doc=
"Perform interpolation over pixels masked as NaN?"
826 " NB: This is independent of doNanMasking; if that is False this plane"
827 " will likely be blank, resulting in a no-op here.",
830 doNanInterpAfterFlat = pexConfig.Field(
832 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
833 "also have to interpolate them before flat-fielding."),
836 maskListToInterpolate = pexConfig.ListField(
838 doc=
"List of mask planes that should be interpolated.",
839 default=[
'SAT',
'BAD'],
841 doSaveInterpPixels = pexConfig.Field(
843 doc=
"Save a copy of the pre-interpolated pixel values?",
848 fluxMag0T1 = pexConfig.DictField(
851 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
852 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
855 defaultFluxMag0T1 = pexConfig.Field(
857 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
858 default=pow(10.0, 0.4*28.0)
862 doVignette = pexConfig.Field(
864 doc=
"Apply vignetting parameters?",
867 vignette = pexConfig.ConfigurableField(
869 doc=
"Vignetting task.",
873 doAttachTransmissionCurve = pexConfig.Field(
876 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
878 doUseOpticsTransmission = pexConfig.Field(
881 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
883 doUseFilterTransmission = pexConfig.Field(
886 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
888 doUseSensorTransmission = pexConfig.Field(
891 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
893 doUseAtmosphereTransmission = pexConfig.Field(
896 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
900 doIlluminationCorrection = pexConfig.Field(
903 doc=
"Perform illumination correction?"
905 illuminationCorrectionDataProductName = pexConfig.Field(
907 doc=
"Name of the illumination correction data product.",
910 illumScale = pexConfig.Field(
912 doc=
"Scale factor for the illumination correction.",
915 illumFilters = pexConfig.ListField(
918 doc=
"Only perform illumination correction for these filters."
923 doWrite = pexConfig.Field(
925 doc=
"Persist postISRCCD?",
932 raise ValueError(
"You may not specify both doFlat and doApplyGains")
934 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
943 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
944 """Apply common instrument signature correction algorithms to a raw frame.
946 The process for correcting imaging data is very similar from
947 camera to camera. This task provides a vanilla implementation of
948 doing these corrections, including the ability to turn certain
949 corrections off if they are not needed. The inputs to the primary
950 method, `run()`, are a raw exposure to be corrected and the
951 calibration data products. The raw input is a single chip sized
952 mosaic of all amps including overscans and other non-science
953 pixels. The method `runDataRef()` identifies and defines the
954 calibration data products, and is intended for use by a
955 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
956 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
957 subclassed for different camera, although the most camera specific
958 methods have been split into subtasks that can be redirected
961 The __init__ method sets up the subtasks for ISR processing, using
962 the defaults from `lsst.ip.isr`.
967 Positional arguments passed to the Task constructor.
968 None used at this time.
969 kwargs : `dict`, optional
970 Keyword arguments passed on to the Task constructor.
971 None used at this time.
973 ConfigClass = IsrTaskConfig
978 self.makeSubtask(
"assembleCcd")
979 self.makeSubtask(
"crosstalk")
980 self.makeSubtask(
"strayLight")
981 self.makeSubtask(
"fringe")
982 self.makeSubtask(
"masking")
983 self.makeSubtask(
"overscan")
984 self.makeSubtask(
"vignette")
985 self.makeSubtask(
"ampOffset")
988 inputs = butlerQC.get(inputRefs)
991 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
992 except Exception
as e:
993 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
996 inputs[
'isGen3'] =
True
998 detector = inputs[
'ccdExposure'].getDetector()
1000 if self.config.doCrosstalk
is True:
1003 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
1004 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
1005 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
1007 coeffVector = (self.config.crosstalk.crosstalkValues
1008 if self.config.crosstalk.useConfigCoefficients
else None)
1009 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1010 inputs[
'crosstalk'] = crosstalkCalib
1011 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
1012 if 'crosstalkSources' not in inputs:
1013 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
1016 if 'linearizer' in inputs:
1017 if isinstance(inputs[
'linearizer'], dict):
1019 linearizer.fromYaml(inputs[
'linearizer'])
1020 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1021 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1025 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1027 linearizer = inputs[
'linearizer']
1028 linearizer.log = self.log
1029 inputs[
'linearizer'] = linearizer
1032 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1034 if self.config.doDefect
is True:
1035 if "defects" in inputs
and inputs[
'defects']
is not None:
1039 if not isinstance(inputs[
"defects"], Defects):
1040 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1044 if self.config.doBrighterFatter:
1045 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1046 if brighterFatterKernel
is None:
1047 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1049 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1051 detName = detector.getName()
1052 level = brighterFatterKernel.level
1055 inputs[
'bfGains'] = brighterFatterKernel.gain
1056 if self.config.brighterFatterLevel ==
'DETECTOR':
1057 if level ==
'DETECTOR':
1058 if detName
in brighterFatterKernel.detKernels:
1059 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1061 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1062 elif level ==
'AMP':
1063 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1065 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1066 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1067 elif self.config.brighterFatterLevel ==
'AMP':
1068 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1070 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1071 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
1072 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1074 assembler=self.assembleCcd
1075 if self.config.doAssembleIsrExposures
else None)
1077 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1079 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1080 if 'strayLightData' not in inputs:
1081 inputs[
'strayLightData'] =
None
1083 outputs = self.
runrun(**inputs)
1084 butlerQC.put(outputs, outputRefs)
1087 """Retrieve necessary frames for instrument signature removal.
1089 Pre-fetching all required ISR data products limits the IO
1090 required by the ISR. Any conflict between the calibration data
1091 available and that needed for ISR is also detected prior to
1092 doing processing, allowing it to fail quickly.
1096 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1097 Butler reference of the detector data to be processed
1098 rawExposure : `afw.image.Exposure`
1099 The raw exposure that will later be corrected with the
1100 retrieved calibration data; should not be modified in this
1105 result : `lsst.pipe.base.Struct`
1106 Result struct with components (which may be `None`):
1107 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1108 - ``linearizer``: functor for linearization
1109 (`ip.isr.linearize.LinearizeBase`)
1110 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1111 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1112 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1113 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1114 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1115 - ``fringes``: `lsst.pipe.base.Struct` with components:
1116 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1117 - ``seed``: random seed derived from the ccdExposureId for random
1118 number generator (`uint32`).
1119 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1120 A ``TransmissionCurve`` that represents the throughput of the
1121 optics, to be evaluated in focal-plane coordinates.
1122 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1123 A ``TransmissionCurve`` that represents the throughput of the
1124 filter itself, to be evaluated in focal-plane coordinates.
1125 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1126 A ``TransmissionCurve`` that represents the throughput of the
1127 sensor itself, to be evaluated in post-assembly trimmed
1128 detector coordinates.
1129 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1130 A ``TransmissionCurve`` that represents the throughput of the
1131 atmosphere, assumed to be spatially constant.
1132 - ``strayLightData`` : `object`
1133 An opaque object containing calibration information for
1134 stray-light correction. If `None`, no correction will be
1136 - ``illumMaskedImage`` : illumination correction image
1137 (`lsst.afw.image.MaskedImage`)
1141 NotImplementedError :
1142 Raised if a per-amplifier brighter-fatter kernel is requested by
1146 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1147 dateObs = dateObs.toPython().isoformat()
1148 except RuntimeError:
1149 self.log.warning(
"Unable to identify dateObs for rawExposure.")
1152 ccd = rawExposure.getDetector()
1153 filterLabel = rawExposure.getFilterLabel()
1154 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1155 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1156 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1157 if self.config.doBias
else None)
1160 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1162 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1163 linearizer.log = self.log
1164 if isinstance(linearizer, numpy.ndarray):
1167 crosstalkCalib =
None
1168 if self.config.doCrosstalk:
1170 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1172 coeffVector = (self.config.crosstalk.crosstalkValues
1173 if self.config.crosstalk.useConfigCoefficients
else None)
1174 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1175 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1176 if self.config.doCrosstalk
else None)
1178 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1179 if self.config.doDark
else None)
1180 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1182 if self.config.doFlat
else None)
1184 brighterFatterKernel =
None
1185 brighterFatterGains =
None
1186 if self.config.doBrighterFatter
is True:
1191 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1192 brighterFatterGains = brighterFatterKernel.gain
1193 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1196 brighterFatterKernel = dataRef.get(
"bfKernel")
1197 self.log.info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1199 brighterFatterKernel =
None
1200 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1203 if self.config.brighterFatterLevel ==
'DETECTOR':
1204 if brighterFatterKernel.detKernels:
1205 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1207 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1210 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1212 defectList = (dataRef.get(
"defects")
1213 if self.config.doDefect
else None)
1214 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1215 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1216 if self.config.doAssembleIsrExposures
else None)
1217 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1218 else pipeBase.Struct(fringes=
None))
1220 if self.config.doAttachTransmissionCurve:
1221 opticsTransmission = (dataRef.get(
"transmission_optics")
1222 if self.config.doUseOpticsTransmission
else None)
1223 filterTransmission = (dataRef.get(
"transmission_filter")
1224 if self.config.doUseFilterTransmission
else None)
1225 sensorTransmission = (dataRef.get(
"transmission_sensor")
1226 if self.config.doUseSensorTransmission
else None)
1227 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1228 if self.config.doUseAtmosphereTransmission
else None)
1230 opticsTransmission =
None
1231 filterTransmission =
None
1232 sensorTransmission =
None
1233 atmosphereTransmission =
None
1235 if self.config.doStrayLight:
1236 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1238 strayLightData =
None
1241 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1242 if (self.config.doIlluminationCorrection
1243 and physicalFilter
in self.config.illumFilters)
1247 return pipeBase.Struct(bias=biasExposure,
1248 linearizer=linearizer,
1249 crosstalk=crosstalkCalib,
1250 crosstalkSources=crosstalkSources,
1253 bfKernel=brighterFatterKernel,
1254 bfGains=brighterFatterGains,
1256 fringes=fringeStruct,
1257 opticsTransmission=opticsTransmission,
1258 filterTransmission=filterTransmission,
1259 sensorTransmission=sensorTransmission,
1260 atmosphereTransmission=atmosphereTransmission,
1261 strayLightData=strayLightData,
1262 illumMaskedImage=illumMaskedImage
1265 @pipeBase.timeMethod
1266 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1267 crosstalk=None, crosstalkSources=None,
1268 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1269 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1270 sensorTransmission=
None, atmosphereTransmission=
None,
1271 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1274 """Perform instrument signature removal on an exposure.
1276 Steps included in the ISR processing, in order performed, are:
1277 - saturation and suspect pixel masking
1278 - overscan subtraction
1279 - CCD assembly of individual amplifiers
1281 - variance image construction
1282 - linearization of non-linear response
1284 - brighter-fatter correction
1287 - stray light subtraction
1289 - masking of known defects and camera specific features
1290 - vignette calculation
1291 - appending transmission curve and distortion model
1295 ccdExposure : `lsst.afw.image.Exposure`
1296 The raw exposure that is to be run through ISR. The
1297 exposure is modified by this method.
1298 camera : `lsst.afw.cameraGeom.Camera`, optional
1299 The camera geometry for this exposure. Required if
1300 one or more of ``ccdExposure``, ``bias``, ``dark``, or
1301 ``flat`` does not have an associated detector.
1302 bias : `lsst.afw.image.Exposure`, optional
1303 Bias calibration frame.
1304 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1305 Functor for linearization.
1306 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1307 Calibration for crosstalk.
1308 crosstalkSources : `list`, optional
1309 List of possible crosstalk sources.
1310 dark : `lsst.afw.image.Exposure`, optional
1311 Dark calibration frame.
1312 flat : `lsst.afw.image.Exposure`, optional
1313 Flat calibration frame.
1314 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1315 Photon transfer curve dataset, with, e.g., gains
1317 bfKernel : `numpy.ndarray`, optional
1318 Brighter-fatter kernel.
1319 bfGains : `dict` of `float`, optional
1320 Gains used to override the detector's nominal gains for the
1321 brighter-fatter correction. A dict keyed by amplifier name for
1322 the detector in question.
1323 defects : `lsst.ip.isr.Defects`, optional
1325 fringes : `lsst.pipe.base.Struct`, optional
1326 Struct containing the fringe correction data, with
1328 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1329 - ``seed``: random seed derived from the ccdExposureId for random
1330 number generator (`uint32`)
1331 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1332 A ``TransmissionCurve`` that represents the throughput of the,
1333 optics, to be evaluated in focal-plane coordinates.
1334 filterTransmission : `lsst.afw.image.TransmissionCurve`
1335 A ``TransmissionCurve`` that represents the throughput of the
1336 filter itself, to be evaluated in focal-plane coordinates.
1337 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1338 A ``TransmissionCurve`` that represents the throughput of the
1339 sensor itself, to be evaluated in post-assembly trimmed detector
1341 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1342 A ``TransmissionCurve`` that represents the throughput of the
1343 atmosphere, assumed to be spatially constant.
1344 detectorNum : `int`, optional
1345 The integer number for the detector to process.
1346 isGen3 : bool, optional
1347 Flag this call to run() as using the Gen3 butler environment.
1348 strayLightData : `object`, optional
1349 Opaque object containing calibration information for stray-light
1350 correction. If `None`, no correction will be performed.
1351 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1352 Illumination correction image.
1356 result : `lsst.pipe.base.Struct`
1357 Result struct with component:
1358 - ``exposure`` : `afw.image.Exposure`
1359 The fully ISR corrected exposure.
1360 - ``outputExposure`` : `afw.image.Exposure`
1361 An alias for `exposure`
1362 - ``ossThumb`` : `numpy.ndarray`
1363 Thumbnail image of the exposure after overscan subtraction.
1364 - ``flattenedThumb`` : `numpy.ndarray`
1365 Thumbnail image of the exposure after flat-field correction.
1370 Raised if a configuration option is set to True, but the
1371 required calibration data has not been specified.
1375 The current processed exposure can be viewed by setting the
1376 appropriate lsstDebug entries in the `debug.display`
1377 dictionary. The names of these entries correspond to some of
1378 the IsrTaskConfig Boolean options, with the value denoting the
1379 frame to use. The exposure is shown inside the matching
1380 option check and after the processing of that step has
1381 finished. The steps with debug points are:
1392 In addition, setting the "postISRCCD" entry displays the
1393 exposure after all ISR processing has finished.
1402 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1403 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1404 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1405 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1407 if isinstance(ccdExposure, ButlerDataRef):
1408 return self.
runDataRefrunDataRef(ccdExposure)
1410 ccd = ccdExposure.getDetector()
1411 filterLabel = ccdExposure.getFilterLabel()
1412 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1415 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1416 ccd = [
FakeAmp(ccdExposure, self.config)]
1419 if self.config.doBias
and bias
is None:
1420 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1421 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1422 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1423 if self.config.doBrighterFatter
and bfKernel
is None:
1424 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1425 if self.config.doDark
and dark
is None:
1426 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1427 if self.config.doFlat
and flat
is None:
1428 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1429 if self.config.doDefect
and defects
is None:
1430 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1431 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1432 and fringes.fringes
is None):
1437 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1438 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1439 and illumMaskedImage
is None):
1440 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1443 if self.config.doConvertIntToFloat:
1444 self.log.info(
"Converting exposure to floating point values.")
1447 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1448 self.log.info(
"Applying bias correction.")
1449 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1450 trimToFit=self.config.doTrimToMatchCalib)
1451 self.
debugViewdebugView(ccdExposure,
"doBias")
1458 if ccdExposure.getBBox().contains(amp.getBBox()):
1461 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1463 if self.config.doOverscan
and not badAmp:
1466 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1467 if overscanResults
is not None and \
1468 self.config.qa
is not None and self.config.qa.saveStats
is True:
1469 if isinstance(overscanResults.overscanFit, float):
1470 qaMedian = overscanResults.overscanFit
1471 qaStdev = float(
"NaN")
1473 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1474 afwMath.MEDIAN | afwMath.STDEVCLIP)
1475 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1476 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1478 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1479 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1480 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1481 amp.getName(), qaMedian, qaStdev)
1484 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1485 afwMath.MEDIAN | afwMath.STDEVCLIP)
1486 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1487 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1489 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1490 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1491 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1492 amp.getName(), qaMedianAfter, qaStdevAfter)
1494 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1497 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1498 overscanResults =
None
1500 overscans.append(overscanResults
if overscanResults
is not None else None)
1502 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1504 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1505 self.log.info(
"Applying crosstalk correction.")
1506 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1507 crosstalkSources=crosstalkSources, camera=camera)
1508 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1510 if self.config.doAssembleCcd:
1511 self.log.info(
"Assembling CCD from amplifiers.")
1512 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1514 if self.config.expectWcs
and not ccdExposure.getWcs():
1515 self.log.warning(
"No WCS found in input exposure.")
1516 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1519 if self.config.qa.doThumbnailOss:
1520 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1522 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1523 self.log.info(
"Applying bias correction.")
1524 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1525 trimToFit=self.config.doTrimToMatchCalib)
1526 self.
debugViewdebugView(ccdExposure,
"doBias")
1528 if self.config.doVariance:
1529 for amp, overscanResults
in zip(ccd, overscans):
1530 if ccdExposure.getBBox().contains(amp.getBBox()):
1531 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1532 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1533 if overscanResults
is not None:
1535 overscanImage=overscanResults.overscanImage,
1541 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1542 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1543 afwMath.MEDIAN | afwMath.STDEVCLIP)
1544 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1545 qaStats.getValue(afwMath.MEDIAN))
1546 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1547 qaStats.getValue(afwMath.STDEVCLIP))
1548 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1549 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1550 qaStats.getValue(afwMath.STDEVCLIP))
1551 if self.config.maskNegativeVariance:
1555 self.log.info(
"Applying linearizer.")
1556 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1557 detector=ccd, log=self.log)
1559 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1560 self.log.info(
"Applying crosstalk correction.")
1561 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1562 crosstalkSources=crosstalkSources, isTrimmed=
True)
1563 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1568 if self.config.doDefect:
1569 self.log.info(
"Masking defects.")
1570 self.
maskDefectmaskDefect(ccdExposure, defects)
1572 if self.config.numEdgeSuspect > 0:
1573 self.log.info(
"Masking edges as SUSPECT.")
1574 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1575 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1577 if self.config.doNanMasking:
1578 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1579 self.
maskNanmaskNan(ccdExposure)
1581 if self.config.doWidenSaturationTrails:
1582 self.log.info(
"Widening saturation trails.")
1583 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1585 if self.config.doCameraSpecificMasking:
1586 self.log.info(
"Masking regions for camera specific reasons.")
1587 self.masking.
run(ccdExposure)
1589 if self.config.doBrighterFatter:
1599 interpExp = ccdExposure.clone()
1600 with self.
flatContextflatContext(interpExp, flat, dark):
1601 isrFunctions.interpolateFromMask(
1602 maskedImage=interpExp.getMaskedImage(),
1603 fwhm=self.config.fwhm,
1604 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1605 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1607 bfExp = interpExp.clone()
1609 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1610 type(bfKernel), type(bfGains))
1611 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1612 self.config.brighterFatterMaxIter,
1613 self.config.brighterFatterThreshold,
1614 self.config.brighterFatterApplyGain,
1616 if bfResults[1] == self.config.brighterFatterMaxIter:
1617 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1620 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1622 image = ccdExposure.getMaskedImage().getImage()
1623 bfCorr = bfExp.getMaskedImage().getImage()
1624 bfCorr -= interpExp.getMaskedImage().getImage()
1633 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1634 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1637 if self.config.brighterFatterMaskGrowSize > 0:
1638 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1639 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1640 isrFunctions.growMasks(ccdExposure.getMask(),
1641 radius=self.config.brighterFatterMaskGrowSize,
1642 maskNameList=maskPlane,
1643 maskValue=maskPlane)
1645 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1647 if self.config.doDark:
1648 self.log.info(
"Applying dark correction.")
1650 self.
debugViewdebugView(ccdExposure,
"doDark")
1652 if self.config.doFringe
and not self.config.fringeAfterFlat:
1653 self.log.info(
"Applying fringe correction before flat.")
1654 self.fringe.
run(ccdExposure, **fringes.getDict())
1655 self.
debugViewdebugView(ccdExposure,
"doFringe")
1657 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1658 self.log.info(
"Checking strayLight correction.")
1659 self.strayLight.
run(ccdExposure, strayLightData)
1660 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1662 if self.config.doFlat:
1663 self.log.info(
"Applying flat correction.")
1665 self.
debugViewdebugView(ccdExposure,
"doFlat")
1667 if self.config.doApplyGains:
1668 self.log.info(
"Applying gain correction instead of flat.")
1669 if self.config.usePtcGains:
1670 self.log.info(
"Using gains from the Photon Transfer Curve.")
1671 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1674 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1676 if self.config.doFringe
and self.config.fringeAfterFlat:
1677 self.log.info(
"Applying fringe correction after flat.")
1678 self.fringe.
run(ccdExposure, **fringes.getDict())
1680 if self.config.doVignette:
1681 self.log.info(
"Constructing Vignette polygon.")
1684 if self.config.vignette.doWriteVignettePolygon:
1687 if self.config.doAttachTransmissionCurve:
1688 self.log.info(
"Adding transmission curves.")
1689 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1690 filterTransmission=filterTransmission,
1691 sensorTransmission=sensorTransmission,
1692 atmosphereTransmission=atmosphereTransmission)
1694 flattenedThumb =
None
1695 if self.config.qa.doThumbnailFlattened:
1696 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1698 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1699 self.log.info(
"Performing illumination correction.")
1700 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1701 illumMaskedImage, illumScale=self.config.illumScale,
1702 trimToFit=self.config.doTrimToMatchCalib)
1705 if self.config.doSaveInterpPixels:
1706 preInterpExp = ccdExposure.clone()
1721 if self.config.doSetBadRegions:
1722 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1723 if badPixelCount > 0:
1724 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1726 if self.config.doInterpolate:
1727 self.log.info(
"Interpolating masked pixels.")
1728 isrFunctions.interpolateFromMask(
1729 maskedImage=ccdExposure.getMaskedImage(),
1730 fwhm=self.config.fwhm,
1731 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1732 maskNameList=list(self.config.maskListToInterpolate)
1738 if self.config.doAmpOffset:
1739 self.log.info(
"Correcting amp offsets.")
1740 self.ampOffset.
run(ccdExposure)
1742 if self.config.doMeasureBackground:
1743 self.log.info(
"Measuring background level.")
1746 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1748 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1749 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1750 afwMath.MEDIAN | afwMath.STDEVCLIP)
1751 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1752 qaStats.getValue(afwMath.MEDIAN))
1753 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1754 qaStats.getValue(afwMath.STDEVCLIP))
1755 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1756 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1757 qaStats.getValue(afwMath.STDEVCLIP))
1759 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1761 return pipeBase.Struct(
1762 exposure=ccdExposure,
1764 flattenedThumb=flattenedThumb,
1766 preInterpExposure=preInterpExp,
1767 outputExposure=ccdExposure,
1768 outputOssThumbnail=ossThumb,
1769 outputFlattenedThumbnail=flattenedThumb,
1772 @pipeBase.timeMethod
1774 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1776 This method contains the `CmdLineTask` interface to the ISR
1777 processing. All IO is handled here, freeing the `run()` method
1778 to manage only pixel-level calculations. The steps performed
1780 - Read in necessary detrending/isr/calibration data.
1781 - Process raw exposure in `run()`.
1782 - Persist the ISR-corrected exposure as "postISRCCD" if
1783 config.doWrite=True.
1787 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1788 DataRef of the detector data to be processed
1792 result : `lsst.pipe.base.Struct`
1793 Result struct with component:
1794 - ``exposure`` : `afw.image.Exposure`
1795 The fully ISR corrected exposure.
1800 Raised if a configuration option is set to True, but the
1801 required calibration data does not exist.
1804 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1806 ccdExposure = sensorRef.get(self.config.datasetType)
1808 camera = sensorRef.get(
"camera")
1809 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1811 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1813 if self.config.doWrite:
1814 sensorRef.put(result.exposure,
"postISRCCD")
1815 if result.preInterpExposure
is not None:
1816 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1817 if result.ossThumb
is not None:
1818 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1819 if result.flattenedThumb
is not None:
1820 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1825 """Retrieve a calibration dataset for removing instrument signature.
1830 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1831 DataRef of the detector data to find calibration datasets
1834 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1835 dateObs : `str`, optional
1836 Date of the observation. Used to correct butler failures
1837 when using fallback filters.
1839 If True, disable butler proxies to enable error handling
1840 within this routine.
1844 exposure : `lsst.afw.image.Exposure`
1845 Requested calibration frame.
1850 Raised if no matching calibration frame can be found.
1853 exp = dataRef.get(datasetType, immediate=immediate)
1854 except Exception
as exc1:
1855 if not self.config.fallbackFilterName:
1856 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1858 if self.config.useFallbackDate
and dateObs:
1859 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1860 dateObs=dateObs, immediate=immediate)
1862 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1863 except Exception
as exc2:
1864 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1865 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1866 self.log.warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1868 if self.config.doAssembleIsrExposures:
1869 exp = self.assembleCcd.assembleCcd(exp)
1873 """Ensure that the data returned by Butler is a fully constructed exp.
1875 ISR requires exposure-level image data for historical reasons, so if we
1876 did not recieve that from Butler, construct it from what we have,
1877 modifying the input in place.
1881 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1882 or `lsst.afw.image.ImageF`
1883 The input data structure obtained from Butler.
1884 camera : `lsst.afw.cameraGeom.camera`, optional
1885 The camera associated with the image. Used to find the appropriate
1886 detector if detector is not already set.
1887 detectorNum : `int`, optional
1888 The detector in the camera to attach, if the detector is not
1893 inputExp : `lsst.afw.image.Exposure`
1894 The re-constructed exposure, with appropriate detector parameters.
1899 Raised if the input data cannot be used to construct an exposure.
1901 if isinstance(inputExp, afwImage.DecoratedImageU):
1902 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1903 elif isinstance(inputExp, afwImage.ImageF):
1904 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1905 elif isinstance(inputExp, afwImage.MaskedImageF):
1906 inputExp = afwImage.makeExposure(inputExp)
1907 elif isinstance(inputExp, afwImage.Exposure):
1909 elif inputExp
is None:
1913 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1916 if inputExp.getDetector()
is None:
1917 if camera
is None or detectorNum
is None:
1918 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1919 'without a detector set.')
1920 inputExp.setDetector(camera[detectorNum])
1925 """Convert exposure image from uint16 to float.
1927 If the exposure does not need to be converted, the input is
1928 immediately returned. For exposures that are converted to use
1929 floating point pixels, the variance is set to unity and the
1934 exposure : `lsst.afw.image.Exposure`
1935 The raw exposure to be converted.
1939 newexposure : `lsst.afw.image.Exposure`
1940 The input ``exposure``, converted to floating point pixels.
1945 Raised if the exposure type cannot be converted to float.
1948 if isinstance(exposure, afwImage.ExposureF):
1950 self.log.debug(
"Exposure already of type float.")
1952 if not hasattr(exposure,
"convertF"):
1953 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1955 newexposure = exposure.convertF()
1956 newexposure.variance[:] = 1
1957 newexposure.mask[:] = 0x0
1962 """Identify bad amplifiers, saturated and suspect pixels.
1966 ccdExposure : `lsst.afw.image.Exposure`
1967 Input exposure to be masked.
1968 amp : `lsst.afw.table.AmpInfoCatalog`
1969 Catalog of parameters defining the amplifier on this
1971 defects : `lsst.ip.isr.Defects`
1972 List of defects. Used to determine if the entire
1978 If this is true, the entire amplifier area is covered by
1979 defects and unusable.
1982 maskedImage = ccdExposure.getMaskedImage()
1989 if defects
is not None:
1990 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1996 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1998 maskView = dataView.getMask()
1999 maskView |= maskView.getPlaneBitMask(
"BAD")
2007 if self.config.doSaturation
and not badAmp:
2008 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2009 if self.config.doSuspect
and not badAmp:
2010 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2011 if math.isfinite(self.config.saturation):
2012 limits.update({self.config.saturatedMaskName: self.config.saturation})
2014 for maskName, maskThreshold
in limits.items():
2015 if not math.isnan(maskThreshold):
2016 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2017 isrFunctions.makeThresholdMask(
2018 maskedImage=dataView,
2019 threshold=maskThreshold,
2026 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2028 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2029 self.config.suspectMaskName])
2030 if numpy.all(maskView.getArray() & maskVal > 0):
2032 maskView |= maskView.getPlaneBitMask(
"BAD")
2037 """Apply overscan correction in place.
2039 This method does initial pixel rejection of the overscan
2040 region. The overscan can also be optionally segmented to
2041 allow for discontinuous overscan responses to be fit
2042 separately. The actual overscan subtraction is performed by
2043 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2044 which is called here after the amplifier is preprocessed.
2048 ccdExposure : `lsst.afw.image.Exposure`
2049 Exposure to have overscan correction performed.
2050 amp : `lsst.afw.cameraGeom.Amplifer`
2051 The amplifier to consider while correcting the overscan.
2055 overscanResults : `lsst.pipe.base.Struct`
2056 Result struct with components:
2057 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2058 Value or fit subtracted from the amplifier image data.
2059 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2060 Value or fit subtracted from the overscan image data.
2061 - ``overscanImage`` : `lsst.afw.image.Image`
2062 Image of the overscan region with the overscan
2063 correction applied. This quantity is used to estimate
2064 the amplifier read noise empirically.
2069 Raised if the ``amp`` does not contain raw pixel information.
2073 lsst.ip.isr.isrFunctions.overscanCorrection
2075 if amp.getRawHorizontalOverscanBBox().isEmpty():
2076 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2079 statControl = afwMath.StatisticsControl()
2080 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2083 dataBBox = amp.getRawDataBBox()
2084 oscanBBox = amp.getRawHorizontalOverscanBBox()
2088 prescanBBox = amp.getRawPrescanBBox()
2089 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2090 dx0 += self.config.overscanNumLeadingColumnsToSkip
2091 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2093 dx0 += self.config.overscanNumTrailingColumnsToSkip
2094 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2101 if ((self.config.overscanBiasJump
2102 and self.config.overscanBiasJumpLocation)
2103 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2104 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2105 self.config.overscanBiasJumpDevices)):
2106 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2107 yLower = self.config.overscanBiasJumpLocation
2108 yUpper = dataBBox.getHeight() - yLower
2110 yUpper = self.config.overscanBiasJumpLocation
2111 yLower = dataBBox.getHeight() - yUpper
2129 oscanBBox.getHeight())))
2133 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2134 ampImage = ccdExposure.maskedImage[imageBBox]
2135 overscanImage = ccdExposure.maskedImage[overscanBBox]
2137 overscanArray = overscanImage.image.array
2138 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2139 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2140 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2142 statControl = afwMath.StatisticsControl()
2143 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2145 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2148 levelStat = afwMath.MEDIAN
2149 sigmaStat = afwMath.STDEVCLIP
2151 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2152 self.config.qa.flatness.nIter)
2153 metadata = ccdExposure.getMetadata()
2154 ampNum = amp.getName()
2156 if isinstance(overscanResults.overscanFit, float):
2157 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2158 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2160 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2161 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2162 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2164 return overscanResults
2167 """Set the variance plane using the gain and read noise
2169 The read noise is calculated from the ``overscanImage`` if the
2170 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2171 the value from the amplifier data is used.
2175 ampExposure : `lsst.afw.image.Exposure`
2176 Exposure to process.
2177 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2178 Amplifier detector data.
2179 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2180 Image of overscan, required only for empirical read noise.
2181 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2182 PTC dataset containing the gains and read noise.
2188 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2189 are ``True``, but ptcDataset is not provided.
2191 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2192 ``overscanImage`` is ``None``.
2196 lsst.ip.isr.isrFunctions.updateVariance
2198 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2199 if self.config.usePtcGains:
2200 if ptcDataset
is None:
2201 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2203 gain = ptcDataset.gain[amp.getName()]
2204 self.log.info(
"Using gain from Photon Transfer Curve.")
2206 gain = amp.getGain()
2208 if math.isnan(gain):
2210 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2213 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2214 amp.getName(), gain, patchedGain)
2217 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2218 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2220 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2221 stats = afwMath.StatisticsControl()
2222 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2223 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2224 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2225 amp.getName(), readNoise)
2226 elif self.config.usePtcReadNoise:
2227 if ptcDataset
is None:
2228 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2230 readNoise = ptcDataset.noise[amp.getName()]
2231 self.log.info(
"Using read noise from Photon Transfer Curve.")
2233 readNoise = amp.getReadNoise()
2235 isrFunctions.updateVariance(
2236 maskedImage=ampExposure.getMaskedImage(),
2238 readNoise=readNoise,
2242 """Identify and mask pixels with negative variance values.
2246 exposure : `lsst.afw.image.Exposure`
2247 Exposure to process.
2251 lsst.ip.isr.isrFunctions.updateVariance
2253 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2254 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2255 exposure.mask.array[bad] |= maskPlane
2258 """Apply dark correction in place.
2262 exposure : `lsst.afw.image.Exposure`
2263 Exposure to process.
2264 darkExposure : `lsst.afw.image.Exposure`
2265 Dark exposure of the same size as ``exposure``.
2266 invert : `Bool`, optional
2267 If True, re-add the dark to an already corrected image.
2272 Raised if either ``exposure`` or ``darkExposure`` do not
2273 have their dark time defined.
2277 lsst.ip.isr.isrFunctions.darkCorrection
2279 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2280 if math.isnan(expScale):
2281 raise RuntimeError(
"Exposure darktime is NAN.")
2282 if darkExposure.getInfo().getVisitInfo()
is not None \
2283 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2284 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2288 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2291 isrFunctions.darkCorrection(
2292 maskedImage=exposure.getMaskedImage(),
2293 darkMaskedImage=darkExposure.getMaskedImage(),
2295 darkScale=darkScale,
2297 trimToFit=self.config.doTrimToMatchCalib
2301 """Check if linearization is needed for the detector cameraGeom.
2303 Checks config.doLinearize and the linearity type of the first
2308 detector : `lsst.afw.cameraGeom.Detector`
2309 Detector to get linearity type from.
2313 doLinearize : `Bool`
2314 If True, linearization should be performed.
2316 return self.config.doLinearize
and \
2317 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2320 """Apply flat correction in place.
2324 exposure : `lsst.afw.image.Exposure`
2325 Exposure to process.
2326 flatExposure : `lsst.afw.image.Exposure`
2327 Flat exposure of the same size as ``exposure``.
2328 invert : `Bool`, optional
2329 If True, unflatten an already flattened image.
2333 lsst.ip.isr.isrFunctions.flatCorrection
2335 isrFunctions.flatCorrection(
2336 maskedImage=exposure.getMaskedImage(),
2337 flatMaskedImage=flatExposure.getMaskedImage(),
2338 scalingType=self.config.flatScalingType,
2339 userScale=self.config.flatUserScale,
2341 trimToFit=self.config.doTrimToMatchCalib
2345 """Detect and mask saturated pixels in config.saturatedMaskName.
2349 exposure : `lsst.afw.image.Exposure`
2350 Exposure to process. Only the amplifier DataSec is processed.
2351 amp : `lsst.afw.table.AmpInfoCatalog`
2352 Amplifier detector data.
2356 lsst.ip.isr.isrFunctions.makeThresholdMask
2358 if not math.isnan(amp.getSaturation()):
2359 maskedImage = exposure.getMaskedImage()
2360 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2361 isrFunctions.makeThresholdMask(
2362 maskedImage=dataView,
2363 threshold=amp.getSaturation(),
2365 maskName=self.config.saturatedMaskName,
2369 """Interpolate over saturated pixels, in place.
2371 This method should be called after `saturationDetection`, to
2372 ensure that the saturated pixels have been identified in the
2373 SAT mask. It should also be called after `assembleCcd`, since
2374 saturated regions may cross amplifier boundaries.
2378 exposure : `lsst.afw.image.Exposure`
2379 Exposure to process.
2383 lsst.ip.isr.isrTask.saturationDetection
2384 lsst.ip.isr.isrFunctions.interpolateFromMask
2386 isrFunctions.interpolateFromMask(
2387 maskedImage=exposure.getMaskedImage(),
2388 fwhm=self.config.fwhm,
2389 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2390 maskNameList=list(self.config.saturatedMaskName),
2394 """Detect and mask suspect pixels in config.suspectMaskName.
2398 exposure : `lsst.afw.image.Exposure`
2399 Exposure to process. Only the amplifier DataSec is processed.
2400 amp : `lsst.afw.table.AmpInfoCatalog`
2401 Amplifier detector data.
2405 lsst.ip.isr.isrFunctions.makeThresholdMask
2409 Suspect pixels are pixels whose value is greater than
2410 amp.getSuspectLevel(). This is intended to indicate pixels that may be
2411 affected by unknown systematics; for example if non-linearity
2412 corrections above a certain level are unstable then that would be a
2413 useful value for suspectLevel. A value of `nan` indicates that no such
2414 level exists and no pixels are to be masked as suspicious.
2416 suspectLevel = amp.getSuspectLevel()
2417 if math.isnan(suspectLevel):
2420 maskedImage = exposure.getMaskedImage()
2421 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2422 isrFunctions.makeThresholdMask(
2423 maskedImage=dataView,
2424 threshold=suspectLevel,
2426 maskName=self.config.suspectMaskName,
2430 """Mask defects using mask plane "BAD", in place.
2434 exposure : `lsst.afw.image.Exposure`
2435 Exposure to process.
2436 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2437 `lsst.afw.image.DefectBase`.
2438 List of defects to mask.
2442 Call this after CCD assembly, since defects may cross amplifier
2445 maskedImage = exposure.getMaskedImage()
2446 if not isinstance(defectBaseList, Defects):
2448 defectList =
Defects(defectBaseList)
2450 defectList = defectBaseList
2451 defectList.maskPixels(maskedImage, maskName=
"BAD")
2453 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2454 """Mask edge pixels with applicable mask plane.
2458 exposure : `lsst.afw.image.Exposure`
2459 Exposure to process.
2460 numEdgePixels : `int`, optional
2461 Number of edge pixels to mask.
2462 maskPlane : `str`, optional
2463 Mask plane name to use.
2464 level : `str`, optional
2465 Level at which to mask edges.
2467 maskedImage = exposure.getMaskedImage()
2468 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2470 if numEdgePixels > 0:
2471 if level ==
'DETECTOR':
2472 boxes = [maskedImage.getBBox()]
2473 elif level ==
'AMP':
2474 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2479 subImage = maskedImage[box]
2480 box.grow(-numEdgePixels)
2482 SourceDetectionTask.setEdgeBits(
2488 """Mask and interpolate defects using mask plane "BAD", in place.
2492 exposure : `lsst.afw.image.Exposure`
2493 Exposure to process.
2494 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2495 `lsst.afw.image.DefectBase`.
2496 List of defects to mask and interpolate.
2500 lsst.ip.isr.isrTask.maskDefect
2502 self.
maskDefectmaskDefect(exposure, defectBaseList)
2503 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2504 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2505 isrFunctions.interpolateFromMask(
2506 maskedImage=exposure.getMaskedImage(),
2507 fwhm=self.config.fwhm,
2508 growSaturatedFootprints=0,
2509 maskNameList=[
"BAD"],
2513 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2517 exposure : `lsst.afw.image.Exposure`
2518 Exposure to process.
2522 We mask over all non-finite values (NaN, inf), including those
2523 that are masked with other bits (because those may or may not be
2524 interpolated over later, and we want to remove all NaN/infs).
2525 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2526 preserve the historical name.
2528 maskedImage = exposure.getMaskedImage()
2531 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2532 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2533 numNans =
maskNans(maskedImage, maskVal)
2534 self.metadata.set(
"NUMNANS", numNans)
2536 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2539 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2544 exposure : `lsst.afw.image.Exposure`
2545 Exposure to process.
2549 lsst.ip.isr.isrTask.maskNan
2552 isrFunctions.interpolateFromMask(
2553 maskedImage=exposure.getMaskedImage(),
2554 fwhm=self.config.fwhm,
2555 growSaturatedFootprints=0,
2556 maskNameList=[
"UNMASKEDNAN"],
2560 """Measure the image background in subgrids, for quality control.
2564 exposure : `lsst.afw.image.Exposure`
2565 Exposure to process.
2566 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2567 Configuration object containing parameters on which background
2568 statistics and subgrids to use.
2570 if IsrQaConfig
is not None:
2571 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2572 IsrQaConfig.flatness.nIter)
2573 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2574 statsControl.setAndMask(maskVal)
2575 maskedImage = exposure.getMaskedImage()
2576 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2577 skyLevel = stats.getValue(afwMath.MEDIAN)
2578 skySigma = stats.getValue(afwMath.STDEVCLIP)
2579 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2580 metadata = exposure.getMetadata()
2581 metadata.set(
'SKYLEVEL', skyLevel)
2582 metadata.set(
'SKYSIGMA', skySigma)
2585 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2586 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2587 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2588 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2589 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2590 skyLevels = numpy.zeros((nX, nY))
2593 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2595 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2597 xLLC = xc - meshXHalf
2598 yLLC = yc - meshYHalf
2599 xURC = xc + meshXHalf - 1
2600 yURC = yc + meshYHalf - 1
2603 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2605 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2607 good = numpy.where(numpy.isfinite(skyLevels))
2608 skyMedian = numpy.median(skyLevels[good])
2609 flatness = (skyLevels[good] - skyMedian) / skyMedian
2610 flatness_rms = numpy.std(flatness)
2611 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2613 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2614 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2615 nX, nY, flatness_pp, flatness_rms)
2617 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2618 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2619 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2620 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2621 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2624 """Set an approximate magnitude zero point for the exposure.
2628 exposure : `lsst.afw.image.Exposure`
2629 Exposure to process.
2631 filterLabel = exposure.getFilterLabel()
2632 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2634 if physicalFilter
in self.config.fluxMag0T1:
2635 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2637 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2638 fluxMag0 = self.config.defaultFluxMag0T1
2640 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2642 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2645 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2646 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2647 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2650 """Set valid polygon as the intersection of fpPolygon and chip corners.
2654 ccdExposure : `lsst.afw.image.Exposure`
2655 Exposure to process.
2656 fpPolygon : `lsst.afw.geom.Polygon`
2657 Polygon in focal plane coordinates.
2660 ccd = ccdExposure.getDetector()
2661 fpCorners = ccd.getCorners(FOCAL_PLANE)
2662 ccdPolygon = Polygon(fpCorners)
2665 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2668 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2669 validPolygon = Polygon(ccdPoints)
2670 ccdExposure.getInfo().setValidPolygon(validPolygon)
2674 """Context manager that applies and removes flats and darks,
2675 if the task is configured to apply them.
2679 exp : `lsst.afw.image.Exposure`
2680 Exposure to process.
2681 flat : `lsst.afw.image.Exposure`
2682 Flat exposure the same size as ``exp``.
2683 dark : `lsst.afw.image.Exposure`, optional
2684 Dark exposure the same size as ``exp``.
2688 exp : `lsst.afw.image.Exposure`
2689 The flat and dark corrected exposure.
2691 if self.config.doDark
and dark
is not None:
2693 if self.config.doFlat:
2698 if self.config.doFlat:
2700 if self.config.doDark
and dark
is not None:
2704 """Utility function to examine ISR exposure at different stages.
2708 exposure : `lsst.afw.image.Exposure`
2711 State of processing to view.
2713 frame = getDebugFrame(self._display, stepname)
2715 display = getDisplay(frame)
2716 display.scale(
'asinh',
'zscale')
2717 display.mtv(exposure)
2718 prompt =
"Press Enter to continue [c]... "
2720 ans = input(prompt).lower()
2721 if ans
in (
"",
"c",):
2726 """A Detector-like object that supports returning gain and saturation level
2728 This is used when the input exposure does not have a detector.
2732 exposure : `lsst.afw.image.Exposure`
2733 Exposure to generate a fake amplifier for.
2734 config : `lsst.ip.isr.isrTaskConfig`
2735 Configuration to apply to the fake amplifier.
2739 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2741 self.
_gain_gain = config.gain
2746 return self.
_bbox_bbox
2749 return self.
_bbox_bbox
2755 return self.
_gain_gain
2768 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2772 """Task to wrap the default IsrTask to allow it to be retargeted.
2774 The standard IsrTask can be called directly from a command line
2775 program, but doing so removes the ability of the task to be
2776 retargeted. As most cameras override some set of the IsrTask
2777 methods, this would remove those data-specific methods in the
2778 output post-ISR images. This wrapping class fixes the issue,
2779 allowing identical post-ISR images to be generated by both the
2780 processCcd and isrTask code.
2782 ConfigClass = RunIsrConfig
2783 _DefaultName =
"runIsr"
2787 self.makeSubtask(
"isr")
2793 dataRef : `lsst.daf.persistence.ButlerDataRef`
2794 data reference of the detector data to be processed
2798 result : `pipeBase.Struct`
2799 Result struct with component:
2801 - exposure : `lsst.afw.image.Exposure`
2802 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.