29 import lsst.pipe.base
as pipeBase
30 import lsst.pipe.base.connectionTypes
as cT
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
126 crosstalkSources = cT.PrerequisiteInput(
127 name=
"isrOverscanCorrected",
128 doc=
"Overscan corrected input images.",
129 storageClass=
"Exposure",
130 dimensions=[
"instrument",
"exposure",
"detector"],
133 lookupFunction=crosstalkSourceLookup,
136 bias = cT.PrerequisiteInput(
138 doc=
"Input bias calibration.",
139 storageClass=
"ExposureF",
140 dimensions=[
"instrument",
"detector"],
143 dark = cT.PrerequisiteInput(
145 doc=
"Input dark calibration.",
146 storageClass=
"ExposureF",
147 dimensions=[
"instrument",
"detector"],
150 flat = cT.PrerequisiteInput(
152 doc=
"Input flat calibration.",
153 storageClass=
"ExposureF",
154 dimensions=[
"instrument",
"physical_filter",
"detector"],
157 ptc = cT.PrerequisiteInput(
159 doc=
"Input Photon Transfer Curve dataset",
160 storageClass=
"PhotonTransferCurveDataset",
161 dimensions=[
"instrument",
"detector"],
164 fringes = cT.PrerequisiteInput(
166 doc=
"Input fringe calibration.",
167 storageClass=
"ExposureF",
168 dimensions=[
"instrument",
"physical_filter",
"detector"],
172 strayLightData = cT.PrerequisiteInput(
174 doc=
"Input stray light calibration.",
175 storageClass=
"StrayLightData",
176 dimensions=[
"instrument",
"physical_filter",
"detector"],
180 bfKernel = cT.PrerequisiteInput(
182 doc=
"Input brighter-fatter kernel.",
183 storageClass=
"NumpyArray",
184 dimensions=[
"instrument"],
188 newBFKernel = cT.PrerequisiteInput(
189 name=
'brighterFatterKernel',
190 doc=
"Newer complete kernel + gain solutions.",
191 storageClass=
"BrighterFatterKernel",
192 dimensions=[
"instrument",
"detector"],
196 defects = cT.PrerequisiteInput(
198 doc=
"Input defect tables.",
199 storageClass=
"Defects",
200 dimensions=[
"instrument",
"detector"],
203 linearizer = cT.PrerequisiteInput(
205 storageClass=
"Linearizer",
206 doc=
"Linearity correction calibration.",
207 dimensions=[
"instrument",
"detector"],
211 opticsTransmission = cT.PrerequisiteInput(
212 name=
"transmission_optics",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the optics.",
215 dimensions=[
"instrument"],
218 filterTransmission = cT.PrerequisiteInput(
219 name=
"transmission_filter",
220 storageClass=
"TransmissionCurve",
221 doc=
"Transmission curve due to the filter.",
222 dimensions=[
"instrument",
"physical_filter"],
225 sensorTransmission = cT.PrerequisiteInput(
226 name=
"transmission_sensor",
227 storageClass=
"TransmissionCurve",
228 doc=
"Transmission curve due to the sensor.",
229 dimensions=[
"instrument",
"detector"],
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name=
"transmission_atmosphere",
234 storageClass=
"TransmissionCurve",
235 doc=
"Transmission curve due to the atmosphere.",
236 dimensions=[
"instrument"],
239 illumMaskedImage = cT.PrerequisiteInput(
241 doc=
"Input illumination correction.",
242 storageClass=
"MaskedImageF",
243 dimensions=[
"instrument",
"physical_filter",
"detector"],
247 outputExposure = cT.Output(
249 doc=
"Output ISR processed exposure.",
250 storageClass=
"Exposure",
251 dimensions=[
"instrument",
"exposure",
"detector"],
253 preInterpExposure = cT.Output(
254 name=
'preInterpISRCCD',
255 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
256 storageClass=
"ExposureF",
257 dimensions=[
"instrument",
"exposure",
"detector"],
259 outputOssThumbnail = cT.Output(
261 doc=
"Output Overscan-subtracted thumbnail image.",
262 storageClass=
"Thumbnail",
263 dimensions=[
"instrument",
"exposure",
"detector"],
265 outputFlattenedThumbnail = cT.Output(
266 name=
"FlattenedThumb",
267 doc=
"Output flat-corrected thumbnail image.",
268 storageClass=
"Thumbnail",
269 dimensions=[
"instrument",
"exposure",
"detector"],
275 if config.doBias
is not True:
276 self.prerequisiteInputs.discard(
"bias")
277 if config.doLinearize
is not True:
278 self.prerequisiteInputs.discard(
"linearizer")
279 if config.doCrosstalk
is not True:
280 self.inputs.discard(
"crosstalkSources")
281 self.prerequisiteInputs.discard(
"crosstalk")
282 if config.doBrighterFatter
is not True:
283 self.prerequisiteInputs.discard(
"bfKernel")
284 self.prerequisiteInputs.discard(
"newBFKernel")
285 if config.doDefect
is not True:
286 self.prerequisiteInputs.discard(
"defects")
287 if config.doDark
is not True:
288 self.prerequisiteInputs.discard(
"dark")
289 if config.doFlat
is not True:
290 self.prerequisiteInputs.discard(
"flat")
291 if config.doFringe
is not True:
292 self.prerequisiteInputs.discard(
"fringe")
293 if config.doStrayLight
is not True:
294 self.prerequisiteInputs.discard(
"strayLightData")
295 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
296 self.prerequisiteInputs.discard(
"ptc")
297 if config.doAttachTransmissionCurve
is not True:
298 self.prerequisiteInputs.discard(
"opticsTransmission")
299 self.prerequisiteInputs.discard(
"filterTransmission")
300 self.prerequisiteInputs.discard(
"sensorTransmission")
301 self.prerequisiteInputs.discard(
"atmosphereTransmission")
302 if config.doUseOpticsTransmission
is not True:
303 self.prerequisiteInputs.discard(
"opticsTransmission")
304 if config.doUseFilterTransmission
is not True:
305 self.prerequisiteInputs.discard(
"filterTransmission")
306 if config.doUseSensorTransmission
is not True:
307 self.prerequisiteInputs.discard(
"sensorTransmission")
308 if config.doUseAtmosphereTransmission
is not True:
309 self.prerequisiteInputs.discard(
"atmosphereTransmission")
310 if config.doIlluminationCorrection
is not True:
311 self.prerequisiteInputs.discard(
"illumMaskedImage")
313 if config.doWrite
is not True:
314 self.outputs.discard(
"outputExposure")
315 self.outputs.discard(
"preInterpExposure")
316 self.outputs.discard(
"outputFlattenedThumbnail")
317 self.outputs.discard(
"outputOssThumbnail")
318 if config.doSaveInterpPixels
is not True:
319 self.outputs.discard(
"preInterpExposure")
320 if config.qa.doThumbnailOss
is not True:
321 self.outputs.discard(
"outputOssThumbnail")
322 if config.qa.doThumbnailFlattened
is not True:
323 self.outputs.discard(
"outputFlattenedThumbnail")
327 pipelineConnections=IsrTaskConnections):
328 """Configuration parameters for IsrTask.
330 Items are grouped in the order in which they are executed by the task.
332 datasetType = pexConfig.Field(
334 doc=
"Dataset type for input data; users will typically leave this alone, "
335 "but camera-specific ISR tasks will override it",
339 fallbackFilterName = pexConfig.Field(
341 doc=
"Fallback default filter name for calibrations.",
344 useFallbackDate = pexConfig.Field(
346 doc=
"Pass observation date when using fallback filter.",
349 expectWcs = pexConfig.Field(
352 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
354 fwhm = pexConfig.Field(
356 doc=
"FWHM of PSF in arcseconds.",
359 qa = pexConfig.ConfigField(
361 doc=
"QA related configuration options.",
365 doConvertIntToFloat = pexConfig.Field(
367 doc=
"Convert integer raw images to floating point values?",
372 doSaturation = pexConfig.Field(
374 doc=
"Mask saturated pixels? NB: this is totally independent of the"
375 " interpolation option - this is ONLY setting the bits in the mask."
376 " To have them interpolated make sure doSaturationInterpolation=True",
379 saturatedMaskName = pexConfig.Field(
381 doc=
"Name of mask plane to use in saturation detection and interpolation",
384 saturation = pexConfig.Field(
386 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387 default=float(
"NaN"),
389 growSaturationFootprintSize = pexConfig.Field(
391 doc=
"Number of pixels by which to grow the saturation footprints",
396 doSuspect = pexConfig.Field(
398 doc=
"Mask suspect pixels?",
401 suspectMaskName = pexConfig.Field(
403 doc=
"Name of mask plane to use for suspect pixels",
406 numEdgeSuspect = pexConfig.Field(
408 doc=
"Number of edge pixels to be flagged as untrustworthy.",
411 edgeMaskLevel = pexConfig.ChoiceField(
413 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
416 'DETECTOR':
'Mask only the edges of the full detector.',
417 'AMP':
'Mask edges of each amplifier.',
422 doSetBadRegions = pexConfig.Field(
424 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
427 badStatistic = pexConfig.ChoiceField(
429 doc=
"How to estimate the average value for BAD regions.",
432 "MEANCLIP":
"Correct using the (clipped) mean of good data",
433 "MEDIAN":
"Correct using the median of the good data",
438 doOverscan = pexConfig.Field(
440 doc=
"Do overscan subtraction?",
443 overscan = pexConfig.ConfigurableField(
444 target=OverscanCorrectionTask,
445 doc=
"Overscan subtraction task for image segments.",
448 overscanFitType = pexConfig.ChoiceField(
450 doc=
"The method for fitting the overscan bias level.",
453 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
454 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
455 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
456 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
457 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
458 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
459 "MEAN":
"Correct using the mean of the overscan region",
460 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
461 "MEDIAN":
"Correct using the median of the overscan region",
462 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
464 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
465 " This option will no longer be used, and will be removed after v20.")
467 overscanOrder = pexConfig.Field(
469 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
470 "or number of spline knots if overscan fit type is a spline."),
472 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
473 " This option will no longer be used, and will be removed after v20.")
475 overscanNumSigmaClip = pexConfig.Field(
477 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
479 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
480 " This option will no longer be used, and will be removed after v20.")
482 overscanIsInt = pexConfig.Field(
484 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485 " and overscan.FitType=MEDIAN_PER_ROW.",
487 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
488 " This option will no longer be used, and will be removed after v20.")
491 overscanNumLeadingColumnsToSkip = pexConfig.Field(
493 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
496 overscanNumTrailingColumnsToSkip = pexConfig.Field(
498 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
501 overscanMaxDev = pexConfig.Field(
503 doc=
"Maximum deviation from the median for overscan",
504 default=1000.0, check=
lambda x: x > 0
506 overscanBiasJump = pexConfig.Field(
508 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
511 overscanBiasJumpKeyword = pexConfig.Field(
513 doc=
"Header keyword containing information about devices.",
514 default=
"NO_SUCH_KEY",
516 overscanBiasJumpDevices = pexConfig.ListField(
518 doc=
"List of devices that need piecewise overscan correction.",
521 overscanBiasJumpLocation = pexConfig.Field(
523 doc=
"Location of bias jump along y-axis.",
528 doAssembleCcd = pexConfig.Field(
531 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
533 assembleCcd = pexConfig.ConfigurableField(
534 target=AssembleCcdTask,
535 doc=
"CCD assembly task",
539 doAssembleIsrExposures = pexConfig.Field(
542 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
544 doTrimToMatchCalib = pexConfig.Field(
547 doc=
"Trim raw data to match calibration bounding boxes?"
551 doBias = pexConfig.Field(
553 doc=
"Apply bias frame correction?",
556 biasDataProductName = pexConfig.Field(
558 doc=
"Name of the bias data product",
561 doBiasBeforeOverscan = pexConfig.Field(
563 doc=
"Reverse order of overscan and bias correction.",
568 doVariance = pexConfig.Field(
570 doc=
"Calculate variance?",
573 gain = pexConfig.Field(
575 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576 default=float(
"NaN"),
578 readNoise = pexConfig.Field(
580 doc=
"The read noise to use if no Detector is present in the Exposure",
583 doEmpiricalReadNoise = pexConfig.Field(
586 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
588 usePtcReadNoise = pexConfig.Field(
591 doc=
"Use readnoise values from the Photon Transfer Curve?"
593 maskNegativeVariance = pexConfig.Field(
596 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
597 "in the measurement of the overscan at an edge due to the data falling off faster "
598 "than the overscan model can account for it."
600 negativeVarianceMaskName = pexConfig.Field(
603 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
606 doLinearize = pexConfig.Field(
608 doc=
"Correct for nonlinearity of the detector's response?",
613 doCrosstalk = pexConfig.Field(
615 doc=
"Apply intra-CCD crosstalk correction?",
618 doCrosstalkBeforeAssemble = pexConfig.Field(
620 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
623 crosstalk = pexConfig.ConfigurableField(
624 target=CrosstalkTask,
625 doc=
"Intra-CCD crosstalk correction",
629 doDefect = pexConfig.Field(
631 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
634 doNanMasking = pexConfig.Field(
636 doc=
"Mask non-finite (NAN, inf) pixels?",
639 doWidenSaturationTrails = pexConfig.Field(
641 doc=
"Widen bleed trails based on their width?",
646 doBrighterFatter = pexConfig.Field(
649 doc=
"Apply the brighter-fatter correction?"
651 brighterFatterLevel = pexConfig.ChoiceField(
654 doc=
"The level at which to correct for brighter-fatter.",
656 "AMP":
"Every amplifier treated separately.",
657 "DETECTOR":
"One kernel per detector",
660 brighterFatterMaxIter = pexConfig.Field(
663 doc=
"Maximum number of iterations for the brighter-fatter correction"
665 brighterFatterThreshold = pexConfig.Field(
668 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
669 "absolute value of the difference between the current corrected image and the one "
670 "from the previous iteration summed over all the pixels."
672 brighterFatterApplyGain = pexConfig.Field(
675 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
677 brighterFatterMaskListToInterpolate = pexConfig.ListField(
679 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
681 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
683 brighterFatterMaskGrowSize = pexConfig.Field(
686 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
687 "when brighter-fatter correction is applied."
691 doDark = pexConfig.Field(
693 doc=
"Apply dark frame correction?",
696 darkDataProductName = pexConfig.Field(
698 doc=
"Name of the dark data product",
703 doStrayLight = pexConfig.Field(
705 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
708 strayLight = pexConfig.ConfigurableField(
709 target=StrayLightTask,
710 doc=
"y-band stray light correction"
714 doFlat = pexConfig.Field(
716 doc=
"Apply flat field correction?",
719 flatDataProductName = pexConfig.Field(
721 doc=
"Name of the flat data product",
724 flatScalingType = pexConfig.ChoiceField(
726 doc=
"The method for scaling the flat on the fly.",
729 "USER":
"Scale by flatUserScale",
730 "MEAN":
"Scale by the inverse of the mean",
731 "MEDIAN":
"Scale by the inverse of the median",
734 flatUserScale = pexConfig.Field(
736 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
739 doTweakFlat = pexConfig.Field(
741 doc=
"Tweak flats to match observed amplifier ratios?",
746 doApplyGains = pexConfig.Field(
748 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
751 usePtcGains = pexConfig.Field(
753 doc=
"Use the gain values from the Photon Transfer Curve?",
756 normalizeGains = pexConfig.Field(
758 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
763 doFringe = pexConfig.Field(
765 doc=
"Apply fringe correction?",
768 fringe = pexConfig.ConfigurableField(
770 doc=
"Fringe subtraction task",
772 fringeAfterFlat = pexConfig.Field(
774 doc=
"Do fringe subtraction after flat-fielding?",
779 doMeasureBackground = pexConfig.Field(
781 doc=
"Measure the background level on the reduced image?",
786 doCameraSpecificMasking = pexConfig.Field(
788 doc=
"Mask camera-specific bad regions?",
791 masking = pexConfig.ConfigurableField(
798 doInterpolate = pexConfig.Field(
800 doc=
"Interpolate masked pixels?",
803 doSaturationInterpolation = pexConfig.Field(
805 doc=
"Perform interpolation over pixels masked as saturated?"
806 " NB: This is independent of doSaturation; if that is False this plane"
807 " will likely be blank, resulting in a no-op here.",
810 doNanInterpolation = pexConfig.Field(
812 doc=
"Perform interpolation over pixels masked as NaN?"
813 " NB: This is independent of doNanMasking; if that is False this plane"
814 " will likely be blank, resulting in a no-op here.",
817 doNanInterpAfterFlat = pexConfig.Field(
819 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
820 "also have to interpolate them before flat-fielding."),
823 maskListToInterpolate = pexConfig.ListField(
825 doc=
"List of mask planes that should be interpolated.",
826 default=[
'SAT',
'BAD'],
828 doSaveInterpPixels = pexConfig.Field(
830 doc=
"Save a copy of the pre-interpolated pixel values?",
835 fluxMag0T1 = pexConfig.DictField(
838 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
839 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
842 defaultFluxMag0T1 = pexConfig.Field(
844 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
845 default=pow(10.0, 0.4*28.0)
849 doVignette = pexConfig.Field(
851 doc=
"Apply vignetting parameters?",
854 vignette = pexConfig.ConfigurableField(
856 doc=
"Vignetting task.",
860 doAttachTransmissionCurve = pexConfig.Field(
863 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
865 doUseOpticsTransmission = pexConfig.Field(
868 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
870 doUseFilterTransmission = pexConfig.Field(
873 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
875 doUseSensorTransmission = pexConfig.Field(
878 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
880 doUseAtmosphereTransmission = pexConfig.Field(
883 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
887 doIlluminationCorrection = pexConfig.Field(
890 doc=
"Perform illumination correction?"
892 illuminationCorrectionDataProductName = pexConfig.Field(
894 doc=
"Name of the illumination correction data product.",
897 illumScale = pexConfig.Field(
899 doc=
"Scale factor for the illumination correction.",
902 illumFilters = pexConfig.ListField(
905 doc=
"Only perform illumination correction for these filters."
909 doWrite = pexConfig.Field(
911 doc=
"Persist postISRCCD?",
918 raise ValueError(
"You may not specify both doFlat and doApplyGains")
920 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
929 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
930 """Apply common instrument signature correction algorithms to a raw frame.
932 The process for correcting imaging data is very similar from
933 camera to camera. This task provides a vanilla implementation of
934 doing these corrections, including the ability to turn certain
935 corrections off if they are not needed. The inputs to the primary
936 method, `run()`, are a raw exposure to be corrected and the
937 calibration data products. The raw input is a single chip sized
938 mosaic of all amps including overscans and other non-science
939 pixels. The method `runDataRef()` identifies and defines the
940 calibration data products, and is intended for use by a
941 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
942 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
943 subclassed for different camera, although the most camera specific
944 methods have been split into subtasks that can be redirected
947 The __init__ method sets up the subtasks for ISR processing, using
948 the defaults from `lsst.ip.isr`.
953 Positional arguments passed to the Task constructor. None used at this time.
954 kwargs : `dict`, optional
955 Keyword arguments passed on to the Task constructor. None used at this time.
957 ConfigClass = IsrTaskConfig
962 self.makeSubtask(
"assembleCcd")
963 self.makeSubtask(
"crosstalk")
964 self.makeSubtask(
"strayLight")
965 self.makeSubtask(
"fringe")
966 self.makeSubtask(
"masking")
967 self.makeSubtask(
"overscan")
968 self.makeSubtask(
"vignette")
971 inputs = butlerQC.get(inputRefs)
974 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
975 except Exception
as e:
976 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
979 inputs[
'isGen3'] =
True
981 detector = inputs[
'ccdExposure'].getDetector()
983 if self.config.doCrosstalk
is True:
986 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
987 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
988 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
990 coeffVector = (self.config.crosstalk.crosstalkValues
991 if self.config.crosstalk.useConfigCoefficients
else None)
992 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
993 inputs[
'crosstalk'] = crosstalkCalib
994 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
995 if 'crosstalkSources' not in inputs:
996 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
999 if 'linearizer' in inputs:
1000 if isinstance(inputs[
'linearizer'], dict):
1002 linearizer.fromYaml(inputs[
'linearizer'])
1003 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1004 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1008 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1010 linearizer = inputs[
'linearizer']
1011 linearizer.log = self.log
1012 inputs[
'linearizer'] = linearizer
1015 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1017 if self.config.doDefect
is True:
1018 if "defects" in inputs
and inputs[
'defects']
is not None:
1021 if not isinstance(inputs[
"defects"], Defects):
1022 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1026 if self.config.doBrighterFatter:
1027 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1028 if brighterFatterKernel
is None:
1029 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1031 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1033 detName = detector.getName()
1034 level = brighterFatterKernel.level
1037 inputs[
'bfGains'] = brighterFatterKernel.gain
1038 if self.config.brighterFatterLevel ==
'DETECTOR':
1039 if level ==
'DETECTOR':
1040 if detName
in brighterFatterKernel.detKernels:
1041 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1043 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1044 elif level ==
'AMP':
1045 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1047 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1048 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1049 elif self.config.brighterFatterLevel ==
'AMP':
1050 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1052 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1053 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
1054 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1056 assembler=self.assembleCcd
1057 if self.config.doAssembleIsrExposures
else None)
1059 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1061 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1062 if 'strayLightData' not in inputs:
1063 inputs[
'strayLightData'] =
None
1065 outputs = self.
runrun(**inputs)
1066 butlerQC.put(outputs, outputRefs)
1069 """Retrieve necessary frames for instrument signature removal.
1071 Pre-fetching all required ISR data products limits the IO
1072 required by the ISR. Any conflict between the calibration data
1073 available and that needed for ISR is also detected prior to
1074 doing processing, allowing it to fail quickly.
1078 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1079 Butler reference of the detector data to be processed
1080 rawExposure : `afw.image.Exposure`
1081 The raw exposure that will later be corrected with the
1082 retrieved calibration data; should not be modified in this
1087 result : `lsst.pipe.base.Struct`
1088 Result struct with components (which may be `None`):
1089 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1090 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1091 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1092 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1093 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1094 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1095 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1096 - ``fringes``: `lsst.pipe.base.Struct` with components:
1097 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1098 - ``seed``: random seed derived from the ccdExposureId for random
1099 number generator (`uint32`).
1100 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1101 A ``TransmissionCurve`` that represents the throughput of the optics,
1102 to be evaluated in focal-plane coordinates.
1103 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1104 A ``TransmissionCurve`` that represents the throughput of the filter
1105 itself, to be evaluated in focal-plane coordinates.
1106 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1107 A ``TransmissionCurve`` that represents the throughput of the sensor
1108 itself, to be evaluated in post-assembly trimmed detector coordinates.
1109 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1110 A ``TransmissionCurve`` that represents the throughput of the
1111 atmosphere, assumed to be spatially constant.
1112 - ``strayLightData`` : `object`
1113 An opaque object containing calibration information for
1114 stray-light correction. If `None`, no correction will be
1116 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1120 NotImplementedError :
1121 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1124 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1125 dateObs = dateObs.toPython().isoformat()
1126 except RuntimeError:
1127 self.log.warning(
"Unable to identify dateObs for rawExposure.")
1130 ccd = rawExposure.getDetector()
1131 filterLabel = rawExposure.getFilterLabel()
1132 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1133 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1134 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1135 if self.config.doBias
else None)
1137 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1139 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1140 linearizer.log = self.log
1141 if isinstance(linearizer, numpy.ndarray):
1144 crosstalkCalib =
None
1145 if self.config.doCrosstalk:
1147 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1149 coeffVector = (self.config.crosstalk.crosstalkValues
1150 if self.config.crosstalk.useConfigCoefficients
else None)
1151 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1152 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1153 if self.config.doCrosstalk
else None)
1155 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1156 if self.config.doDark
else None)
1157 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1159 if self.config.doFlat
else None)
1161 brighterFatterKernel =
None
1162 brighterFatterGains =
None
1163 if self.config.doBrighterFatter
is True:
1168 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1169 brighterFatterGains = brighterFatterKernel.gain
1170 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1173 brighterFatterKernel = dataRef.get(
"bfKernel")
1174 self.log.info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1176 brighterFatterKernel =
None
1177 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1180 if self.config.brighterFatterLevel ==
'DETECTOR':
1181 if brighterFatterKernel.detKernels:
1182 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1184 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1187 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1189 defectList = (dataRef.get(
"defects")
1190 if self.config.doDefect
else None)
1191 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1192 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1193 if self.config.doAssembleIsrExposures
else None)
1194 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1195 else pipeBase.Struct(fringes=
None))
1197 if self.config.doAttachTransmissionCurve:
1198 opticsTransmission = (dataRef.get(
"transmission_optics")
1199 if self.config.doUseOpticsTransmission
else None)
1200 filterTransmission = (dataRef.get(
"transmission_filter")
1201 if self.config.doUseFilterTransmission
else None)
1202 sensorTransmission = (dataRef.get(
"transmission_sensor")
1203 if self.config.doUseSensorTransmission
else None)
1204 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1205 if self.config.doUseAtmosphereTransmission
else None)
1207 opticsTransmission =
None
1208 filterTransmission =
None
1209 sensorTransmission =
None
1210 atmosphereTransmission =
None
1212 if self.config.doStrayLight:
1213 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1215 strayLightData =
None
1218 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1219 if (self.config.doIlluminationCorrection
1220 and physicalFilter
in self.config.illumFilters)
1224 return pipeBase.Struct(bias=biasExposure,
1225 linearizer=linearizer,
1226 crosstalk=crosstalkCalib,
1227 crosstalkSources=crosstalkSources,
1230 bfKernel=brighterFatterKernel,
1231 bfGains=brighterFatterGains,
1233 fringes=fringeStruct,
1234 opticsTransmission=opticsTransmission,
1235 filterTransmission=filterTransmission,
1236 sensorTransmission=sensorTransmission,
1237 atmosphereTransmission=atmosphereTransmission,
1238 strayLightData=strayLightData,
1239 illumMaskedImage=illumMaskedImage
1242 @pipeBase.timeMethod
1243 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1244 crosstalk=None, crosstalkSources=None,
1245 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1246 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1247 sensorTransmission=
None, atmosphereTransmission=
None,
1248 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1251 """Perform instrument signature removal on an exposure.
1253 Steps included in the ISR processing, in order performed, are:
1254 - saturation and suspect pixel masking
1255 - overscan subtraction
1256 - CCD assembly of individual amplifiers
1258 - variance image construction
1259 - linearization of non-linear response
1261 - brighter-fatter correction
1264 - stray light subtraction
1266 - masking of known defects and camera specific features
1267 - vignette calculation
1268 - appending transmission curve and distortion model
1272 ccdExposure : `lsst.afw.image.Exposure`
1273 The raw exposure that is to be run through ISR. The
1274 exposure is modified by this method.
1275 camera : `lsst.afw.cameraGeom.Camera`, optional
1276 The camera geometry for this exposure. Required if ``isGen3`` is
1277 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1278 ``flat`` does not have an associated detector.
1279 bias : `lsst.afw.image.Exposure`, optional
1280 Bias calibration frame.
1281 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1282 Functor for linearization.
1283 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1284 Calibration for crosstalk.
1285 crosstalkSources : `list`, optional
1286 List of possible crosstalk sources.
1287 dark : `lsst.afw.image.Exposure`, optional
1288 Dark calibration frame.
1289 flat : `lsst.afw.image.Exposure`, optional
1290 Flat calibration frame.
1291 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1292 Photon transfer curve dataset, with, e.g., gains
1294 bfKernel : `numpy.ndarray`, optional
1295 Brighter-fatter kernel.
1296 bfGains : `dict` of `float`, optional
1297 Gains used to override the detector's nominal gains for the
1298 brighter-fatter correction. A dict keyed by amplifier name for
1299 the detector in question.
1300 defects : `lsst.ip.isr.Defects`, optional
1302 fringes : `lsst.pipe.base.Struct`, optional
1303 Struct containing the fringe correction data, with
1305 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1306 - ``seed``: random seed derived from the ccdExposureId for random
1307 number generator (`uint32`)
1308 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1309 A ``TransmissionCurve`` that represents the throughput of the optics,
1310 to be evaluated in focal-plane coordinates.
1311 filterTransmission : `lsst.afw.image.TransmissionCurve`
1312 A ``TransmissionCurve`` that represents the throughput of the filter
1313 itself, to be evaluated in focal-plane coordinates.
1314 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1315 A ``TransmissionCurve`` that represents the throughput of the sensor
1316 itself, to be evaluated in post-assembly trimmed detector coordinates.
1317 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1318 A ``TransmissionCurve`` that represents the throughput of the
1319 atmosphere, assumed to be spatially constant.
1320 detectorNum : `int`, optional
1321 The integer number for the detector to process.
1322 isGen3 : bool, optional
1323 Flag this call to run() as using the Gen3 butler environment.
1324 strayLightData : `object`, optional
1325 Opaque object containing calibration information for stray-light
1326 correction. If `None`, no correction will be performed.
1327 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1328 Illumination correction image.
1332 result : `lsst.pipe.base.Struct`
1333 Result struct with component:
1334 - ``exposure`` : `afw.image.Exposure`
1335 The fully ISR corrected exposure.
1336 - ``outputExposure`` : `afw.image.Exposure`
1337 An alias for `exposure`
1338 - ``ossThumb`` : `numpy.ndarray`
1339 Thumbnail image of the exposure after overscan subtraction.
1340 - ``flattenedThumb`` : `numpy.ndarray`
1341 Thumbnail image of the exposure after flat-field correction.
1346 Raised if a configuration option is set to True, but the
1347 required calibration data has not been specified.
1351 The current processed exposure can be viewed by setting the
1352 appropriate lsstDebug entries in the `debug.display`
1353 dictionary. The names of these entries correspond to some of
1354 the IsrTaskConfig Boolean options, with the value denoting the
1355 frame to use. The exposure is shown inside the matching
1356 option check and after the processing of that step has
1357 finished. The steps with debug points are:
1368 In addition, setting the "postISRCCD" entry displays the
1369 exposure after all ISR processing has finished.
1377 if detectorNum
is None:
1378 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1380 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1381 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1382 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1383 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1385 if isinstance(ccdExposure, ButlerDataRef):
1386 return self.
runDataRefrunDataRef(ccdExposure)
1388 ccd = ccdExposure.getDetector()
1389 filterLabel = ccdExposure.getFilterLabel()
1390 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1393 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1394 ccd = [
FakeAmp(ccdExposure, self.config)]
1397 if self.config.doBias
and bias
is None:
1398 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1399 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1400 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1401 if self.config.doBrighterFatter
and bfKernel
is None:
1402 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1403 if self.config.doDark
and dark
is None:
1404 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1405 if self.config.doFlat
and flat
is None:
1406 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1407 if self.config.doDefect
and defects
is None:
1408 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1409 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1410 and fringes.fringes
is None):
1415 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1416 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1417 and illumMaskedImage
is None):
1418 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1421 if self.config.doConvertIntToFloat:
1422 self.log.info(
"Converting exposure to floating point values.")
1425 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1426 self.log.info(
"Applying bias correction.")
1427 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1428 trimToFit=self.config.doTrimToMatchCalib)
1429 self.
debugViewdebugView(ccdExposure,
"doBias")
1435 if ccdExposure.getBBox().contains(amp.getBBox()):
1437 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1439 if self.config.doOverscan
and not badAmp:
1442 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1443 if overscanResults
is not None and \
1444 self.config.qa
is not None and self.config.qa.saveStats
is True:
1445 if isinstance(overscanResults.overscanFit, float):
1446 qaMedian = overscanResults.overscanFit
1447 qaStdev = float(
"NaN")
1449 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1450 afwMath.MEDIAN | afwMath.STDEVCLIP)
1451 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1452 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1454 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1455 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1456 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1457 amp.getName(), qaMedian, qaStdev)
1460 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1461 afwMath.MEDIAN | afwMath.STDEVCLIP)
1462 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1463 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1465 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1466 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1467 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1468 amp.getName(), qaMedianAfter, qaStdevAfter)
1470 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1473 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1474 overscanResults =
None
1476 overscans.append(overscanResults
if overscanResults
is not None else None)
1478 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1480 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1481 self.log.info(
"Applying crosstalk correction.")
1482 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1483 crosstalkSources=crosstalkSources, camera=camera)
1484 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1486 if self.config.doAssembleCcd:
1487 self.log.info(
"Assembling CCD from amplifiers.")
1488 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1490 if self.config.expectWcs
and not ccdExposure.getWcs():
1491 self.log.warning(
"No WCS found in input exposure.")
1492 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1495 if self.config.qa.doThumbnailOss:
1496 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1498 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1499 self.log.info(
"Applying bias correction.")
1500 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1501 trimToFit=self.config.doTrimToMatchCalib)
1502 self.
debugViewdebugView(ccdExposure,
"doBias")
1504 if self.config.doVariance:
1505 for amp, overscanResults
in zip(ccd, overscans):
1506 if ccdExposure.getBBox().contains(amp.getBBox()):
1507 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1508 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1509 if overscanResults
is not None:
1511 overscanImage=overscanResults.overscanImage,
1517 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1518 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1519 afwMath.MEDIAN | afwMath.STDEVCLIP)
1520 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1521 qaStats.getValue(afwMath.MEDIAN))
1522 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1523 qaStats.getValue(afwMath.STDEVCLIP))
1524 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1525 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1526 qaStats.getValue(afwMath.STDEVCLIP))
1527 if self.config.maskNegativeVariance:
1531 self.log.info(
"Applying linearizer.")
1532 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1533 detector=ccd, log=self.log)
1535 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1536 self.log.info(
"Applying crosstalk correction.")
1537 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1538 crosstalkSources=crosstalkSources, isTrimmed=
True)
1539 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1543 if self.config.doDefect:
1544 self.log.info(
"Masking defects.")
1545 self.
maskDefectmaskDefect(ccdExposure, defects)
1547 if self.config.numEdgeSuspect > 0:
1548 self.log.info(
"Masking edges as SUSPECT.")
1549 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1550 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1552 if self.config.doNanMasking:
1553 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1554 self.
maskNanmaskNan(ccdExposure)
1556 if self.config.doWidenSaturationTrails:
1557 self.log.info(
"Widening saturation trails.")
1558 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1560 if self.config.doCameraSpecificMasking:
1561 self.log.info(
"Masking regions for camera specific reasons.")
1562 self.masking.
run(ccdExposure)
1564 if self.config.doBrighterFatter:
1573 interpExp = ccdExposure.clone()
1574 with self.
flatContextflatContext(interpExp, flat, dark):
1575 isrFunctions.interpolateFromMask(
1576 maskedImage=interpExp.getMaskedImage(),
1577 fwhm=self.config.fwhm,
1578 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1579 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1581 bfExp = interpExp.clone()
1583 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1584 type(bfKernel), type(bfGains))
1585 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1586 self.config.brighterFatterMaxIter,
1587 self.config.brighterFatterThreshold,
1588 self.config.brighterFatterApplyGain,
1590 if bfResults[1] == self.config.brighterFatterMaxIter:
1591 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1594 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1596 image = ccdExposure.getMaskedImage().getImage()
1597 bfCorr = bfExp.getMaskedImage().getImage()
1598 bfCorr -= interpExp.getMaskedImage().getImage()
1607 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1608 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1611 if self.config.brighterFatterMaskGrowSize > 0:
1612 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1613 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1614 isrFunctions.growMasks(ccdExposure.getMask(),
1615 radius=self.config.brighterFatterMaskGrowSize,
1616 maskNameList=maskPlane,
1617 maskValue=maskPlane)
1619 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1621 if self.config.doDark:
1622 self.log.info(
"Applying dark correction.")
1624 self.
debugViewdebugView(ccdExposure,
"doDark")
1626 if self.config.doFringe
and not self.config.fringeAfterFlat:
1627 self.log.info(
"Applying fringe correction before flat.")
1628 self.fringe.
run(ccdExposure, **fringes.getDict())
1629 self.
debugViewdebugView(ccdExposure,
"doFringe")
1631 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1632 self.log.info(
"Checking strayLight correction.")
1633 self.strayLight.
run(ccdExposure, strayLightData)
1634 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1636 if self.config.doFlat:
1637 self.log.info(
"Applying flat correction.")
1639 self.
debugViewdebugView(ccdExposure,
"doFlat")
1641 if self.config.doApplyGains:
1642 self.log.info(
"Applying gain correction instead of flat.")
1643 if self.config.usePtcGains:
1644 self.log.info(
"Using gains from the Photon Transfer Curve.")
1645 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1648 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1650 if self.config.doFringe
and self.config.fringeAfterFlat:
1651 self.log.info(
"Applying fringe correction after flat.")
1652 self.fringe.
run(ccdExposure, **fringes.getDict())
1654 if self.config.doVignette:
1655 self.log.info(
"Constructing Vignette polygon.")
1658 if self.config.vignette.doWriteVignettePolygon:
1661 if self.config.doAttachTransmissionCurve:
1662 self.log.info(
"Adding transmission curves.")
1663 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1664 filterTransmission=filterTransmission,
1665 sensorTransmission=sensorTransmission,
1666 atmosphereTransmission=atmosphereTransmission)
1668 flattenedThumb =
None
1669 if self.config.qa.doThumbnailFlattened:
1670 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1672 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1673 self.log.info(
"Performing illumination correction.")
1674 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1675 illumMaskedImage, illumScale=self.config.illumScale,
1676 trimToFit=self.config.doTrimToMatchCalib)
1679 if self.config.doSaveInterpPixels:
1680 preInterpExp = ccdExposure.clone()
1695 if self.config.doSetBadRegions:
1696 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1697 if badPixelCount > 0:
1698 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1700 if self.config.doInterpolate:
1701 self.log.info(
"Interpolating masked pixels.")
1702 isrFunctions.interpolateFromMask(
1703 maskedImage=ccdExposure.getMaskedImage(),
1704 fwhm=self.config.fwhm,
1705 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1706 maskNameList=list(self.config.maskListToInterpolate)
1711 if self.config.doMeasureBackground:
1712 self.log.info(
"Measuring background level.")
1715 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1717 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1718 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1719 afwMath.MEDIAN | afwMath.STDEVCLIP)
1720 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1721 qaStats.getValue(afwMath.MEDIAN))
1722 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1723 qaStats.getValue(afwMath.STDEVCLIP))
1724 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1725 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1726 qaStats.getValue(afwMath.STDEVCLIP))
1728 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1730 return pipeBase.Struct(
1731 exposure=ccdExposure,
1733 flattenedThumb=flattenedThumb,
1735 preInterpExposure=preInterpExp,
1736 outputExposure=ccdExposure,
1737 outputOssThumbnail=ossThumb,
1738 outputFlattenedThumbnail=flattenedThumb,
1741 @pipeBase.timeMethod
1743 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1745 This method contains the `CmdLineTask` interface to the ISR
1746 processing. All IO is handled here, freeing the `run()` method
1747 to manage only pixel-level calculations. The steps performed
1749 - Read in necessary detrending/isr/calibration data.
1750 - Process raw exposure in `run()`.
1751 - Persist the ISR-corrected exposure as "postISRCCD" if
1752 config.doWrite=True.
1756 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1757 DataRef of the detector data to be processed
1761 result : `lsst.pipe.base.Struct`
1762 Result struct with component:
1763 - ``exposure`` : `afw.image.Exposure`
1764 The fully ISR corrected exposure.
1769 Raised if a configuration option is set to True, but the
1770 required calibration data does not exist.
1773 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1775 ccdExposure = sensorRef.get(self.config.datasetType)
1777 camera = sensorRef.get(
"camera")
1778 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1780 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1782 if self.config.doWrite:
1783 sensorRef.put(result.exposure,
"postISRCCD")
1784 if result.preInterpExposure
is not None:
1785 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1786 if result.ossThumb
is not None:
1787 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1788 if result.flattenedThumb
is not None:
1789 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1794 """Retrieve a calibration dataset for removing instrument signature.
1799 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1800 DataRef of the detector data to find calibration datasets
1803 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1804 dateObs : `str`, optional
1805 Date of the observation. Used to correct butler failures
1806 when using fallback filters.
1808 If True, disable butler proxies to enable error handling
1809 within this routine.
1813 exposure : `lsst.afw.image.Exposure`
1814 Requested calibration frame.
1819 Raised if no matching calibration frame can be found.
1822 exp = dataRef.get(datasetType, immediate=immediate)
1823 except Exception
as exc1:
1824 if not self.config.fallbackFilterName:
1825 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1827 if self.config.useFallbackDate
and dateObs:
1828 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1829 dateObs=dateObs, immediate=immediate)
1831 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1832 except Exception
as exc2:
1833 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1834 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1835 self.log.warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1837 if self.config.doAssembleIsrExposures:
1838 exp = self.assembleCcd.assembleCcd(exp)
1842 """Ensure that the data returned by Butler is a fully constructed exposure.
1844 ISR requires exposure-level image data for historical reasons, so if we did
1845 not recieve that from Butler, construct it from what we have, modifying the
1850 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1851 `lsst.afw.image.ImageF`
1852 The input data structure obtained from Butler.
1853 camera : `lsst.afw.cameraGeom.camera`
1854 The camera associated with the image. Used to find the appropriate
1857 The detector this exposure should match.
1861 inputExp : `lsst.afw.image.Exposure`
1862 The re-constructed exposure, with appropriate detector parameters.
1867 Raised if the input data cannot be used to construct an exposure.
1869 if isinstance(inputExp, afwImage.DecoratedImageU):
1870 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1871 elif isinstance(inputExp, afwImage.ImageF):
1872 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1873 elif isinstance(inputExp, afwImage.MaskedImageF):
1874 inputExp = afwImage.makeExposure(inputExp)
1875 elif isinstance(inputExp, afwImage.Exposure):
1877 elif inputExp
is None:
1881 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1884 if inputExp.getDetector()
is None:
1885 inputExp.setDetector(camera[detectorNum])
1890 """Convert exposure image from uint16 to float.
1892 If the exposure does not need to be converted, the input is
1893 immediately returned. For exposures that are converted to use
1894 floating point pixels, the variance is set to unity and the
1899 exposure : `lsst.afw.image.Exposure`
1900 The raw exposure to be converted.
1904 newexposure : `lsst.afw.image.Exposure`
1905 The input ``exposure``, converted to floating point pixels.
1910 Raised if the exposure type cannot be converted to float.
1913 if isinstance(exposure, afwImage.ExposureF):
1915 self.log.debug(
"Exposure already of type float.")
1917 if not hasattr(exposure,
"convertF"):
1918 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1920 newexposure = exposure.convertF()
1921 newexposure.variance[:] = 1
1922 newexposure.mask[:] = 0x0
1927 """Identify bad amplifiers, saturated and suspect pixels.
1931 ccdExposure : `lsst.afw.image.Exposure`
1932 Input exposure to be masked.
1933 amp : `lsst.afw.table.AmpInfoCatalog`
1934 Catalog of parameters defining the amplifier on this
1936 defects : `lsst.ip.isr.Defects`
1937 List of defects. Used to determine if the entire
1943 If this is true, the entire amplifier area is covered by
1944 defects and unusable.
1947 maskedImage = ccdExposure.getMaskedImage()
1953 if defects
is not None:
1954 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1959 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1961 maskView = dataView.getMask()
1962 maskView |= maskView.getPlaneBitMask(
"BAD")
1969 if self.config.doSaturation
and not badAmp:
1970 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1971 if self.config.doSuspect
and not badAmp:
1972 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1973 if math.isfinite(self.config.saturation):
1974 limits.update({self.config.saturatedMaskName: self.config.saturation})
1976 for maskName, maskThreshold
in limits.items():
1977 if not math.isnan(maskThreshold):
1978 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1979 isrFunctions.makeThresholdMask(
1980 maskedImage=dataView,
1981 threshold=maskThreshold,
1987 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1989 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1990 self.config.suspectMaskName])
1991 if numpy.all(maskView.getArray() & maskVal > 0):
1993 maskView |= maskView.getPlaneBitMask(
"BAD")
1998 """Apply overscan correction in place.
2000 This method does initial pixel rejection of the overscan
2001 region. The overscan can also be optionally segmented to
2002 allow for discontinuous overscan responses to be fit
2003 separately. The actual overscan subtraction is performed by
2004 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2005 which is called here after the amplifier is preprocessed.
2009 ccdExposure : `lsst.afw.image.Exposure`
2010 Exposure to have overscan correction performed.
2011 amp : `lsst.afw.cameraGeom.Amplifer`
2012 The amplifier to consider while correcting the overscan.
2016 overscanResults : `lsst.pipe.base.Struct`
2017 Result struct with components:
2018 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2019 Value or fit subtracted from the amplifier image data.
2020 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2021 Value or fit subtracted from the overscan image data.
2022 - ``overscanImage`` : `lsst.afw.image.Image`
2023 Image of the overscan region with the overscan
2024 correction applied. This quantity is used to estimate
2025 the amplifier read noise empirically.
2030 Raised if the ``amp`` does not contain raw pixel information.
2034 lsst.ip.isr.isrFunctions.overscanCorrection
2036 if amp.getRawHorizontalOverscanBBox().isEmpty():
2037 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2040 statControl = afwMath.StatisticsControl()
2041 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2044 dataBBox = amp.getRawDataBBox()
2045 oscanBBox = amp.getRawHorizontalOverscanBBox()
2049 prescanBBox = amp.getRawPrescanBBox()
2050 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2051 dx0 += self.config.overscanNumLeadingColumnsToSkip
2052 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2054 dx0 += self.config.overscanNumTrailingColumnsToSkip
2055 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2061 if ((self.config.overscanBiasJump
2062 and self.config.overscanBiasJumpLocation)
2063 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2064 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2065 self.config.overscanBiasJumpDevices)):
2066 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2067 yLower = self.config.overscanBiasJumpLocation
2068 yUpper = dataBBox.getHeight() - yLower
2070 yUpper = self.config.overscanBiasJumpLocation
2071 yLower = dataBBox.getHeight() - yUpper
2089 oscanBBox.getHeight())))
2092 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2093 ampImage = ccdExposure.maskedImage[imageBBox]
2094 overscanImage = ccdExposure.maskedImage[overscanBBox]
2096 overscanArray = overscanImage.image.array
2097 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2098 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2099 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2101 statControl = afwMath.StatisticsControl()
2102 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2104 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2107 levelStat = afwMath.MEDIAN
2108 sigmaStat = afwMath.STDEVCLIP
2110 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2111 self.config.qa.flatness.nIter)
2112 metadata = ccdExposure.getMetadata()
2113 ampNum = amp.getName()
2115 if isinstance(overscanResults.overscanFit, float):
2116 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2117 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2119 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2120 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2121 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2123 return overscanResults
2126 """Set the variance plane using the gain and read noise
2128 The read noise is calculated from the ``overscanImage`` if the
2129 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2130 the value from the amplifier data is used.
2134 ampExposure : `lsst.afw.image.Exposure`
2135 Exposure to process.
2136 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2137 Amplifier detector data.
2138 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2139 Image of overscan, required only for empirical read noise.
2140 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2141 PTC dataset containing the gains and read noise.
2147 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2148 are ``True``, but ptcDataset is not provided.
2150 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2151 ``overscanImage`` is ``None``.
2155 lsst.ip.isr.isrFunctions.updateVariance
2157 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2158 if self.config.usePtcGains:
2159 if ptcDataset
is None:
2160 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2162 gain = ptcDataset.gain[amp.getName()]
2163 self.log.info(
"Using gain from Photon Transfer Curve.")
2165 gain = amp.getGain()
2167 if math.isnan(gain):
2169 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2172 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2173 amp.getName(), gain, patchedGain)
2176 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2177 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2179 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2180 stats = afwMath.StatisticsControl()
2181 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2182 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2183 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2184 amp.getName(), readNoise)
2185 elif self.config.usePtcReadNoise:
2186 if ptcDataset
is None:
2187 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2189 readNoise = ptcDataset.noise[amp.getName()]
2190 self.log.info(
"Using read noise from Photon Transfer Curve.")
2192 readNoise = amp.getReadNoise()
2194 isrFunctions.updateVariance(
2195 maskedImage=ampExposure.getMaskedImage(),
2197 readNoise=readNoise,
2201 """Identify and mask pixels with negative variance values.
2205 exposure : `lsst.afw.image.Exposure`
2206 Exposure to process.
2210 lsst.ip.isr.isrFunctions.updateVariance
2212 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2213 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2214 exposure.mask.array[bad] |= maskPlane
2217 """Apply dark correction in place.
2221 exposure : `lsst.afw.image.Exposure`
2222 Exposure to process.
2223 darkExposure : `lsst.afw.image.Exposure`
2224 Dark exposure of the same size as ``exposure``.
2225 invert : `Bool`, optional
2226 If True, re-add the dark to an already corrected image.
2231 Raised if either ``exposure`` or ``darkExposure`` do not
2232 have their dark time defined.
2236 lsst.ip.isr.isrFunctions.darkCorrection
2238 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2239 if math.isnan(expScale):
2240 raise RuntimeError(
"Exposure darktime is NAN.")
2241 if darkExposure.getInfo().getVisitInfo()
is not None \
2242 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2243 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2247 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2250 isrFunctions.darkCorrection(
2251 maskedImage=exposure.getMaskedImage(),
2252 darkMaskedImage=darkExposure.getMaskedImage(),
2254 darkScale=darkScale,
2256 trimToFit=self.config.doTrimToMatchCalib
2260 """Check if linearization is needed for the detector cameraGeom.
2262 Checks config.doLinearize and the linearity type of the first
2267 detector : `lsst.afw.cameraGeom.Detector`
2268 Detector to get linearity type from.
2272 doLinearize : `Bool`
2273 If True, linearization should be performed.
2275 return self.config.doLinearize
and \
2276 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2279 """Apply flat correction in place.
2283 exposure : `lsst.afw.image.Exposure`
2284 Exposure to process.
2285 flatExposure : `lsst.afw.image.Exposure`
2286 Flat exposure of the same size as ``exposure``.
2287 invert : `Bool`, optional
2288 If True, unflatten an already flattened image.
2292 lsst.ip.isr.isrFunctions.flatCorrection
2294 isrFunctions.flatCorrection(
2295 maskedImage=exposure.getMaskedImage(),
2296 flatMaskedImage=flatExposure.getMaskedImage(),
2297 scalingType=self.config.flatScalingType,
2298 userScale=self.config.flatUserScale,
2300 trimToFit=self.config.doTrimToMatchCalib
2304 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2308 exposure : `lsst.afw.image.Exposure`
2309 Exposure to process. Only the amplifier DataSec is processed.
2310 amp : `lsst.afw.table.AmpInfoCatalog`
2311 Amplifier detector data.
2315 lsst.ip.isr.isrFunctions.makeThresholdMask
2317 if not math.isnan(amp.getSaturation()):
2318 maskedImage = exposure.getMaskedImage()
2319 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2320 isrFunctions.makeThresholdMask(
2321 maskedImage=dataView,
2322 threshold=amp.getSaturation(),
2324 maskName=self.config.saturatedMaskName,
2328 """Interpolate over saturated pixels, in place.
2330 This method should be called after `saturationDetection`, to
2331 ensure that the saturated pixels have been identified in the
2332 SAT mask. It should also be called after `assembleCcd`, since
2333 saturated regions may cross amplifier boundaries.
2337 exposure : `lsst.afw.image.Exposure`
2338 Exposure to process.
2342 lsst.ip.isr.isrTask.saturationDetection
2343 lsst.ip.isr.isrFunctions.interpolateFromMask
2345 isrFunctions.interpolateFromMask(
2346 maskedImage=exposure.getMaskedImage(),
2347 fwhm=self.config.fwhm,
2348 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2349 maskNameList=list(self.config.saturatedMaskName),
2353 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2357 exposure : `lsst.afw.image.Exposure`
2358 Exposure to process. Only the amplifier DataSec is processed.
2359 amp : `lsst.afw.table.AmpInfoCatalog`
2360 Amplifier detector data.
2364 lsst.ip.isr.isrFunctions.makeThresholdMask
2368 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2369 This is intended to indicate pixels that may be affected by unknown systematics;
2370 for example if non-linearity corrections above a certain level are unstable
2371 then that would be a useful value for suspectLevel. A value of `nan` indicates
2372 that no such level exists and no pixels are to be masked as suspicious.
2374 suspectLevel = amp.getSuspectLevel()
2375 if math.isnan(suspectLevel):
2378 maskedImage = exposure.getMaskedImage()
2379 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2380 isrFunctions.makeThresholdMask(
2381 maskedImage=dataView,
2382 threshold=suspectLevel,
2384 maskName=self.config.suspectMaskName,
2388 """Mask defects using mask plane "BAD", in place.
2392 exposure : `lsst.afw.image.Exposure`
2393 Exposure to process.
2394 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2395 `lsst.afw.image.DefectBase`.
2396 List of defects to mask.
2400 Call this after CCD assembly, since defects may cross amplifier boundaries.
2402 maskedImage = exposure.getMaskedImage()
2403 if not isinstance(defectBaseList, Defects):
2405 defectList =
Defects(defectBaseList)
2407 defectList = defectBaseList
2408 defectList.maskPixels(maskedImage, maskName=
"BAD")
2410 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2411 """Mask edge pixels with applicable mask plane.
2415 exposure : `lsst.afw.image.Exposure`
2416 Exposure to process.
2417 numEdgePixels : `int`, optional
2418 Number of edge pixels to mask.
2419 maskPlane : `str`, optional
2420 Mask plane name to use.
2421 level : `str`, optional
2422 Level at which to mask edges.
2424 maskedImage = exposure.getMaskedImage()
2425 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2427 if numEdgePixels > 0:
2428 if level ==
'DETECTOR':
2429 boxes = [maskedImage.getBBox()]
2430 elif level ==
'AMP':
2431 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2435 subImage = maskedImage[box]
2436 box.grow(-numEdgePixels)
2438 SourceDetectionTask.setEdgeBits(
2444 """Mask and interpolate defects using mask plane "BAD", in place.
2448 exposure : `lsst.afw.image.Exposure`
2449 Exposure to process.
2450 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2451 `lsst.afw.image.DefectBase`.
2452 List of defects to mask and interpolate.
2456 lsst.ip.isr.isrTask.maskDefect
2458 self.
maskDefectmaskDefect(exposure, defectBaseList)
2459 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2460 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2461 isrFunctions.interpolateFromMask(
2462 maskedImage=exposure.getMaskedImage(),
2463 fwhm=self.config.fwhm,
2464 growSaturatedFootprints=0,
2465 maskNameList=[
"BAD"],
2469 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2473 exposure : `lsst.afw.image.Exposure`
2474 Exposure to process.
2478 We mask over all non-finite values (NaN, inf), including those
2479 that are masked with other bits (because those may or may not be
2480 interpolated over later, and we want to remove all NaN/infs).
2481 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2482 preserve the historical name.
2484 maskedImage = exposure.getMaskedImage()
2487 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2488 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2489 numNans =
maskNans(maskedImage, maskVal)
2490 self.metadata.set(
"NUMNANS", numNans)
2492 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2495 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2500 exposure : `lsst.afw.image.Exposure`
2501 Exposure to process.
2505 lsst.ip.isr.isrTask.maskNan
2508 isrFunctions.interpolateFromMask(
2509 maskedImage=exposure.getMaskedImage(),
2510 fwhm=self.config.fwhm,
2511 growSaturatedFootprints=0,
2512 maskNameList=[
"UNMASKEDNAN"],
2516 """Measure the image background in subgrids, for quality control purposes.
2520 exposure : `lsst.afw.image.Exposure`
2521 Exposure to process.
2522 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2523 Configuration object containing parameters on which background
2524 statistics and subgrids to use.
2526 if IsrQaConfig
is not None:
2527 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2528 IsrQaConfig.flatness.nIter)
2529 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2530 statsControl.setAndMask(maskVal)
2531 maskedImage = exposure.getMaskedImage()
2532 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2533 skyLevel = stats.getValue(afwMath.MEDIAN)
2534 skySigma = stats.getValue(afwMath.STDEVCLIP)
2535 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2536 metadata = exposure.getMetadata()
2537 metadata.set(
'SKYLEVEL', skyLevel)
2538 metadata.set(
'SKYSIGMA', skySigma)
2541 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2542 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2543 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2544 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2545 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2546 skyLevels = numpy.zeros((nX, nY))
2549 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2551 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2553 xLLC = xc - meshXHalf
2554 yLLC = yc - meshYHalf
2555 xURC = xc + meshXHalf - 1
2556 yURC = yc + meshYHalf - 1
2559 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2561 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2563 good = numpy.where(numpy.isfinite(skyLevels))
2564 skyMedian = numpy.median(skyLevels[good])
2565 flatness = (skyLevels[good] - skyMedian) / skyMedian
2566 flatness_rms = numpy.std(flatness)
2567 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2569 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2570 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2571 nX, nY, flatness_pp, flatness_rms)
2573 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2574 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2575 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2576 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2577 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2580 """Set an approximate magnitude zero point for the exposure.
2584 exposure : `lsst.afw.image.Exposure`
2585 Exposure to process.
2587 filterLabel = exposure.getFilterLabel()
2588 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2590 if physicalFilter
in self.config.fluxMag0T1:
2591 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2593 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2594 fluxMag0 = self.config.defaultFluxMag0T1
2596 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2598 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2601 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2602 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2603 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2606 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2610 ccdExposure : `lsst.afw.image.Exposure`
2611 Exposure to process.
2612 fpPolygon : `lsst.afw.geom.Polygon`
2613 Polygon in focal plane coordinates.
2616 ccd = ccdExposure.getDetector()
2617 fpCorners = ccd.getCorners(FOCAL_PLANE)
2618 ccdPolygon = Polygon(fpCorners)
2621 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2624 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2625 validPolygon = Polygon(ccdPoints)
2626 ccdExposure.getInfo().setValidPolygon(validPolygon)
2630 """Context manager that applies and removes flats and darks,
2631 if the task is configured to apply them.
2635 exp : `lsst.afw.image.Exposure`
2636 Exposure to process.
2637 flat : `lsst.afw.image.Exposure`
2638 Flat exposure the same size as ``exp``.
2639 dark : `lsst.afw.image.Exposure`, optional
2640 Dark exposure the same size as ``exp``.
2644 exp : `lsst.afw.image.Exposure`
2645 The flat and dark corrected exposure.
2647 if self.config.doDark
and dark
is not None:
2649 if self.config.doFlat:
2654 if self.config.doFlat:
2656 if self.config.doDark
and dark
is not None:
2660 """Utility function to examine ISR exposure at different stages.
2664 exposure : `lsst.afw.image.Exposure`
2667 State of processing to view.
2669 frame = getDebugFrame(self._display, stepname)
2671 display = getDisplay(frame)
2672 display.scale(
'asinh',
'zscale')
2673 display.mtv(exposure)
2674 prompt =
"Press Enter to continue [c]... "
2676 ans = input(prompt).lower()
2677 if ans
in (
"",
"c",):
2682 """A Detector-like object that supports returning gain and saturation level
2684 This is used when the input exposure does not have a detector.
2688 exposure : `lsst.afw.image.Exposure`
2689 Exposure to generate a fake amplifier for.
2690 config : `lsst.ip.isr.isrTaskConfig`
2691 Configuration to apply to the fake amplifier.
2695 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2697 self.
_gain_gain = config.gain
2702 return self.
_bbox_bbox
2705 return self.
_bbox_bbox
2711 return self.
_gain_gain
2724 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2728 """Task to wrap the default IsrTask to allow it to be retargeted.
2730 The standard IsrTask can be called directly from a command line
2731 program, but doing so removes the ability of the task to be
2732 retargeted. As most cameras override some set of the IsrTask
2733 methods, this would remove those data-specific methods in the
2734 output post-ISR images. This wrapping class fixes the issue,
2735 allowing identical post-ISR images to be generated by both the
2736 processCcd and isrTask code.
2738 ConfigClass = RunIsrConfig
2739 _DefaultName =
"runIsr"
2743 self.makeSubtask(
"isr")
2749 dataRef : `lsst.daf.persistence.ButlerDataRef`
2750 data reference of the detector data to be processed
2754 result : `pipeBase.Struct`
2755 Result struct with component:
2757 - exposure : `lsst.afw.image.Exposure`
2758 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def ensureExposure(self, inputExp, camera, detectorNum)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.