29 import lsst.pipe.base
as pipeBase
30 import lsst.pipe.base.connectionTypes
as cT
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
126 crosstalkSources = cT.PrerequisiteInput(
127 name=
"isrOverscanCorrected",
128 doc=
"Overscan corrected input images.",
129 storageClass=
"Exposure",
130 dimensions=[
"instrument",
"exposure",
"detector"],
133 lookupFunction=crosstalkSourceLookup,
136 bias = cT.PrerequisiteInput(
138 doc=
"Input bias calibration.",
139 storageClass=
"ExposureF",
140 dimensions=[
"instrument",
"detector"],
143 dark = cT.PrerequisiteInput(
145 doc=
"Input dark calibration.",
146 storageClass=
"ExposureF",
147 dimensions=[
"instrument",
"detector"],
150 flat = cT.PrerequisiteInput(
152 doc=
"Input flat calibration.",
153 storageClass=
"ExposureF",
154 dimensions=[
"instrument",
"physical_filter",
"detector"],
157 ptc = cT.PrerequisiteInput(
159 doc=
"Input Photon Transfer Curve dataset",
160 storageClass=
"PhotonTransferCurveDataset",
161 dimensions=[
"instrument",
"detector"],
164 fringes = cT.PrerequisiteInput(
166 doc=
"Input fringe calibration.",
167 storageClass=
"ExposureF",
168 dimensions=[
"instrument",
"physical_filter",
"detector"],
172 strayLightData = cT.PrerequisiteInput(
174 doc=
"Input stray light calibration.",
175 storageClass=
"StrayLightData",
176 dimensions=[
"instrument",
"physical_filter",
"detector"],
180 bfKernel = cT.PrerequisiteInput(
182 doc=
"Input brighter-fatter kernel.",
183 storageClass=
"NumpyArray",
184 dimensions=[
"instrument"],
188 newBFKernel = cT.PrerequisiteInput(
189 name=
'brighterFatterKernel',
190 doc=
"Newer complete kernel + gain solutions.",
191 storageClass=
"BrighterFatterKernel",
192 dimensions=[
"instrument",
"detector"],
196 defects = cT.PrerequisiteInput(
198 doc=
"Input defect tables.",
199 storageClass=
"Defects",
200 dimensions=[
"instrument",
"detector"],
203 linearizer = cT.PrerequisiteInput(
205 storageClass=
"Linearizer",
206 doc=
"Linearity correction calibration.",
207 dimensions=[
"instrument",
"detector"],
211 opticsTransmission = cT.PrerequisiteInput(
212 name=
"transmission_optics",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the optics.",
215 dimensions=[
"instrument"],
218 filterTransmission = cT.PrerequisiteInput(
219 name=
"transmission_filter",
220 storageClass=
"TransmissionCurve",
221 doc=
"Transmission curve due to the filter.",
222 dimensions=[
"instrument",
"physical_filter"],
225 sensorTransmission = cT.PrerequisiteInput(
226 name=
"transmission_sensor",
227 storageClass=
"TransmissionCurve",
228 doc=
"Transmission curve due to the sensor.",
229 dimensions=[
"instrument",
"detector"],
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name=
"transmission_atmosphere",
234 storageClass=
"TransmissionCurve",
235 doc=
"Transmission curve due to the atmosphere.",
236 dimensions=[
"instrument"],
239 illumMaskedImage = cT.PrerequisiteInput(
241 doc=
"Input illumination correction.",
242 storageClass=
"MaskedImageF",
243 dimensions=[
"instrument",
"physical_filter",
"detector"],
247 outputExposure = cT.Output(
249 doc=
"Output ISR processed exposure.",
250 storageClass=
"Exposure",
251 dimensions=[
"instrument",
"exposure",
"detector"],
253 preInterpExposure = cT.Output(
254 name=
'preInterpISRCCD',
255 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
256 storageClass=
"ExposureF",
257 dimensions=[
"instrument",
"exposure",
"detector"],
259 outputOssThumbnail = cT.Output(
261 doc=
"Output Overscan-subtracted thumbnail image.",
262 storageClass=
"Thumbnail",
263 dimensions=[
"instrument",
"exposure",
"detector"],
265 outputFlattenedThumbnail = cT.Output(
266 name=
"FlattenedThumb",
267 doc=
"Output flat-corrected thumbnail image.",
268 storageClass=
"Thumbnail",
269 dimensions=[
"instrument",
"exposure",
"detector"],
275 if config.doBias
is not True:
276 self.prerequisiteInputs.discard(
"bias")
277 if config.doLinearize
is not True:
278 self.prerequisiteInputs.discard(
"linearizer")
279 if config.doCrosstalk
is not True:
280 self.inputs.discard(
"crosstalkSources")
281 self.prerequisiteInputs.discard(
"crosstalk")
282 if config.doBrighterFatter
is not True:
283 self.prerequisiteInputs.discard(
"bfKernel")
284 self.prerequisiteInputs.discard(
"newBFKernel")
285 if config.doDefect
is not True:
286 self.prerequisiteInputs.discard(
"defects")
287 if config.doDark
is not True:
288 self.prerequisiteInputs.discard(
"dark")
289 if config.doFlat
is not True:
290 self.prerequisiteInputs.discard(
"flat")
291 if config.doFringe
is not True:
292 self.prerequisiteInputs.discard(
"fringe")
293 if config.doStrayLight
is not True:
294 self.prerequisiteInputs.discard(
"strayLightData")
295 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
296 self.prerequisiteInputs.discard(
"ptc")
297 if config.doAttachTransmissionCurve
is not True:
298 self.prerequisiteInputs.discard(
"opticsTransmission")
299 self.prerequisiteInputs.discard(
"filterTransmission")
300 self.prerequisiteInputs.discard(
"sensorTransmission")
301 self.prerequisiteInputs.discard(
"atmosphereTransmission")
302 if config.doUseOpticsTransmission
is not True:
303 self.prerequisiteInputs.discard(
"opticsTransmission")
304 if config.doUseFilterTransmission
is not True:
305 self.prerequisiteInputs.discard(
"filterTransmission")
306 if config.doUseSensorTransmission
is not True:
307 self.prerequisiteInputs.discard(
"sensorTransmission")
308 if config.doUseAtmosphereTransmission
is not True:
309 self.prerequisiteInputs.discard(
"atmosphereTransmission")
310 if config.doIlluminationCorrection
is not True:
311 self.prerequisiteInputs.discard(
"illumMaskedImage")
313 if config.doWrite
is not True:
314 self.outputs.discard(
"outputExposure")
315 self.outputs.discard(
"preInterpExposure")
316 self.outputs.discard(
"outputFlattenedThumbnail")
317 self.outputs.discard(
"outputOssThumbnail")
318 if config.doSaveInterpPixels
is not True:
319 self.outputs.discard(
"preInterpExposure")
320 if config.qa.doThumbnailOss
is not True:
321 self.outputs.discard(
"outputOssThumbnail")
322 if config.qa.doThumbnailFlattened
is not True:
323 self.outputs.discard(
"outputFlattenedThumbnail")
327 pipelineConnections=IsrTaskConnections):
328 """Configuration parameters for IsrTask.
330 Items are grouped in the order in which they are executed by the task.
332 datasetType = pexConfig.Field(
334 doc=
"Dataset type for input data; users will typically leave this alone, "
335 "but camera-specific ISR tasks will override it",
339 fallbackFilterName = pexConfig.Field(
341 doc=
"Fallback default filter name for calibrations.",
344 useFallbackDate = pexConfig.Field(
346 doc=
"Pass observation date when using fallback filter.",
349 expectWcs = pexConfig.Field(
352 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
354 fwhm = pexConfig.Field(
356 doc=
"FWHM of PSF in arcseconds.",
359 qa = pexConfig.ConfigField(
361 doc=
"QA related configuration options.",
365 doConvertIntToFloat = pexConfig.Field(
367 doc=
"Convert integer raw images to floating point values?",
372 doSaturation = pexConfig.Field(
374 doc=
"Mask saturated pixels? NB: this is totally independent of the"
375 " interpolation option - this is ONLY setting the bits in the mask."
376 " To have them interpolated make sure doSaturationInterpolation=True",
379 saturatedMaskName = pexConfig.Field(
381 doc=
"Name of mask plane to use in saturation detection and interpolation",
384 saturation = pexConfig.Field(
386 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387 default=float(
"NaN"),
389 growSaturationFootprintSize = pexConfig.Field(
391 doc=
"Number of pixels by which to grow the saturation footprints",
396 doSuspect = pexConfig.Field(
398 doc=
"Mask suspect pixels?",
401 suspectMaskName = pexConfig.Field(
403 doc=
"Name of mask plane to use for suspect pixels",
406 numEdgeSuspect = pexConfig.Field(
408 doc=
"Number of edge pixels to be flagged as untrustworthy.",
411 edgeMaskLevel = pexConfig.ChoiceField(
413 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
416 'DETECTOR':
'Mask only the edges of the full detector.',
417 'AMP':
'Mask edges of each amplifier.',
422 doSetBadRegions = pexConfig.Field(
424 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
427 badStatistic = pexConfig.ChoiceField(
429 doc=
"How to estimate the average value for BAD regions.",
432 "MEANCLIP":
"Correct using the (clipped) mean of good data",
433 "MEDIAN":
"Correct using the median of the good data",
438 doOverscan = pexConfig.Field(
440 doc=
"Do overscan subtraction?",
443 overscan = pexConfig.ConfigurableField(
444 target=OverscanCorrectionTask,
445 doc=
"Overscan subtraction task for image segments.",
448 overscanFitType = pexConfig.ChoiceField(
450 doc=
"The method for fitting the overscan bias level.",
453 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
454 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
455 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
456 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
457 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
458 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
459 "MEAN":
"Correct using the mean of the overscan region",
460 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
461 "MEDIAN":
"Correct using the median of the overscan region",
462 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
464 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
465 " This option will no longer be used, and will be removed after v20.")
467 overscanOrder = pexConfig.Field(
469 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
470 "or number of spline knots if overscan fit type is a spline."),
472 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
473 " This option will no longer be used, and will be removed after v20.")
475 overscanNumSigmaClip = pexConfig.Field(
477 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
479 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
480 " This option will no longer be used, and will be removed after v20.")
482 overscanIsInt = pexConfig.Field(
484 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485 " and overscan.FitType=MEDIAN_PER_ROW.",
487 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
488 " This option will no longer be used, and will be removed after v20.")
491 overscanNumLeadingColumnsToSkip = pexConfig.Field(
493 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
496 overscanNumTrailingColumnsToSkip = pexConfig.Field(
498 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
501 overscanMaxDev = pexConfig.Field(
503 doc=
"Maximum deviation from the median for overscan",
504 default=1000.0, check=
lambda x: x > 0
506 overscanBiasJump = pexConfig.Field(
508 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
511 overscanBiasJumpKeyword = pexConfig.Field(
513 doc=
"Header keyword containing information about devices.",
514 default=
"NO_SUCH_KEY",
516 overscanBiasJumpDevices = pexConfig.ListField(
518 doc=
"List of devices that need piecewise overscan correction.",
521 overscanBiasJumpLocation = pexConfig.Field(
523 doc=
"Location of bias jump along y-axis.",
528 doAssembleCcd = pexConfig.Field(
531 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
533 assembleCcd = pexConfig.ConfigurableField(
534 target=AssembleCcdTask,
535 doc=
"CCD assembly task",
539 doAssembleIsrExposures = pexConfig.Field(
542 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
544 doTrimToMatchCalib = pexConfig.Field(
547 doc=
"Trim raw data to match calibration bounding boxes?"
551 doBias = pexConfig.Field(
553 doc=
"Apply bias frame correction?",
556 biasDataProductName = pexConfig.Field(
558 doc=
"Name of the bias data product",
561 doBiasBeforeOverscan = pexConfig.Field(
563 doc=
"Reverse order of overscan and bias correction.",
568 doVariance = pexConfig.Field(
570 doc=
"Calculate variance?",
573 gain = pexConfig.Field(
575 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576 default=float(
"NaN"),
578 readNoise = pexConfig.Field(
580 doc=
"The read noise to use if no Detector is present in the Exposure",
583 doEmpiricalReadNoise = pexConfig.Field(
586 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
588 usePtcReadNoise = pexConfig.Field(
591 doc=
"Use readnoise values from the Photon Transfer Curve?"
594 doLinearize = pexConfig.Field(
596 doc=
"Correct for nonlinearity of the detector's response?",
601 doCrosstalk = pexConfig.Field(
603 doc=
"Apply intra-CCD crosstalk correction?",
606 doCrosstalkBeforeAssemble = pexConfig.Field(
608 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
611 crosstalk = pexConfig.ConfigurableField(
612 target=CrosstalkTask,
613 doc=
"Intra-CCD crosstalk correction",
617 doDefect = pexConfig.Field(
619 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
622 doNanMasking = pexConfig.Field(
624 doc=
"Mask non-finite (NAN, inf) pixels?",
627 doWidenSaturationTrails = pexConfig.Field(
629 doc=
"Widen bleed trails based on their width?",
634 doBrighterFatter = pexConfig.Field(
637 doc=
"Apply the brighter-fatter correction?"
639 brighterFatterLevel = pexConfig.ChoiceField(
642 doc=
"The level at which to correct for brighter-fatter.",
644 "AMP":
"Every amplifier treated separately.",
645 "DETECTOR":
"One kernel per detector",
648 brighterFatterMaxIter = pexConfig.Field(
651 doc=
"Maximum number of iterations for the brighter-fatter correction"
653 brighterFatterThreshold = pexConfig.Field(
656 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
657 "absolute value of the difference between the current corrected image and the one "
658 "from the previous iteration summed over all the pixels."
660 brighterFatterApplyGain = pexConfig.Field(
663 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
665 brighterFatterMaskListToInterpolate = pexConfig.ListField(
667 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
669 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
671 brighterFatterMaskGrowSize = pexConfig.Field(
674 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
675 "when brighter-fatter correction is applied."
679 doDark = pexConfig.Field(
681 doc=
"Apply dark frame correction?",
684 darkDataProductName = pexConfig.Field(
686 doc=
"Name of the dark data product",
691 doStrayLight = pexConfig.Field(
693 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
696 strayLight = pexConfig.ConfigurableField(
697 target=StrayLightTask,
698 doc=
"y-band stray light correction"
702 doFlat = pexConfig.Field(
704 doc=
"Apply flat field correction?",
707 flatDataProductName = pexConfig.Field(
709 doc=
"Name of the flat data product",
712 flatScalingType = pexConfig.ChoiceField(
714 doc=
"The method for scaling the flat on the fly.",
717 "USER":
"Scale by flatUserScale",
718 "MEAN":
"Scale by the inverse of the mean",
719 "MEDIAN":
"Scale by the inverse of the median",
722 flatUserScale = pexConfig.Field(
724 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
727 doTweakFlat = pexConfig.Field(
729 doc=
"Tweak flats to match observed amplifier ratios?",
734 doApplyGains = pexConfig.Field(
736 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
739 usePtcGains = pexConfig.Field(
741 doc=
"Use the gain values from the Photon Transfer Curve?",
744 normalizeGains = pexConfig.Field(
746 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
751 doFringe = pexConfig.Field(
753 doc=
"Apply fringe correction?",
756 fringe = pexConfig.ConfigurableField(
758 doc=
"Fringe subtraction task",
760 fringeAfterFlat = pexConfig.Field(
762 doc=
"Do fringe subtraction after flat-fielding?",
767 doMeasureBackground = pexConfig.Field(
769 doc=
"Measure the background level on the reduced image?",
774 doCameraSpecificMasking = pexConfig.Field(
776 doc=
"Mask camera-specific bad regions?",
779 masking = pexConfig.ConfigurableField(
786 doInterpolate = pexConfig.Field(
788 doc=
"Interpolate masked pixels?",
791 doSaturationInterpolation = pexConfig.Field(
793 doc=
"Perform interpolation over pixels masked as saturated?"
794 " NB: This is independent of doSaturation; if that is False this plane"
795 " will likely be blank, resulting in a no-op here.",
798 doNanInterpolation = pexConfig.Field(
800 doc=
"Perform interpolation over pixels masked as NaN?"
801 " NB: This is independent of doNanMasking; if that is False this plane"
802 " will likely be blank, resulting in a no-op here.",
805 doNanInterpAfterFlat = pexConfig.Field(
807 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
808 "also have to interpolate them before flat-fielding."),
811 maskListToInterpolate = pexConfig.ListField(
813 doc=
"List of mask planes that should be interpolated.",
814 default=[
'SAT',
'BAD'],
816 doSaveInterpPixels = pexConfig.Field(
818 doc=
"Save a copy of the pre-interpolated pixel values?",
823 fluxMag0T1 = pexConfig.DictField(
826 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
827 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
830 defaultFluxMag0T1 = pexConfig.Field(
832 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
833 default=pow(10.0, 0.4*28.0)
837 doVignette = pexConfig.Field(
839 doc=
"Apply vignetting parameters?",
842 vignette = pexConfig.ConfigurableField(
844 doc=
"Vignetting task.",
848 doAttachTransmissionCurve = pexConfig.Field(
851 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
853 doUseOpticsTransmission = pexConfig.Field(
856 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
858 doUseFilterTransmission = pexConfig.Field(
861 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
863 doUseSensorTransmission = pexConfig.Field(
866 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
868 doUseAtmosphereTransmission = pexConfig.Field(
871 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
875 doIlluminationCorrection = pexConfig.Field(
878 doc=
"Perform illumination correction?"
880 illuminationCorrectionDataProductName = pexConfig.Field(
882 doc=
"Name of the illumination correction data product.",
885 illumScale = pexConfig.Field(
887 doc=
"Scale factor for the illumination correction.",
890 illumFilters = pexConfig.ListField(
893 doc=
"Only perform illumination correction for these filters."
897 doWrite = pexConfig.Field(
899 doc=
"Persist postISRCCD?",
906 raise ValueError(
"You may not specify both doFlat and doApplyGains")
908 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
917 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
918 """Apply common instrument signature correction algorithms to a raw frame.
920 The process for correcting imaging data is very similar from
921 camera to camera. This task provides a vanilla implementation of
922 doing these corrections, including the ability to turn certain
923 corrections off if they are not needed. The inputs to the primary
924 method, `run()`, are a raw exposure to be corrected and the
925 calibration data products. The raw input is a single chip sized
926 mosaic of all amps including overscans and other non-science
927 pixels. The method `runDataRef()` identifies and defines the
928 calibration data products, and is intended for use by a
929 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
930 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
931 subclassed for different camera, although the most camera specific
932 methods have been split into subtasks that can be redirected
935 The __init__ method sets up the subtasks for ISR processing, using
936 the defaults from `lsst.ip.isr`.
941 Positional arguments passed to the Task constructor. None used at this time.
942 kwargs : `dict`, optional
943 Keyword arguments passed on to the Task constructor. None used at this time.
945 ConfigClass = IsrTaskConfig
950 self.makeSubtask(
"assembleCcd")
951 self.makeSubtask(
"crosstalk")
952 self.makeSubtask(
"strayLight")
953 self.makeSubtask(
"fringe")
954 self.makeSubtask(
"masking")
955 self.makeSubtask(
"overscan")
956 self.makeSubtask(
"vignette")
959 inputs = butlerQC.get(inputRefs)
962 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
963 except Exception
as e:
964 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
967 inputs[
'isGen3'] =
True
969 detector = inputs[
'ccdExposure'].getDetector()
971 if self.config.doCrosstalk
is True:
974 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
975 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
976 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
978 coeffVector = (self.config.crosstalk.crosstalkValues
979 if self.config.crosstalk.useConfigCoefficients
else None)
980 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
981 inputs[
'crosstalk'] = crosstalkCalib
982 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
983 if 'crosstalkSources' not in inputs:
984 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
987 if 'linearizer' in inputs:
988 if isinstance(inputs[
'linearizer'], dict):
990 linearizer.fromYaml(inputs[
'linearizer'])
991 self.log.warn(
"Dictionary linearizers will be deprecated in DM-28741.")
992 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
996 self.log.warn(
"Bare lookup table linearizers will be deprecated in DM-28741.")
998 linearizer = inputs[
'linearizer']
999 linearizer.log = self.log
1000 inputs[
'linearizer'] = linearizer
1003 self.log.warn(
"Constructing linearizer from cameraGeom information.")
1005 if self.config.doDefect
is True:
1006 if "defects" in inputs
and inputs[
'defects']
is not None:
1009 if not isinstance(inputs[
"defects"], Defects):
1010 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1014 if self.config.doBrighterFatter:
1015 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1016 if brighterFatterKernel
is None:
1017 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1019 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1021 detName = detector.getName()
1022 level = brighterFatterKernel.level
1025 inputs[
'bfGains'] = brighterFatterKernel.gain
1026 if self.config.brighterFatterLevel ==
'DETECTOR':
1027 if level ==
'DETECTOR':
1028 if detName
in brighterFatterKernel.detKernels:
1029 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1031 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1032 elif level ==
'AMP':
1033 self.log.warn(
"Making DETECTOR level kernel from AMP based brighter fatter kernels.")
1034 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1035 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1036 elif self.config.brighterFatterLevel ==
'AMP':
1037 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1039 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1040 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
1041 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1043 assembler=self.assembleCcd
1044 if self.config.doAssembleIsrExposures
else None)
1046 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1048 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1049 if 'strayLightData' not in inputs:
1050 inputs[
'strayLightData'] =
None
1052 outputs = self.
runrun(**inputs)
1053 butlerQC.put(outputs, outputRefs)
1056 """Retrieve necessary frames for instrument signature removal.
1058 Pre-fetching all required ISR data products limits the IO
1059 required by the ISR. Any conflict between the calibration data
1060 available and that needed for ISR is also detected prior to
1061 doing processing, allowing it to fail quickly.
1065 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1066 Butler reference of the detector data to be processed
1067 rawExposure : `afw.image.Exposure`
1068 The raw exposure that will later be corrected with the
1069 retrieved calibration data; should not be modified in this
1074 result : `lsst.pipe.base.Struct`
1075 Result struct with components (which may be `None`):
1076 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1077 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1078 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1079 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1080 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1081 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1082 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1083 - ``fringes``: `lsst.pipe.base.Struct` with components:
1084 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1085 - ``seed``: random seed derived from the ccdExposureId for random
1086 number generator (`uint32`).
1087 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1088 A ``TransmissionCurve`` that represents the throughput of the optics,
1089 to be evaluated in focal-plane coordinates.
1090 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1091 A ``TransmissionCurve`` that represents the throughput of the filter
1092 itself, to be evaluated in focal-plane coordinates.
1093 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1094 A ``TransmissionCurve`` that represents the throughput of the sensor
1095 itself, to be evaluated in post-assembly trimmed detector coordinates.
1096 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1097 A ``TransmissionCurve`` that represents the throughput of the
1098 atmosphere, assumed to be spatially constant.
1099 - ``strayLightData`` : `object`
1100 An opaque object containing calibration information for
1101 stray-light correction. If `None`, no correction will be
1103 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1107 NotImplementedError :
1108 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1111 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1112 dateObs = dateObs.toPython().isoformat()
1113 except RuntimeError:
1114 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1117 ccd = rawExposure.getDetector()
1118 filterLabel = rawExposure.getFilterLabel()
1119 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1120 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1121 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1122 if self.config.doBias
else None)
1124 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1126 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1127 linearizer.log = self.log
1128 if isinstance(linearizer, numpy.ndarray):
1131 crosstalkCalib =
None
1132 if self.config.doCrosstalk:
1134 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1136 coeffVector = (self.config.crosstalk.crosstalkValues
1137 if self.config.crosstalk.useConfigCoefficients
else None)
1138 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1139 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1140 if self.config.doCrosstalk
else None)
1142 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1143 if self.config.doDark
else None)
1144 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1146 if self.config.doFlat
else None)
1148 brighterFatterKernel =
None
1149 brighterFatterGains =
None
1150 if self.config.doBrighterFatter
is True:
1155 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1156 brighterFatterGains = brighterFatterKernel.gain
1157 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1160 brighterFatterKernel = dataRef.get(
"bfKernel")
1161 self.log.info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1163 brighterFatterKernel =
None
1164 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1167 if self.config.brighterFatterLevel ==
'DETECTOR':
1168 if brighterFatterKernel.detKernels:
1169 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1171 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1174 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1176 defectList = (dataRef.get(
"defects")
1177 if self.config.doDefect
else None)
1178 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1179 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1180 if self.config.doAssembleIsrExposures
else None)
1181 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1182 else pipeBase.Struct(fringes=
None))
1184 if self.config.doAttachTransmissionCurve:
1185 opticsTransmission = (dataRef.get(
"transmission_optics")
1186 if self.config.doUseOpticsTransmission
else None)
1187 filterTransmission = (dataRef.get(
"transmission_filter")
1188 if self.config.doUseFilterTransmission
else None)
1189 sensorTransmission = (dataRef.get(
"transmission_sensor")
1190 if self.config.doUseSensorTransmission
else None)
1191 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1192 if self.config.doUseAtmosphereTransmission
else None)
1194 opticsTransmission =
None
1195 filterTransmission =
None
1196 sensorTransmission =
None
1197 atmosphereTransmission =
None
1199 if self.config.doStrayLight:
1200 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1202 strayLightData =
None
1205 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1206 if (self.config.doIlluminationCorrection
1207 and physicalFilter
in self.config.illumFilters)
1211 return pipeBase.Struct(bias=biasExposure,
1212 linearizer=linearizer,
1213 crosstalk=crosstalkCalib,
1214 crosstalkSources=crosstalkSources,
1217 bfKernel=brighterFatterKernel,
1218 bfGains=brighterFatterGains,
1220 fringes=fringeStruct,
1221 opticsTransmission=opticsTransmission,
1222 filterTransmission=filterTransmission,
1223 sensorTransmission=sensorTransmission,
1224 atmosphereTransmission=atmosphereTransmission,
1225 strayLightData=strayLightData,
1226 illumMaskedImage=illumMaskedImage
1229 @pipeBase.timeMethod
1230 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1231 crosstalk=None, crosstalkSources=None,
1232 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1233 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1234 sensorTransmission=
None, atmosphereTransmission=
None,
1235 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1238 """Perform instrument signature removal on an exposure.
1240 Steps included in the ISR processing, in order performed, are:
1241 - saturation and suspect pixel masking
1242 - overscan subtraction
1243 - CCD assembly of individual amplifiers
1245 - variance image construction
1246 - linearization of non-linear response
1248 - brighter-fatter correction
1251 - stray light subtraction
1253 - masking of known defects and camera specific features
1254 - vignette calculation
1255 - appending transmission curve and distortion model
1259 ccdExposure : `lsst.afw.image.Exposure`
1260 The raw exposure that is to be run through ISR. The
1261 exposure is modified by this method.
1262 camera : `lsst.afw.cameraGeom.Camera`, optional
1263 The camera geometry for this exposure. Required if ``isGen3`` is
1264 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1265 ``flat`` does not have an associated detector.
1266 bias : `lsst.afw.image.Exposure`, optional
1267 Bias calibration frame.
1268 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1269 Functor for linearization.
1270 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1271 Calibration for crosstalk.
1272 crosstalkSources : `list`, optional
1273 List of possible crosstalk sources.
1274 dark : `lsst.afw.image.Exposure`, optional
1275 Dark calibration frame.
1276 flat : `lsst.afw.image.Exposure`, optional
1277 Flat calibration frame.
1278 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1279 Photon transfer curve dataset, with, e.g., gains
1281 bfKernel : `numpy.ndarray`, optional
1282 Brighter-fatter kernel.
1283 bfGains : `dict` of `float`, optional
1284 Gains used to override the detector's nominal gains for the
1285 brighter-fatter correction. A dict keyed by amplifier name for
1286 the detector in question.
1287 defects : `lsst.ip.isr.Defects`, optional
1289 fringes : `lsst.pipe.base.Struct`, optional
1290 Struct containing the fringe correction data, with
1292 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1293 - ``seed``: random seed derived from the ccdExposureId for random
1294 number generator (`uint32`)
1295 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1296 A ``TransmissionCurve`` that represents the throughput of the optics,
1297 to be evaluated in focal-plane coordinates.
1298 filterTransmission : `lsst.afw.image.TransmissionCurve`
1299 A ``TransmissionCurve`` that represents the throughput of the filter
1300 itself, to be evaluated in focal-plane coordinates.
1301 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1302 A ``TransmissionCurve`` that represents the throughput of the sensor
1303 itself, to be evaluated in post-assembly trimmed detector coordinates.
1304 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1305 A ``TransmissionCurve`` that represents the throughput of the
1306 atmosphere, assumed to be spatially constant.
1307 detectorNum : `int`, optional
1308 The integer number for the detector to process.
1309 isGen3 : bool, optional
1310 Flag this call to run() as using the Gen3 butler environment.
1311 strayLightData : `object`, optional
1312 Opaque object containing calibration information for stray-light
1313 correction. If `None`, no correction will be performed.
1314 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1315 Illumination correction image.
1319 result : `lsst.pipe.base.Struct`
1320 Result struct with component:
1321 - ``exposure`` : `afw.image.Exposure`
1322 The fully ISR corrected exposure.
1323 - ``outputExposure`` : `afw.image.Exposure`
1324 An alias for `exposure`
1325 - ``ossThumb`` : `numpy.ndarray`
1326 Thumbnail image of the exposure after overscan subtraction.
1327 - ``flattenedThumb`` : `numpy.ndarray`
1328 Thumbnail image of the exposure after flat-field correction.
1333 Raised if a configuration option is set to True, but the
1334 required calibration data has not been specified.
1338 The current processed exposure can be viewed by setting the
1339 appropriate lsstDebug entries in the `debug.display`
1340 dictionary. The names of these entries correspond to some of
1341 the IsrTaskConfig Boolean options, with the value denoting the
1342 frame to use. The exposure is shown inside the matching
1343 option check and after the processing of that step has
1344 finished. The steps with debug points are:
1355 In addition, setting the "postISRCCD" entry displays the
1356 exposure after all ISR processing has finished.
1364 if detectorNum
is None:
1365 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1367 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1368 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1369 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1370 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1372 if isinstance(ccdExposure, ButlerDataRef):
1373 return self.
runDataRefrunDataRef(ccdExposure)
1375 ccd = ccdExposure.getDetector()
1376 filterLabel = ccdExposure.getFilterLabel()
1377 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1380 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1381 ccd = [
FakeAmp(ccdExposure, self.config)]
1384 if self.config.doBias
and bias
is None:
1385 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1386 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1387 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1388 if self.config.doBrighterFatter
and bfKernel
is None:
1389 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1390 if self.config.doDark
and dark
is None:
1391 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1392 if self.config.doFlat
and flat
is None:
1393 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1394 if self.config.doDefect
and defects
is None:
1395 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1396 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1397 and fringes.fringes
is None):
1402 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1403 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1404 and illumMaskedImage
is None):
1405 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1408 if self.config.doConvertIntToFloat:
1409 self.log.info(
"Converting exposure to floating point values.")
1412 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1413 self.log.info(
"Applying bias correction.")
1414 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1415 trimToFit=self.config.doTrimToMatchCalib)
1416 self.
debugViewdebugView(ccdExposure,
"doBias")
1422 if ccdExposure.getBBox().contains(amp.getBBox()):
1424 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1426 if self.config.doOverscan
and not badAmp:
1429 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1430 if overscanResults
is not None and \
1431 self.config.qa
is not None and self.config.qa.saveStats
is True:
1432 if isinstance(overscanResults.overscanFit, float):
1433 qaMedian = overscanResults.overscanFit
1434 qaStdev = float(
"NaN")
1436 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1437 afwMath.MEDIAN | afwMath.STDEVCLIP)
1438 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1439 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1441 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1442 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1443 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1444 amp.getName(), qaMedian, qaStdev)
1447 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1448 afwMath.MEDIAN | afwMath.STDEVCLIP)
1449 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1450 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1452 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1453 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1454 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1455 amp.getName(), qaMedianAfter, qaStdevAfter)
1457 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1460 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1461 overscanResults =
None
1463 overscans.append(overscanResults
if overscanResults
is not None else None)
1465 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1467 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1468 self.log.info(
"Applying crosstalk correction.")
1469 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1470 crosstalkSources=crosstalkSources, camera=camera)
1471 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1473 if self.config.doAssembleCcd:
1474 self.log.info(
"Assembling CCD from amplifiers.")
1475 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1477 if self.config.expectWcs
and not ccdExposure.getWcs():
1478 self.log.warn(
"No WCS found in input exposure.")
1479 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1482 if self.config.qa.doThumbnailOss:
1483 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1485 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1486 self.log.info(
"Applying bias correction.")
1487 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1488 trimToFit=self.config.doTrimToMatchCalib)
1489 self.
debugViewdebugView(ccdExposure,
"doBias")
1491 if self.config.doVariance:
1492 for amp, overscanResults
in zip(ccd, overscans):
1493 if ccdExposure.getBBox().contains(amp.getBBox()):
1494 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1495 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1496 if overscanResults
is not None:
1498 overscanImage=overscanResults.overscanImage,
1504 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1505 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1506 afwMath.MEDIAN | afwMath.STDEVCLIP)
1507 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1508 qaStats.getValue(afwMath.MEDIAN))
1509 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1510 qaStats.getValue(afwMath.STDEVCLIP))
1511 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1512 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1513 qaStats.getValue(afwMath.STDEVCLIP))
1516 self.log.info(
"Applying linearizer.")
1517 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1518 detector=ccd, log=self.log)
1520 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1521 self.log.info(
"Applying crosstalk correction.")
1522 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1523 crosstalkSources=crosstalkSources, isTrimmed=
True)
1524 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1528 if self.config.doDefect:
1529 self.log.info(
"Masking defects.")
1530 self.
maskDefectmaskDefect(ccdExposure, defects)
1532 if self.config.numEdgeSuspect > 0:
1533 self.log.info(
"Masking edges as SUSPECT.")
1534 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1535 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1537 if self.config.doNanMasking:
1538 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1539 self.
maskNanmaskNan(ccdExposure)
1541 if self.config.doWidenSaturationTrails:
1542 self.log.info(
"Widening saturation trails.")
1543 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1545 if self.config.doCameraSpecificMasking:
1546 self.log.info(
"Masking regions for camera specific reasons.")
1547 self.masking.
run(ccdExposure)
1549 if self.config.doBrighterFatter:
1558 interpExp = ccdExposure.clone()
1559 with self.
flatContextflatContext(interpExp, flat, dark):
1560 isrFunctions.interpolateFromMask(
1561 maskedImage=interpExp.getMaskedImage(),
1562 fwhm=self.config.fwhm,
1563 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1564 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1566 bfExp = interpExp.clone()
1568 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1569 type(bfKernel), type(bfGains))
1570 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1571 self.config.brighterFatterMaxIter,
1572 self.config.brighterFatterThreshold,
1573 self.config.brighterFatterApplyGain,
1575 if bfResults[1] == self.config.brighterFatterMaxIter:
1576 self.log.warn(
"Brighter-fatter correction did not converge, final difference %f.",
1579 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1581 image = ccdExposure.getMaskedImage().getImage()
1582 bfCorr = bfExp.getMaskedImage().getImage()
1583 bfCorr -= interpExp.getMaskedImage().getImage()
1592 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1593 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1596 if self.config.brighterFatterMaskGrowSize > 0:
1597 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1598 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1599 isrFunctions.growMasks(ccdExposure.getMask(),
1600 radius=self.config.brighterFatterMaskGrowSize,
1601 maskNameList=maskPlane,
1602 maskValue=maskPlane)
1604 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1606 if self.config.doDark:
1607 self.log.info(
"Applying dark correction.")
1609 self.
debugViewdebugView(ccdExposure,
"doDark")
1611 if self.config.doFringe
and not self.config.fringeAfterFlat:
1612 self.log.info(
"Applying fringe correction before flat.")
1613 self.fringe.
run(ccdExposure, **fringes.getDict())
1614 self.
debugViewdebugView(ccdExposure,
"doFringe")
1616 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1617 self.log.info(
"Checking strayLight correction.")
1618 self.strayLight.
run(ccdExposure, strayLightData)
1619 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1621 if self.config.doFlat:
1622 self.log.info(
"Applying flat correction.")
1624 self.
debugViewdebugView(ccdExposure,
"doFlat")
1626 if self.config.doApplyGains:
1627 self.log.info(
"Applying gain correction instead of flat.")
1628 if self.config.usePtcGains:
1629 self.log.info(
"Using gains from the Photon Transfer Curve.")
1630 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1633 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1635 if self.config.doFringe
and self.config.fringeAfterFlat:
1636 self.log.info(
"Applying fringe correction after flat.")
1637 self.fringe.
run(ccdExposure, **fringes.getDict())
1639 if self.config.doVignette:
1640 self.log.info(
"Constructing Vignette polygon.")
1643 if self.config.vignette.doWriteVignettePolygon:
1646 if self.config.doAttachTransmissionCurve:
1647 self.log.info(
"Adding transmission curves.")
1648 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1649 filterTransmission=filterTransmission,
1650 sensorTransmission=sensorTransmission,
1651 atmosphereTransmission=atmosphereTransmission)
1653 flattenedThumb =
None
1654 if self.config.qa.doThumbnailFlattened:
1655 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1657 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1658 self.log.info(
"Performing illumination correction.")
1659 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1660 illumMaskedImage, illumScale=self.config.illumScale,
1661 trimToFit=self.config.doTrimToMatchCalib)
1664 if self.config.doSaveInterpPixels:
1665 preInterpExp = ccdExposure.clone()
1680 if self.config.doSetBadRegions:
1681 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1682 if badPixelCount > 0:
1683 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1685 if self.config.doInterpolate:
1686 self.log.info(
"Interpolating masked pixels.")
1687 isrFunctions.interpolateFromMask(
1688 maskedImage=ccdExposure.getMaskedImage(),
1689 fwhm=self.config.fwhm,
1690 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1691 maskNameList=list(self.config.maskListToInterpolate)
1696 if self.config.doMeasureBackground:
1697 self.log.info(
"Measuring background level.")
1700 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1702 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1703 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1704 afwMath.MEDIAN | afwMath.STDEVCLIP)
1705 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1706 qaStats.getValue(afwMath.MEDIAN))
1707 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1708 qaStats.getValue(afwMath.STDEVCLIP))
1709 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1710 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1711 qaStats.getValue(afwMath.STDEVCLIP))
1713 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1715 return pipeBase.Struct(
1716 exposure=ccdExposure,
1718 flattenedThumb=flattenedThumb,
1720 preInterpExposure=preInterpExp,
1721 outputExposure=ccdExposure,
1722 outputOssThumbnail=ossThumb,
1723 outputFlattenedThumbnail=flattenedThumb,
1726 @pipeBase.timeMethod
1728 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1730 This method contains the `CmdLineTask` interface to the ISR
1731 processing. All IO is handled here, freeing the `run()` method
1732 to manage only pixel-level calculations. The steps performed
1734 - Read in necessary detrending/isr/calibration data.
1735 - Process raw exposure in `run()`.
1736 - Persist the ISR-corrected exposure as "postISRCCD" if
1737 config.doWrite=True.
1741 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1742 DataRef of the detector data to be processed
1746 result : `lsst.pipe.base.Struct`
1747 Result struct with component:
1748 - ``exposure`` : `afw.image.Exposure`
1749 The fully ISR corrected exposure.
1754 Raised if a configuration option is set to True, but the
1755 required calibration data does not exist.
1758 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1760 ccdExposure = sensorRef.get(self.config.datasetType)
1762 camera = sensorRef.get(
"camera")
1763 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1765 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1767 if self.config.doWrite:
1768 sensorRef.put(result.exposure,
"postISRCCD")
1769 if result.preInterpExposure
is not None:
1770 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1771 if result.ossThumb
is not None:
1772 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1773 if result.flattenedThumb
is not None:
1774 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1779 """Retrieve a calibration dataset for removing instrument signature.
1784 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1785 DataRef of the detector data to find calibration datasets
1788 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1789 dateObs : `str`, optional
1790 Date of the observation. Used to correct butler failures
1791 when using fallback filters.
1793 If True, disable butler proxies to enable error handling
1794 within this routine.
1798 exposure : `lsst.afw.image.Exposure`
1799 Requested calibration frame.
1804 Raised if no matching calibration frame can be found.
1807 exp = dataRef.get(datasetType, immediate=immediate)
1808 except Exception
as exc1:
1809 if not self.config.fallbackFilterName:
1810 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1812 if self.config.useFallbackDate
and dateObs:
1813 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1814 dateObs=dateObs, immediate=immediate)
1816 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1817 except Exception
as exc2:
1818 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1819 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1820 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1822 if self.config.doAssembleIsrExposures:
1823 exp = self.assembleCcd.assembleCcd(exp)
1827 """Ensure that the data returned by Butler is a fully constructed exposure.
1829 ISR requires exposure-level image data for historical reasons, so if we did
1830 not recieve that from Butler, construct it from what we have, modifying the
1835 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1836 `lsst.afw.image.ImageF`
1837 The input data structure obtained from Butler.
1838 camera : `lsst.afw.cameraGeom.camera`
1839 The camera associated with the image. Used to find the appropriate
1842 The detector this exposure should match.
1846 inputExp : `lsst.afw.image.Exposure`
1847 The re-constructed exposure, with appropriate detector parameters.
1852 Raised if the input data cannot be used to construct an exposure.
1854 if isinstance(inputExp, afwImage.DecoratedImageU):
1855 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1856 elif isinstance(inputExp, afwImage.ImageF):
1857 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1858 elif isinstance(inputExp, afwImage.MaskedImageF):
1859 inputExp = afwImage.makeExposure(inputExp)
1860 elif isinstance(inputExp, afwImage.Exposure):
1862 elif inputExp
is None:
1866 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1869 if inputExp.getDetector()
is None:
1870 inputExp.setDetector(camera[detectorNum])
1875 """Convert exposure image from uint16 to float.
1877 If the exposure does not need to be converted, the input is
1878 immediately returned. For exposures that are converted to use
1879 floating point pixels, the variance is set to unity and the
1884 exposure : `lsst.afw.image.Exposure`
1885 The raw exposure to be converted.
1889 newexposure : `lsst.afw.image.Exposure`
1890 The input ``exposure``, converted to floating point pixels.
1895 Raised if the exposure type cannot be converted to float.
1898 if isinstance(exposure, afwImage.ExposureF):
1900 self.log.debug(
"Exposure already of type float.")
1902 if not hasattr(exposure,
"convertF"):
1903 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1905 newexposure = exposure.convertF()
1906 newexposure.variance[:] = 1
1907 newexposure.mask[:] = 0x0
1912 """Identify bad amplifiers, saturated and suspect pixels.
1916 ccdExposure : `lsst.afw.image.Exposure`
1917 Input exposure to be masked.
1918 amp : `lsst.afw.table.AmpInfoCatalog`
1919 Catalog of parameters defining the amplifier on this
1921 defects : `lsst.ip.isr.Defects`
1922 List of defects. Used to determine if the entire
1928 If this is true, the entire amplifier area is covered by
1929 defects and unusable.
1932 maskedImage = ccdExposure.getMaskedImage()
1938 if defects
is not None:
1939 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1944 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1946 maskView = dataView.getMask()
1947 maskView |= maskView.getPlaneBitMask(
"BAD")
1954 if self.config.doSaturation
and not badAmp:
1955 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1956 if self.config.doSuspect
and not badAmp:
1957 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1958 if math.isfinite(self.config.saturation):
1959 limits.update({self.config.saturatedMaskName: self.config.saturation})
1961 for maskName, maskThreshold
in limits.items():
1962 if not math.isnan(maskThreshold):
1963 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1964 isrFunctions.makeThresholdMask(
1965 maskedImage=dataView,
1966 threshold=maskThreshold,
1972 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1974 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1975 self.config.suspectMaskName])
1976 if numpy.all(maskView.getArray() & maskVal > 0):
1978 maskView |= maskView.getPlaneBitMask(
"BAD")
1983 """Apply overscan correction in place.
1985 This method does initial pixel rejection of the overscan
1986 region. The overscan can also be optionally segmented to
1987 allow for discontinuous overscan responses to be fit
1988 separately. The actual overscan subtraction is performed by
1989 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1990 which is called here after the amplifier is preprocessed.
1994 ccdExposure : `lsst.afw.image.Exposure`
1995 Exposure to have overscan correction performed.
1996 amp : `lsst.afw.cameraGeom.Amplifer`
1997 The amplifier to consider while correcting the overscan.
2001 overscanResults : `lsst.pipe.base.Struct`
2002 Result struct with components:
2003 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2004 Value or fit subtracted from the amplifier image data.
2005 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2006 Value or fit subtracted from the overscan image data.
2007 - ``overscanImage`` : `lsst.afw.image.Image`
2008 Image of the overscan region with the overscan
2009 correction applied. This quantity is used to estimate
2010 the amplifier read noise empirically.
2015 Raised if the ``amp`` does not contain raw pixel information.
2019 lsst.ip.isr.isrFunctions.overscanCorrection
2021 if amp.getRawHorizontalOverscanBBox().isEmpty():
2022 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2025 statControl = afwMath.StatisticsControl()
2026 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2029 dataBBox = amp.getRawDataBBox()
2030 oscanBBox = amp.getRawHorizontalOverscanBBox()
2034 prescanBBox = amp.getRawPrescanBBox()
2035 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2036 dx0 += self.config.overscanNumLeadingColumnsToSkip
2037 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2039 dx0 += self.config.overscanNumTrailingColumnsToSkip
2040 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2046 if ((self.config.overscanBiasJump
2047 and self.config.overscanBiasJumpLocation)
2048 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2049 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2050 self.config.overscanBiasJumpDevices)):
2051 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2052 yLower = self.config.overscanBiasJumpLocation
2053 yUpper = dataBBox.getHeight() - yLower
2055 yUpper = self.config.overscanBiasJumpLocation
2056 yLower = dataBBox.getHeight() - yUpper
2074 oscanBBox.getHeight())))
2077 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2078 ampImage = ccdExposure.maskedImage[imageBBox]
2079 overscanImage = ccdExposure.maskedImage[overscanBBox]
2081 overscanArray = overscanImage.image.array
2082 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2083 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2084 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2086 statControl = afwMath.StatisticsControl()
2087 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2089 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2092 levelStat = afwMath.MEDIAN
2093 sigmaStat = afwMath.STDEVCLIP
2095 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2096 self.config.qa.flatness.nIter)
2097 metadata = ccdExposure.getMetadata()
2098 ampNum = amp.getName()
2100 if isinstance(overscanResults.overscanFit, float):
2101 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2102 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2104 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2105 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2106 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2108 return overscanResults
2111 """Set the variance plane using the gain and read noise
2113 The read noise is calculated from the ``overscanImage`` if the
2114 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2115 the value from the amplifier data is used.
2119 ampExposure : `lsst.afw.image.Exposure`
2120 Exposure to process.
2121 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2122 Amplifier detector data.
2123 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2124 Image of overscan, required only for empirical read noise.
2125 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2126 PTC dataset containing the gains and read noise.
2132 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2133 are ``True``, but ptcDataset is not provided.
2135 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2136 ``overscanImage`` is ``None``.
2140 lsst.ip.isr.isrFunctions.updateVariance
2142 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2143 if self.config.usePtcGains:
2144 if ptcDataset
is None:
2145 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2147 gain = ptcDataset.gain[amp.getName()]
2148 self.log.info(
"Using gain from Photon Transfer Curve.")
2150 gain = amp.getGain()
2152 if math.isnan(gain):
2154 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2157 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2158 amp.getName(), gain, patchedGain)
2161 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2162 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2164 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2165 stats = afwMath.StatisticsControl()
2166 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2167 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2168 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2169 amp.getName(), readNoise)
2170 elif self.config.usePtcReadNoise:
2171 if ptcDataset
is None:
2172 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2174 readNoise = ptcDataset.noise[amp.getName()]
2175 self.log.info(
"Using read noise from Photon Transfer Curve.")
2177 readNoise = amp.getReadNoise()
2179 isrFunctions.updateVariance(
2180 maskedImage=ampExposure.getMaskedImage(),
2182 readNoise=readNoise,
2186 """Apply dark correction in place.
2190 exposure : `lsst.afw.image.Exposure`
2191 Exposure to process.
2192 darkExposure : `lsst.afw.image.Exposure`
2193 Dark exposure of the same size as ``exposure``.
2194 invert : `Bool`, optional
2195 If True, re-add the dark to an already corrected image.
2200 Raised if either ``exposure`` or ``darkExposure`` do not
2201 have their dark time defined.
2205 lsst.ip.isr.isrFunctions.darkCorrection
2207 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2208 if math.isnan(expScale):
2209 raise RuntimeError(
"Exposure darktime is NAN.")
2210 if darkExposure.getInfo().getVisitInfo()
is not None \
2211 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2212 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2216 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2219 isrFunctions.darkCorrection(
2220 maskedImage=exposure.getMaskedImage(),
2221 darkMaskedImage=darkExposure.getMaskedImage(),
2223 darkScale=darkScale,
2225 trimToFit=self.config.doTrimToMatchCalib
2229 """Check if linearization is needed for the detector cameraGeom.
2231 Checks config.doLinearize and the linearity type of the first
2236 detector : `lsst.afw.cameraGeom.Detector`
2237 Detector to get linearity type from.
2241 doLinearize : `Bool`
2242 If True, linearization should be performed.
2244 return self.config.doLinearize
and \
2245 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2248 """Apply flat correction in place.
2252 exposure : `lsst.afw.image.Exposure`
2253 Exposure to process.
2254 flatExposure : `lsst.afw.image.Exposure`
2255 Flat exposure of the same size as ``exposure``.
2256 invert : `Bool`, optional
2257 If True, unflatten an already flattened image.
2261 lsst.ip.isr.isrFunctions.flatCorrection
2263 isrFunctions.flatCorrection(
2264 maskedImage=exposure.getMaskedImage(),
2265 flatMaskedImage=flatExposure.getMaskedImage(),
2266 scalingType=self.config.flatScalingType,
2267 userScale=self.config.flatUserScale,
2269 trimToFit=self.config.doTrimToMatchCalib
2273 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2277 exposure : `lsst.afw.image.Exposure`
2278 Exposure to process. Only the amplifier DataSec is processed.
2279 amp : `lsst.afw.table.AmpInfoCatalog`
2280 Amplifier detector data.
2284 lsst.ip.isr.isrFunctions.makeThresholdMask
2286 if not math.isnan(amp.getSaturation()):
2287 maskedImage = exposure.getMaskedImage()
2288 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2289 isrFunctions.makeThresholdMask(
2290 maskedImage=dataView,
2291 threshold=amp.getSaturation(),
2293 maskName=self.config.saturatedMaskName,
2297 """Interpolate over saturated pixels, in place.
2299 This method should be called after `saturationDetection`, to
2300 ensure that the saturated pixels have been identified in the
2301 SAT mask. It should also be called after `assembleCcd`, since
2302 saturated regions may cross amplifier boundaries.
2306 exposure : `lsst.afw.image.Exposure`
2307 Exposure to process.
2311 lsst.ip.isr.isrTask.saturationDetection
2312 lsst.ip.isr.isrFunctions.interpolateFromMask
2314 isrFunctions.interpolateFromMask(
2315 maskedImage=exposure.getMaskedImage(),
2316 fwhm=self.config.fwhm,
2317 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2318 maskNameList=list(self.config.saturatedMaskName),
2322 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2326 exposure : `lsst.afw.image.Exposure`
2327 Exposure to process. Only the amplifier DataSec is processed.
2328 amp : `lsst.afw.table.AmpInfoCatalog`
2329 Amplifier detector data.
2333 lsst.ip.isr.isrFunctions.makeThresholdMask
2337 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2338 This is intended to indicate pixels that may be affected by unknown systematics;
2339 for example if non-linearity corrections above a certain level are unstable
2340 then that would be a useful value for suspectLevel. A value of `nan` indicates
2341 that no such level exists and no pixels are to be masked as suspicious.
2343 suspectLevel = amp.getSuspectLevel()
2344 if math.isnan(suspectLevel):
2347 maskedImage = exposure.getMaskedImage()
2348 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2349 isrFunctions.makeThresholdMask(
2350 maskedImage=dataView,
2351 threshold=suspectLevel,
2353 maskName=self.config.suspectMaskName,
2357 """Mask defects using mask plane "BAD", in place.
2361 exposure : `lsst.afw.image.Exposure`
2362 Exposure to process.
2363 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2364 `lsst.afw.image.DefectBase`.
2365 List of defects to mask.
2369 Call this after CCD assembly, since defects may cross amplifier boundaries.
2371 maskedImage = exposure.getMaskedImage()
2372 if not isinstance(defectBaseList, Defects):
2374 defectList =
Defects(defectBaseList)
2376 defectList = defectBaseList
2377 defectList.maskPixels(maskedImage, maskName=
"BAD")
2379 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2380 """Mask edge pixels with applicable mask plane.
2384 exposure : `lsst.afw.image.Exposure`
2385 Exposure to process.
2386 numEdgePixels : `int`, optional
2387 Number of edge pixels to mask.
2388 maskPlane : `str`, optional
2389 Mask plane name to use.
2390 level : `str`, optional
2391 Level at which to mask edges.
2393 maskedImage = exposure.getMaskedImage()
2394 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2396 if numEdgePixels > 0:
2397 if level ==
'DETECTOR':
2398 boxes = [maskedImage.getBBox()]
2399 elif level ==
'AMP':
2400 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2404 subImage = maskedImage[box]
2405 box.grow(-numEdgePixels)
2407 SourceDetectionTask.setEdgeBits(
2413 """Mask and interpolate defects using mask plane "BAD", in place.
2417 exposure : `lsst.afw.image.Exposure`
2418 Exposure to process.
2419 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2420 `lsst.afw.image.DefectBase`.
2421 List of defects to mask and interpolate.
2425 lsst.ip.isr.isrTask.maskDefect
2427 self.
maskDefectmaskDefect(exposure, defectBaseList)
2428 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2429 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2430 isrFunctions.interpolateFromMask(
2431 maskedImage=exposure.getMaskedImage(),
2432 fwhm=self.config.fwhm,
2433 growSaturatedFootprints=0,
2434 maskNameList=[
"BAD"],
2438 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2442 exposure : `lsst.afw.image.Exposure`
2443 Exposure to process.
2447 We mask over all non-finite values (NaN, inf), including those
2448 that are masked with other bits (because those may or may not be
2449 interpolated over later, and we want to remove all NaN/infs).
2450 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2451 preserve the historical name.
2453 maskedImage = exposure.getMaskedImage()
2456 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2457 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2458 numNans =
maskNans(maskedImage, maskVal)
2459 self.metadata.set(
"NUMNANS", numNans)
2461 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2464 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2469 exposure : `lsst.afw.image.Exposure`
2470 Exposure to process.
2474 lsst.ip.isr.isrTask.maskNan
2477 isrFunctions.interpolateFromMask(
2478 maskedImage=exposure.getMaskedImage(),
2479 fwhm=self.config.fwhm,
2480 growSaturatedFootprints=0,
2481 maskNameList=[
"UNMASKEDNAN"],
2485 """Measure the image background in subgrids, for quality control purposes.
2489 exposure : `lsst.afw.image.Exposure`
2490 Exposure to process.
2491 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2492 Configuration object containing parameters on which background
2493 statistics and subgrids to use.
2495 if IsrQaConfig
is not None:
2496 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2497 IsrQaConfig.flatness.nIter)
2498 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2499 statsControl.setAndMask(maskVal)
2500 maskedImage = exposure.getMaskedImage()
2501 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2502 skyLevel = stats.getValue(afwMath.MEDIAN)
2503 skySigma = stats.getValue(afwMath.STDEVCLIP)
2504 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2505 metadata = exposure.getMetadata()
2506 metadata.set(
'SKYLEVEL', skyLevel)
2507 metadata.set(
'SKYSIGMA', skySigma)
2510 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2511 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2512 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2513 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2514 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2515 skyLevels = numpy.zeros((nX, nY))
2518 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2520 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2522 xLLC = xc - meshXHalf
2523 yLLC = yc - meshYHalf
2524 xURC = xc + meshXHalf - 1
2525 yURC = yc + meshYHalf - 1
2528 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2530 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2532 good = numpy.where(numpy.isfinite(skyLevels))
2533 skyMedian = numpy.median(skyLevels[good])
2534 flatness = (skyLevels[good] - skyMedian) / skyMedian
2535 flatness_rms = numpy.std(flatness)
2536 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2538 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2539 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2540 nX, nY, flatness_pp, flatness_rms)
2542 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2543 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2544 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2545 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2546 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2549 """Set an approximate magnitude zero point for the exposure.
2553 exposure : `lsst.afw.image.Exposure`
2554 Exposure to process.
2556 filterLabel = exposure.getFilterLabel()
2557 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2559 if physicalFilter
in self.config.fluxMag0T1:
2560 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2562 self.log.warn(
"No rough magnitude zero point defined for filter {}.".format(physicalFilter))
2563 fluxMag0 = self.config.defaultFluxMag0T1
2565 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2567 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2570 self.log.info(
"Setting rough magnitude zero point for filter {}: {}".
2571 format(physicalFilter, 2.5*math.log10(fluxMag0*expTime)))
2572 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2575 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2579 ccdExposure : `lsst.afw.image.Exposure`
2580 Exposure to process.
2581 fpPolygon : `lsst.afw.geom.Polygon`
2582 Polygon in focal plane coordinates.
2585 ccd = ccdExposure.getDetector()
2586 fpCorners = ccd.getCorners(FOCAL_PLANE)
2587 ccdPolygon = Polygon(fpCorners)
2590 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2593 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2594 validPolygon = Polygon(ccdPoints)
2595 ccdExposure.getInfo().setValidPolygon(validPolygon)
2599 """Context manager that applies and removes flats and darks,
2600 if the task is configured to apply them.
2604 exp : `lsst.afw.image.Exposure`
2605 Exposure to process.
2606 flat : `lsst.afw.image.Exposure`
2607 Flat exposure the same size as ``exp``.
2608 dark : `lsst.afw.image.Exposure`, optional
2609 Dark exposure the same size as ``exp``.
2613 exp : `lsst.afw.image.Exposure`
2614 The flat and dark corrected exposure.
2616 if self.config.doDark
and dark
is not None:
2618 if self.config.doFlat:
2623 if self.config.doFlat:
2625 if self.config.doDark
and dark
is not None:
2629 """Utility function to examine ISR exposure at different stages.
2633 exposure : `lsst.afw.image.Exposure`
2636 State of processing to view.
2638 frame = getDebugFrame(self._display, stepname)
2640 display = getDisplay(frame)
2641 display.scale(
'asinh',
'zscale')
2642 display.mtv(exposure)
2643 prompt =
"Press Enter to continue [c]... "
2645 ans = input(prompt).lower()
2646 if ans
in (
"",
"c",):
2651 """A Detector-like object that supports returning gain and saturation level
2653 This is used when the input exposure does not have a detector.
2657 exposure : `lsst.afw.image.Exposure`
2658 Exposure to generate a fake amplifier for.
2659 config : `lsst.ip.isr.isrTaskConfig`
2660 Configuration to apply to the fake amplifier.
2664 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2666 self.
_gain_gain = config.gain
2671 return self.
_bbox_bbox
2674 return self.
_bbox_bbox
2680 return self.
_gain_gain
2693 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2697 """Task to wrap the default IsrTask to allow it to be retargeted.
2699 The standard IsrTask can be called directly from a command line
2700 program, but doing so removes the ability of the task to be
2701 retargeted. As most cameras override some set of the IsrTask
2702 methods, this would remove those data-specific methods in the
2703 output post-ISR images. This wrapping class fixes the issue,
2704 allowing identical post-ISR images to be generated by both the
2705 processCcd and isrTask code.
2707 ConfigClass = RunIsrConfig
2708 _DefaultName =
"runIsr"
2712 self.makeSubtask(
"isr")
2718 dataRef : `lsst.daf.persistence.ButlerDataRef`
2719 data reference of the detector data to be processed
2723 result : `pipeBase.Struct`
2724 Result struct with component:
2726 - exposure : `lsst.afw.image.Exposure`
2727 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def ensureExposure(self, inputExp, camera, detectorNum)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.