29 import lsst.pipe.base
as pipeBase
30 import lsst.pipe.base.connectionTypes
as cT
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
126 crosstalkSources = cT.PrerequisiteInput(
127 name=
"isrOverscanCorrected",
128 doc=
"Overscan corrected input images.",
129 storageClass=
"Exposure",
130 dimensions=[
"instrument",
"exposure",
"detector"],
133 lookupFunction=crosstalkSourceLookup,
136 bias = cT.PrerequisiteInput(
138 doc=
"Input bias calibration.",
139 storageClass=
"ExposureF",
140 dimensions=[
"instrument",
"detector"],
143 dark = cT.PrerequisiteInput(
145 doc=
"Input dark calibration.",
146 storageClass=
"ExposureF",
147 dimensions=[
"instrument",
"detector"],
150 flat = cT.PrerequisiteInput(
152 doc=
"Input flat calibration.",
153 storageClass=
"ExposureF",
154 dimensions=[
"instrument",
"physical_filter",
"detector"],
157 ptc = cT.PrerequisiteInput(
159 doc=
"Input Photon Transfer Curve dataset",
160 storageClass=
"PhotonTransferCurveDataset",
161 dimensions=[
"instrument",
"detector"],
164 fringes = cT.PrerequisiteInput(
166 doc=
"Input fringe calibration.",
167 storageClass=
"ExposureF",
168 dimensions=[
"instrument",
"physical_filter",
"detector"],
172 strayLightData = cT.PrerequisiteInput(
174 doc=
"Input stray light calibration.",
175 storageClass=
"StrayLightData",
176 dimensions=[
"instrument",
"physical_filter",
"detector"],
180 bfKernel = cT.PrerequisiteInput(
182 doc=
"Input brighter-fatter kernel.",
183 storageClass=
"NumpyArray",
184 dimensions=[
"instrument"],
188 newBFKernel = cT.PrerequisiteInput(
189 name=
'brighterFatterKernel',
190 doc=
"Newer complete kernel + gain solutions.",
191 storageClass=
"BrighterFatterKernel",
192 dimensions=[
"instrument",
"detector"],
196 defects = cT.PrerequisiteInput(
198 doc=
"Input defect tables.",
199 storageClass=
"Defects",
200 dimensions=[
"instrument",
"detector"],
203 linearizer = cT.PrerequisiteInput(
205 storageClass=
"Linearizer",
206 doc=
"Linearity correction calibration.",
207 dimensions=[
"instrument",
"detector"],
211 opticsTransmission = cT.PrerequisiteInput(
212 name=
"transmission_optics",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the optics.",
215 dimensions=[
"instrument"],
218 filterTransmission = cT.PrerequisiteInput(
219 name=
"transmission_filter",
220 storageClass=
"TransmissionCurve",
221 doc=
"Transmission curve due to the filter.",
222 dimensions=[
"instrument",
"physical_filter"],
225 sensorTransmission = cT.PrerequisiteInput(
226 name=
"transmission_sensor",
227 storageClass=
"TransmissionCurve",
228 doc=
"Transmission curve due to the sensor.",
229 dimensions=[
"instrument",
"detector"],
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name=
"transmission_atmosphere",
234 storageClass=
"TransmissionCurve",
235 doc=
"Transmission curve due to the atmosphere.",
236 dimensions=[
"instrument"],
239 illumMaskedImage = cT.PrerequisiteInput(
241 doc=
"Input illumination correction.",
242 storageClass=
"MaskedImageF",
243 dimensions=[
"instrument",
"physical_filter",
"detector"],
247 outputExposure = cT.Output(
249 doc=
"Output ISR processed exposure.",
250 storageClass=
"Exposure",
251 dimensions=[
"instrument",
"exposure",
"detector"],
253 preInterpExposure = cT.Output(
254 name=
'preInterpISRCCD',
255 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
256 storageClass=
"ExposureF",
257 dimensions=[
"instrument",
"exposure",
"detector"],
259 outputOssThumbnail = cT.Output(
261 doc=
"Output Overscan-subtracted thumbnail image.",
262 storageClass=
"Thumbnail",
263 dimensions=[
"instrument",
"exposure",
"detector"],
265 outputFlattenedThumbnail = cT.Output(
266 name=
"FlattenedThumb",
267 doc=
"Output flat-corrected thumbnail image.",
268 storageClass=
"Thumbnail",
269 dimensions=[
"instrument",
"exposure",
"detector"],
275 if config.doBias
is not True:
276 self.prerequisiteInputs.discard(
"bias")
277 if config.doLinearize
is not True:
278 self.prerequisiteInputs.discard(
"linearizer")
279 if config.doCrosstalk
is not True:
280 self.inputs.discard(
"crosstalkSources")
281 self.prerequisiteInputs.discard(
"crosstalk")
282 if config.doBrighterFatter
is not True:
283 self.prerequisiteInputs.discard(
"bfKernel")
284 self.prerequisiteInputs.discard(
"newBFKernel")
285 if config.doDefect
is not True:
286 self.prerequisiteInputs.discard(
"defects")
287 if config.doDark
is not True:
288 self.prerequisiteInputs.discard(
"dark")
289 if config.doFlat
is not True:
290 self.prerequisiteInputs.discard(
"flat")
291 if config.doFringe
is not True:
292 self.prerequisiteInputs.discard(
"fringe")
293 if config.doStrayLight
is not True:
294 self.prerequisiteInputs.discard(
"strayLightData")
295 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
296 self.prerequisiteInputs.discard(
"ptc")
297 if config.doAttachTransmissionCurve
is not True:
298 self.prerequisiteInputs.discard(
"opticsTransmission")
299 self.prerequisiteInputs.discard(
"filterTransmission")
300 self.prerequisiteInputs.discard(
"sensorTransmission")
301 self.prerequisiteInputs.discard(
"atmosphereTransmission")
302 if config.doUseOpticsTransmission
is not True:
303 self.prerequisiteInputs.discard(
"opticsTransmission")
304 if config.doUseFilterTransmission
is not True:
305 self.prerequisiteInputs.discard(
"filterTransmission")
306 if config.doUseSensorTransmission
is not True:
307 self.prerequisiteInputs.discard(
"sensorTransmission")
308 if config.doUseAtmosphereTransmission
is not True:
309 self.prerequisiteInputs.discard(
"atmosphereTransmission")
310 if config.doIlluminationCorrection
is not True:
311 self.prerequisiteInputs.discard(
"illumMaskedImage")
313 if config.doWrite
is not True:
314 self.outputs.discard(
"outputExposure")
315 self.outputs.discard(
"preInterpExposure")
316 self.outputs.discard(
"outputFlattenedThumbnail")
317 self.outputs.discard(
"outputOssThumbnail")
318 if config.doSaveInterpPixels
is not True:
319 self.outputs.discard(
"preInterpExposure")
320 if config.qa.doThumbnailOss
is not True:
321 self.outputs.discard(
"outputOssThumbnail")
322 if config.qa.doThumbnailFlattened
is not True:
323 self.outputs.discard(
"outputFlattenedThumbnail")
327 pipelineConnections=IsrTaskConnections):
328 """Configuration parameters for IsrTask.
330 Items are grouped in the order in which they are executed by the task.
332 datasetType = pexConfig.Field(
334 doc=
"Dataset type for input data; users will typically leave this alone, "
335 "but camera-specific ISR tasks will override it",
339 fallbackFilterName = pexConfig.Field(
341 doc=
"Fallback default filter name for calibrations.",
344 useFallbackDate = pexConfig.Field(
346 doc=
"Pass observation date when using fallback filter.",
349 expectWcs = pexConfig.Field(
352 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
354 fwhm = pexConfig.Field(
356 doc=
"FWHM of PSF in arcseconds.",
359 qa = pexConfig.ConfigField(
361 doc=
"QA related configuration options.",
365 doConvertIntToFloat = pexConfig.Field(
367 doc=
"Convert integer raw images to floating point values?",
372 doSaturation = pexConfig.Field(
374 doc=
"Mask saturated pixels? NB: this is totally independent of the"
375 " interpolation option - this is ONLY setting the bits in the mask."
376 " To have them interpolated make sure doSaturationInterpolation=True",
379 saturatedMaskName = pexConfig.Field(
381 doc=
"Name of mask plane to use in saturation detection and interpolation",
384 saturation = pexConfig.Field(
386 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387 default=float(
"NaN"),
389 growSaturationFootprintSize = pexConfig.Field(
391 doc=
"Number of pixels by which to grow the saturation footprints",
396 doSuspect = pexConfig.Field(
398 doc=
"Mask suspect pixels?",
401 suspectMaskName = pexConfig.Field(
403 doc=
"Name of mask plane to use for suspect pixels",
406 numEdgeSuspect = pexConfig.Field(
408 doc=
"Number of edge pixels to be flagged as untrustworthy.",
411 edgeMaskLevel = pexConfig.ChoiceField(
413 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
416 'DETECTOR':
'Mask only the edges of the full detector.',
417 'AMP':
'Mask edges of each amplifier.',
422 doSetBadRegions = pexConfig.Field(
424 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
427 badStatistic = pexConfig.ChoiceField(
429 doc=
"How to estimate the average value for BAD regions.",
432 "MEANCLIP":
"Correct using the (clipped) mean of good data",
433 "MEDIAN":
"Correct using the median of the good data",
438 doOverscan = pexConfig.Field(
440 doc=
"Do overscan subtraction?",
443 overscan = pexConfig.ConfigurableField(
444 target=OverscanCorrectionTask,
445 doc=
"Overscan subtraction task for image segments.",
448 overscanFitType = pexConfig.ChoiceField(
450 doc=
"The method for fitting the overscan bias level.",
453 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
454 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
455 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
456 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
457 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
458 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
459 "MEAN":
"Correct using the mean of the overscan region",
460 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
461 "MEDIAN":
"Correct using the median of the overscan region",
462 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
464 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
465 " This option will no longer be used, and will be removed after v20.")
467 overscanOrder = pexConfig.Field(
469 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
470 "or number of spline knots if overscan fit type is a spline."),
472 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
473 " This option will no longer be used, and will be removed after v20.")
475 overscanNumSigmaClip = pexConfig.Field(
477 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
479 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
480 " This option will no longer be used, and will be removed after v20.")
482 overscanIsInt = pexConfig.Field(
484 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485 " and overscan.FitType=MEDIAN_PER_ROW.",
487 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
488 " This option will no longer be used, and will be removed after v20.")
491 overscanNumLeadingColumnsToSkip = pexConfig.Field(
493 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
496 overscanNumTrailingColumnsToSkip = pexConfig.Field(
498 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
501 overscanMaxDev = pexConfig.Field(
503 doc=
"Maximum deviation from the median for overscan",
504 default=1000.0, check=
lambda x: x > 0
506 overscanBiasJump = pexConfig.Field(
508 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
511 overscanBiasJumpKeyword = pexConfig.Field(
513 doc=
"Header keyword containing information about devices.",
514 default=
"NO_SUCH_KEY",
516 overscanBiasJumpDevices = pexConfig.ListField(
518 doc=
"List of devices that need piecewise overscan correction.",
521 overscanBiasJumpLocation = pexConfig.Field(
523 doc=
"Location of bias jump along y-axis.",
528 doAssembleCcd = pexConfig.Field(
531 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
533 assembleCcd = pexConfig.ConfigurableField(
534 target=AssembleCcdTask,
535 doc=
"CCD assembly task",
539 doAssembleIsrExposures = pexConfig.Field(
542 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
544 doTrimToMatchCalib = pexConfig.Field(
547 doc=
"Trim raw data to match calibration bounding boxes?"
551 doBias = pexConfig.Field(
553 doc=
"Apply bias frame correction?",
556 biasDataProductName = pexConfig.Field(
558 doc=
"Name of the bias data product",
561 doBiasBeforeOverscan = pexConfig.Field(
563 doc=
"Reverse order of overscan and bias correction.",
568 doVariance = pexConfig.Field(
570 doc=
"Calculate variance?",
573 gain = pexConfig.Field(
575 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576 default=float(
"NaN"),
578 readNoise = pexConfig.Field(
580 doc=
"The read noise to use if no Detector is present in the Exposure",
583 doEmpiricalReadNoise = pexConfig.Field(
586 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
588 usePtcReadNoise = pexConfig.Field(
591 doc=
"Use readnoise values from the Photon Transfer Curve?"
594 doLinearize = pexConfig.Field(
596 doc=
"Correct for nonlinearity of the detector's response?",
601 doCrosstalk = pexConfig.Field(
603 doc=
"Apply intra-CCD crosstalk correction?",
606 doCrosstalkBeforeAssemble = pexConfig.Field(
608 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
611 crosstalk = pexConfig.ConfigurableField(
612 target=CrosstalkTask,
613 doc=
"Intra-CCD crosstalk correction",
617 doDefect = pexConfig.Field(
619 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
622 doNanMasking = pexConfig.Field(
624 doc=
"Mask non-finite (NAN, inf) pixels?",
627 doWidenSaturationTrails = pexConfig.Field(
629 doc=
"Widen bleed trails based on their width?",
634 doBrighterFatter = pexConfig.Field(
637 doc=
"Apply the brighter-fatter correction?"
639 brighterFatterLevel = pexConfig.ChoiceField(
642 doc=
"The level at which to correct for brighter-fatter.",
644 "AMP":
"Every amplifier treated separately.",
645 "DETECTOR":
"One kernel per detector",
648 brighterFatterMaxIter = pexConfig.Field(
651 doc=
"Maximum number of iterations for the brighter-fatter correction"
653 brighterFatterThreshold = pexConfig.Field(
656 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
657 "absolute value of the difference between the current corrected image and the one "
658 "from the previous iteration summed over all the pixels."
660 brighterFatterApplyGain = pexConfig.Field(
663 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
665 brighterFatterMaskListToInterpolate = pexConfig.ListField(
667 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
669 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
671 brighterFatterMaskGrowSize = pexConfig.Field(
674 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
675 "when brighter-fatter correction is applied."
679 doDark = pexConfig.Field(
681 doc=
"Apply dark frame correction?",
684 darkDataProductName = pexConfig.Field(
686 doc=
"Name of the dark data product",
691 doStrayLight = pexConfig.Field(
693 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
696 strayLight = pexConfig.ConfigurableField(
697 target=StrayLightTask,
698 doc=
"y-band stray light correction"
702 doFlat = pexConfig.Field(
704 doc=
"Apply flat field correction?",
707 flatDataProductName = pexConfig.Field(
709 doc=
"Name of the flat data product",
712 flatScalingType = pexConfig.ChoiceField(
714 doc=
"The method for scaling the flat on the fly.",
717 "USER":
"Scale by flatUserScale",
718 "MEAN":
"Scale by the inverse of the mean",
719 "MEDIAN":
"Scale by the inverse of the median",
722 flatUserScale = pexConfig.Field(
724 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
727 doTweakFlat = pexConfig.Field(
729 doc=
"Tweak flats to match observed amplifier ratios?",
734 doApplyGains = pexConfig.Field(
736 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
739 usePtcGains = pexConfig.Field(
741 doc=
"Use the gain values from the Photon Transfer Curve?",
744 normalizeGains = pexConfig.Field(
746 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
751 doFringe = pexConfig.Field(
753 doc=
"Apply fringe correction?",
756 fringe = pexConfig.ConfigurableField(
758 doc=
"Fringe subtraction task",
760 fringeAfterFlat = pexConfig.Field(
762 doc=
"Do fringe subtraction after flat-fielding?",
767 doMeasureBackground = pexConfig.Field(
769 doc=
"Measure the background level on the reduced image?",
774 doCameraSpecificMasking = pexConfig.Field(
776 doc=
"Mask camera-specific bad regions?",
779 masking = pexConfig.ConfigurableField(
786 doInterpolate = pexConfig.Field(
788 doc=
"Interpolate masked pixels?",
791 doSaturationInterpolation = pexConfig.Field(
793 doc=
"Perform interpolation over pixels masked as saturated?"
794 " NB: This is independent of doSaturation; if that is False this plane"
795 " will likely be blank, resulting in a no-op here.",
798 doNanInterpolation = pexConfig.Field(
800 doc=
"Perform interpolation over pixels masked as NaN?"
801 " NB: This is independent of doNanMasking; if that is False this plane"
802 " will likely be blank, resulting in a no-op here.",
805 doNanInterpAfterFlat = pexConfig.Field(
807 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
808 "also have to interpolate them before flat-fielding."),
811 maskListToInterpolate = pexConfig.ListField(
813 doc=
"List of mask planes that should be interpolated.",
814 default=[
'SAT',
'BAD'],
816 doSaveInterpPixels = pexConfig.Field(
818 doc=
"Save a copy of the pre-interpolated pixel values?",
823 fluxMag0T1 = pexConfig.DictField(
826 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
827 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
830 defaultFluxMag0T1 = pexConfig.Field(
832 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
833 default=pow(10.0, 0.4*28.0)
837 doVignette = pexConfig.Field(
839 doc=
"Apply vignetting parameters?",
842 vignette = pexConfig.ConfigurableField(
844 doc=
"Vignetting task.",
848 doAttachTransmissionCurve = pexConfig.Field(
851 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
853 doUseOpticsTransmission = pexConfig.Field(
856 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
858 doUseFilterTransmission = pexConfig.Field(
861 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
863 doUseSensorTransmission = pexConfig.Field(
866 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
868 doUseAtmosphereTransmission = pexConfig.Field(
871 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
875 doIlluminationCorrection = pexConfig.Field(
878 doc=
"Perform illumination correction?"
880 illuminationCorrectionDataProductName = pexConfig.Field(
882 doc=
"Name of the illumination correction data product.",
885 illumScale = pexConfig.Field(
887 doc=
"Scale factor for the illumination correction.",
890 illumFilters = pexConfig.ListField(
893 doc=
"Only perform illumination correction for these filters."
897 doWrite = pexConfig.Field(
899 doc=
"Persist postISRCCD?",
906 raise ValueError(
"You may not specify both doFlat and doApplyGains")
908 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
917 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
918 """Apply common instrument signature correction algorithms to a raw frame.
920 The process for correcting imaging data is very similar from
921 camera to camera. This task provides a vanilla implementation of
922 doing these corrections, including the ability to turn certain
923 corrections off if they are not needed. The inputs to the primary
924 method, `run()`, are a raw exposure to be corrected and the
925 calibration data products. The raw input is a single chip sized
926 mosaic of all amps including overscans and other non-science
927 pixels. The method `runDataRef()` identifies and defines the
928 calibration data products, and is intended for use by a
929 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
930 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
931 subclassed for different camera, although the most camera specific
932 methods have been split into subtasks that can be redirected
935 The __init__ method sets up the subtasks for ISR processing, using
936 the defaults from `lsst.ip.isr`.
941 Positional arguments passed to the Task constructor. None used at this time.
942 kwargs : `dict`, optional
943 Keyword arguments passed on to the Task constructor. None used at this time.
945 ConfigClass = IsrTaskConfig
950 self.makeSubtask(
"assembleCcd")
951 self.makeSubtask(
"crosstalk")
952 self.makeSubtask(
"strayLight")
953 self.makeSubtask(
"fringe")
954 self.makeSubtask(
"masking")
955 self.makeSubtask(
"overscan")
956 self.makeSubtask(
"vignette")
959 inputs = butlerQC.get(inputRefs)
962 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
963 except Exception
as e:
964 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
967 inputs[
'isGen3'] =
True
969 detector = inputs[
'ccdExposure'].getDetector()
971 if self.config.doCrosstalk
is True:
974 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
975 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
976 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
978 coeffVector = (self.config.crosstalk.crosstalkValues
979 if self.config.crosstalk.useConfigCoefficients
else None)
980 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
981 inputs[
'crosstalk'] = crosstalkCalib
982 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
983 if 'crosstalkSources' not in inputs:
984 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
987 if 'linearizer' in inputs:
988 if isinstance(inputs[
'linearizer'], dict):
990 linearizer.fromYaml(inputs[
'linearizer'])
991 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
992 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
996 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
998 linearizer = inputs[
'linearizer']
999 linearizer.log = self.log
1000 inputs[
'linearizer'] = linearizer
1003 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1005 if self.config.doDefect
is True:
1006 if "defects" in inputs
and inputs[
'defects']
is not None:
1009 if not isinstance(inputs[
"defects"], Defects):
1010 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1014 if self.config.doBrighterFatter:
1015 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1016 if brighterFatterKernel
is None:
1017 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1019 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1021 detName = detector.getName()
1022 level = brighterFatterKernel.level
1025 inputs[
'bfGains'] = brighterFatterKernel.gain
1026 if self.config.brighterFatterLevel ==
'DETECTOR':
1027 if level ==
'DETECTOR':
1028 if detName
in brighterFatterKernel.detKernels:
1029 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1031 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1032 elif level ==
'AMP':
1033 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1035 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1036 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1037 elif self.config.brighterFatterLevel ==
'AMP':
1038 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1040 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1041 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
1042 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1044 assembler=self.assembleCcd
1045 if self.config.doAssembleIsrExposures
else None)
1047 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1049 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1050 if 'strayLightData' not in inputs:
1051 inputs[
'strayLightData'] =
None
1053 outputs = self.
runrun(**inputs)
1054 butlerQC.put(outputs, outputRefs)
1057 """Retrieve necessary frames for instrument signature removal.
1059 Pre-fetching all required ISR data products limits the IO
1060 required by the ISR. Any conflict between the calibration data
1061 available and that needed for ISR is also detected prior to
1062 doing processing, allowing it to fail quickly.
1066 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1067 Butler reference of the detector data to be processed
1068 rawExposure : `afw.image.Exposure`
1069 The raw exposure that will later be corrected with the
1070 retrieved calibration data; should not be modified in this
1075 result : `lsst.pipe.base.Struct`
1076 Result struct with components (which may be `None`):
1077 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1078 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1079 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1080 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1081 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1082 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1083 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1084 - ``fringes``: `lsst.pipe.base.Struct` with components:
1085 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1086 - ``seed``: random seed derived from the ccdExposureId for random
1087 number generator (`uint32`).
1088 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1089 A ``TransmissionCurve`` that represents the throughput of the optics,
1090 to be evaluated in focal-plane coordinates.
1091 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1092 A ``TransmissionCurve`` that represents the throughput of the filter
1093 itself, to be evaluated in focal-plane coordinates.
1094 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1095 A ``TransmissionCurve`` that represents the throughput of the sensor
1096 itself, to be evaluated in post-assembly trimmed detector coordinates.
1097 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1098 A ``TransmissionCurve`` that represents the throughput of the
1099 atmosphere, assumed to be spatially constant.
1100 - ``strayLightData`` : `object`
1101 An opaque object containing calibration information for
1102 stray-light correction. If `None`, no correction will be
1104 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1108 NotImplementedError :
1109 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1112 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1113 dateObs = dateObs.toPython().isoformat()
1114 except RuntimeError:
1115 self.log.warning(
"Unable to identify dateObs for rawExposure.")
1118 ccd = rawExposure.getDetector()
1119 filterLabel = rawExposure.getFilterLabel()
1120 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1121 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1122 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1123 if self.config.doBias
else None)
1125 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1127 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1128 linearizer.log = self.log
1129 if isinstance(linearizer, numpy.ndarray):
1132 crosstalkCalib =
None
1133 if self.config.doCrosstalk:
1135 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1137 coeffVector = (self.config.crosstalk.crosstalkValues
1138 if self.config.crosstalk.useConfigCoefficients
else None)
1139 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1140 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1141 if self.config.doCrosstalk
else None)
1143 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1144 if self.config.doDark
else None)
1145 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1147 if self.config.doFlat
else None)
1149 brighterFatterKernel =
None
1150 brighterFatterGains =
None
1151 if self.config.doBrighterFatter
is True:
1156 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1157 brighterFatterGains = brighterFatterKernel.gain
1158 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1161 brighterFatterKernel = dataRef.get(
"bfKernel")
1162 self.log.info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1164 brighterFatterKernel =
None
1165 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1168 if self.config.brighterFatterLevel ==
'DETECTOR':
1169 if brighterFatterKernel.detKernels:
1170 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1172 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1175 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1177 defectList = (dataRef.get(
"defects")
1178 if self.config.doDefect
else None)
1179 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1180 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1181 if self.config.doAssembleIsrExposures
else None)
1182 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1183 else pipeBase.Struct(fringes=
None))
1185 if self.config.doAttachTransmissionCurve:
1186 opticsTransmission = (dataRef.get(
"transmission_optics")
1187 if self.config.doUseOpticsTransmission
else None)
1188 filterTransmission = (dataRef.get(
"transmission_filter")
1189 if self.config.doUseFilterTransmission
else None)
1190 sensorTransmission = (dataRef.get(
"transmission_sensor")
1191 if self.config.doUseSensorTransmission
else None)
1192 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1193 if self.config.doUseAtmosphereTransmission
else None)
1195 opticsTransmission =
None
1196 filterTransmission =
None
1197 sensorTransmission =
None
1198 atmosphereTransmission =
None
1200 if self.config.doStrayLight:
1201 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1203 strayLightData =
None
1206 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1207 if (self.config.doIlluminationCorrection
1208 and physicalFilter
in self.config.illumFilters)
1212 return pipeBase.Struct(bias=biasExposure,
1213 linearizer=linearizer,
1214 crosstalk=crosstalkCalib,
1215 crosstalkSources=crosstalkSources,
1218 bfKernel=brighterFatterKernel,
1219 bfGains=brighterFatterGains,
1221 fringes=fringeStruct,
1222 opticsTransmission=opticsTransmission,
1223 filterTransmission=filterTransmission,
1224 sensorTransmission=sensorTransmission,
1225 atmosphereTransmission=atmosphereTransmission,
1226 strayLightData=strayLightData,
1227 illumMaskedImage=illumMaskedImage
1230 @pipeBase.timeMethod
1231 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1232 crosstalk=None, crosstalkSources=None,
1233 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1234 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1235 sensorTransmission=
None, atmosphereTransmission=
None,
1236 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1239 """Perform instrument signature removal on an exposure.
1241 Steps included in the ISR processing, in order performed, are:
1242 - saturation and suspect pixel masking
1243 - overscan subtraction
1244 - CCD assembly of individual amplifiers
1246 - variance image construction
1247 - linearization of non-linear response
1249 - brighter-fatter correction
1252 - stray light subtraction
1254 - masking of known defects and camera specific features
1255 - vignette calculation
1256 - appending transmission curve and distortion model
1260 ccdExposure : `lsst.afw.image.Exposure`
1261 The raw exposure that is to be run through ISR. The
1262 exposure is modified by this method.
1263 camera : `lsst.afw.cameraGeom.Camera`, optional
1264 The camera geometry for this exposure. Required if ``isGen3`` is
1265 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1266 ``flat`` does not have an associated detector.
1267 bias : `lsst.afw.image.Exposure`, optional
1268 Bias calibration frame.
1269 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1270 Functor for linearization.
1271 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1272 Calibration for crosstalk.
1273 crosstalkSources : `list`, optional
1274 List of possible crosstalk sources.
1275 dark : `lsst.afw.image.Exposure`, optional
1276 Dark calibration frame.
1277 flat : `lsst.afw.image.Exposure`, optional
1278 Flat calibration frame.
1279 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1280 Photon transfer curve dataset, with, e.g., gains
1282 bfKernel : `numpy.ndarray`, optional
1283 Brighter-fatter kernel.
1284 bfGains : `dict` of `float`, optional
1285 Gains used to override the detector's nominal gains for the
1286 brighter-fatter correction. A dict keyed by amplifier name for
1287 the detector in question.
1288 defects : `lsst.ip.isr.Defects`, optional
1290 fringes : `lsst.pipe.base.Struct`, optional
1291 Struct containing the fringe correction data, with
1293 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1294 - ``seed``: random seed derived from the ccdExposureId for random
1295 number generator (`uint32`)
1296 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1297 A ``TransmissionCurve`` that represents the throughput of the optics,
1298 to be evaluated in focal-plane coordinates.
1299 filterTransmission : `lsst.afw.image.TransmissionCurve`
1300 A ``TransmissionCurve`` that represents the throughput of the filter
1301 itself, to be evaluated in focal-plane coordinates.
1302 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1303 A ``TransmissionCurve`` that represents the throughput of the sensor
1304 itself, to be evaluated in post-assembly trimmed detector coordinates.
1305 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1306 A ``TransmissionCurve`` that represents the throughput of the
1307 atmosphere, assumed to be spatially constant.
1308 detectorNum : `int`, optional
1309 The integer number for the detector to process.
1310 isGen3 : bool, optional
1311 Flag this call to run() as using the Gen3 butler environment.
1312 strayLightData : `object`, optional
1313 Opaque object containing calibration information for stray-light
1314 correction. If `None`, no correction will be performed.
1315 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1316 Illumination correction image.
1320 result : `lsst.pipe.base.Struct`
1321 Result struct with component:
1322 - ``exposure`` : `afw.image.Exposure`
1323 The fully ISR corrected exposure.
1324 - ``outputExposure`` : `afw.image.Exposure`
1325 An alias for `exposure`
1326 - ``ossThumb`` : `numpy.ndarray`
1327 Thumbnail image of the exposure after overscan subtraction.
1328 - ``flattenedThumb`` : `numpy.ndarray`
1329 Thumbnail image of the exposure after flat-field correction.
1334 Raised if a configuration option is set to True, but the
1335 required calibration data has not been specified.
1339 The current processed exposure can be viewed by setting the
1340 appropriate lsstDebug entries in the `debug.display`
1341 dictionary. The names of these entries correspond to some of
1342 the IsrTaskConfig Boolean options, with the value denoting the
1343 frame to use. The exposure is shown inside the matching
1344 option check and after the processing of that step has
1345 finished. The steps with debug points are:
1356 In addition, setting the "postISRCCD" entry displays the
1357 exposure after all ISR processing has finished.
1365 if detectorNum
is None:
1366 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1368 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1369 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1370 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1371 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1373 if isinstance(ccdExposure, ButlerDataRef):
1374 return self.
runDataRefrunDataRef(ccdExposure)
1376 ccd = ccdExposure.getDetector()
1377 filterLabel = ccdExposure.getFilterLabel()
1378 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1381 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1382 ccd = [
FakeAmp(ccdExposure, self.config)]
1385 if self.config.doBias
and bias
is None:
1386 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1387 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1388 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1389 if self.config.doBrighterFatter
and bfKernel
is None:
1390 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1391 if self.config.doDark
and dark
is None:
1392 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1393 if self.config.doFlat
and flat
is None:
1394 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1395 if self.config.doDefect
and defects
is None:
1396 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1397 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1398 and fringes.fringes
is None):
1403 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1404 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1405 and illumMaskedImage
is None):
1406 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1409 if self.config.doConvertIntToFloat:
1410 self.log.info(
"Converting exposure to floating point values.")
1413 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1414 self.log.info(
"Applying bias correction.")
1415 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1416 trimToFit=self.config.doTrimToMatchCalib)
1417 self.
debugViewdebugView(ccdExposure,
"doBias")
1423 if ccdExposure.getBBox().contains(amp.getBBox()):
1425 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1427 if self.config.doOverscan
and not badAmp:
1430 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1431 if overscanResults
is not None and \
1432 self.config.qa
is not None and self.config.qa.saveStats
is True:
1433 if isinstance(overscanResults.overscanFit, float):
1434 qaMedian = overscanResults.overscanFit
1435 qaStdev = float(
"NaN")
1437 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1438 afwMath.MEDIAN | afwMath.STDEVCLIP)
1439 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1440 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1442 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1443 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1444 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1445 amp.getName(), qaMedian, qaStdev)
1448 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1449 afwMath.MEDIAN | afwMath.STDEVCLIP)
1450 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1451 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1453 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1454 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1455 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1456 amp.getName(), qaMedianAfter, qaStdevAfter)
1458 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1461 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1462 overscanResults =
None
1464 overscans.append(overscanResults
if overscanResults
is not None else None)
1466 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1468 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1469 self.log.info(
"Applying crosstalk correction.")
1470 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1471 crosstalkSources=crosstalkSources, camera=camera)
1472 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1474 if self.config.doAssembleCcd:
1475 self.log.info(
"Assembling CCD from amplifiers.")
1476 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1478 if self.config.expectWcs
and not ccdExposure.getWcs():
1479 self.log.warning(
"No WCS found in input exposure.")
1480 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1483 if self.config.qa.doThumbnailOss:
1484 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1486 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1487 self.log.info(
"Applying bias correction.")
1488 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1489 trimToFit=self.config.doTrimToMatchCalib)
1490 self.
debugViewdebugView(ccdExposure,
"doBias")
1492 if self.config.doVariance:
1493 for amp, overscanResults
in zip(ccd, overscans):
1494 if ccdExposure.getBBox().contains(amp.getBBox()):
1495 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1496 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1497 if overscanResults
is not None:
1499 overscanImage=overscanResults.overscanImage,
1505 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1506 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1507 afwMath.MEDIAN | afwMath.STDEVCLIP)
1508 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1509 qaStats.getValue(afwMath.MEDIAN))
1510 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1511 qaStats.getValue(afwMath.STDEVCLIP))
1512 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1513 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1514 qaStats.getValue(afwMath.STDEVCLIP))
1517 self.log.info(
"Applying linearizer.")
1518 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1519 detector=ccd, log=self.log)
1521 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1522 self.log.info(
"Applying crosstalk correction.")
1523 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1524 crosstalkSources=crosstalkSources, isTrimmed=
True)
1525 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1529 if self.config.doDefect:
1530 self.log.info(
"Masking defects.")
1531 self.
maskDefectmaskDefect(ccdExposure, defects)
1533 if self.config.numEdgeSuspect > 0:
1534 self.log.info(
"Masking edges as SUSPECT.")
1535 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1536 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1538 if self.config.doNanMasking:
1539 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1540 self.
maskNanmaskNan(ccdExposure)
1542 if self.config.doWidenSaturationTrails:
1543 self.log.info(
"Widening saturation trails.")
1544 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1546 if self.config.doCameraSpecificMasking:
1547 self.log.info(
"Masking regions for camera specific reasons.")
1548 self.masking.
run(ccdExposure)
1550 if self.config.doBrighterFatter:
1559 interpExp = ccdExposure.clone()
1560 with self.
flatContextflatContext(interpExp, flat, dark):
1561 isrFunctions.interpolateFromMask(
1562 maskedImage=interpExp.getMaskedImage(),
1563 fwhm=self.config.fwhm,
1564 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1565 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1567 bfExp = interpExp.clone()
1569 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1570 type(bfKernel), type(bfGains))
1571 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1572 self.config.brighterFatterMaxIter,
1573 self.config.brighterFatterThreshold,
1574 self.config.brighterFatterApplyGain,
1576 if bfResults[1] == self.config.brighterFatterMaxIter:
1577 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1580 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1582 image = ccdExposure.getMaskedImage().getImage()
1583 bfCorr = bfExp.getMaskedImage().getImage()
1584 bfCorr -= interpExp.getMaskedImage().getImage()
1593 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1594 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1597 if self.config.brighterFatterMaskGrowSize > 0:
1598 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1599 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1600 isrFunctions.growMasks(ccdExposure.getMask(),
1601 radius=self.config.brighterFatterMaskGrowSize,
1602 maskNameList=maskPlane,
1603 maskValue=maskPlane)
1605 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1607 if self.config.doDark:
1608 self.log.info(
"Applying dark correction.")
1610 self.
debugViewdebugView(ccdExposure,
"doDark")
1612 if self.config.doFringe
and not self.config.fringeAfterFlat:
1613 self.log.info(
"Applying fringe correction before flat.")
1614 self.fringe.
run(ccdExposure, **fringes.getDict())
1615 self.
debugViewdebugView(ccdExposure,
"doFringe")
1617 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1618 self.log.info(
"Checking strayLight correction.")
1619 self.strayLight.
run(ccdExposure, strayLightData)
1620 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1622 if self.config.doFlat:
1623 self.log.info(
"Applying flat correction.")
1625 self.
debugViewdebugView(ccdExposure,
"doFlat")
1627 if self.config.doApplyGains:
1628 self.log.info(
"Applying gain correction instead of flat.")
1629 if self.config.usePtcGains:
1630 self.log.info(
"Using gains from the Photon Transfer Curve.")
1631 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1634 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1636 if self.config.doFringe
and self.config.fringeAfterFlat:
1637 self.log.info(
"Applying fringe correction after flat.")
1638 self.fringe.
run(ccdExposure, **fringes.getDict())
1640 if self.config.doVignette:
1641 self.log.info(
"Constructing Vignette polygon.")
1644 if self.config.vignette.doWriteVignettePolygon:
1647 if self.config.doAttachTransmissionCurve:
1648 self.log.info(
"Adding transmission curves.")
1649 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1650 filterTransmission=filterTransmission,
1651 sensorTransmission=sensorTransmission,
1652 atmosphereTransmission=atmosphereTransmission)
1654 flattenedThumb =
None
1655 if self.config.qa.doThumbnailFlattened:
1656 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1658 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1659 self.log.info(
"Performing illumination correction.")
1660 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1661 illumMaskedImage, illumScale=self.config.illumScale,
1662 trimToFit=self.config.doTrimToMatchCalib)
1665 if self.config.doSaveInterpPixels:
1666 preInterpExp = ccdExposure.clone()
1681 if self.config.doSetBadRegions:
1682 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1683 if badPixelCount > 0:
1684 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1686 if self.config.doInterpolate:
1687 self.log.info(
"Interpolating masked pixels.")
1688 isrFunctions.interpolateFromMask(
1689 maskedImage=ccdExposure.getMaskedImage(),
1690 fwhm=self.config.fwhm,
1691 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1692 maskNameList=list(self.config.maskListToInterpolate)
1697 if self.config.doMeasureBackground:
1698 self.log.info(
"Measuring background level.")
1701 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1703 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1704 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1705 afwMath.MEDIAN | afwMath.STDEVCLIP)
1706 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1707 qaStats.getValue(afwMath.MEDIAN))
1708 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1709 qaStats.getValue(afwMath.STDEVCLIP))
1710 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1711 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1712 qaStats.getValue(afwMath.STDEVCLIP))
1714 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1716 return pipeBase.Struct(
1717 exposure=ccdExposure,
1719 flattenedThumb=flattenedThumb,
1721 preInterpExposure=preInterpExp,
1722 outputExposure=ccdExposure,
1723 outputOssThumbnail=ossThumb,
1724 outputFlattenedThumbnail=flattenedThumb,
1727 @pipeBase.timeMethod
1729 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1731 This method contains the `CmdLineTask` interface to the ISR
1732 processing. All IO is handled here, freeing the `run()` method
1733 to manage only pixel-level calculations. The steps performed
1735 - Read in necessary detrending/isr/calibration data.
1736 - Process raw exposure in `run()`.
1737 - Persist the ISR-corrected exposure as "postISRCCD" if
1738 config.doWrite=True.
1742 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1743 DataRef of the detector data to be processed
1747 result : `lsst.pipe.base.Struct`
1748 Result struct with component:
1749 - ``exposure`` : `afw.image.Exposure`
1750 The fully ISR corrected exposure.
1755 Raised if a configuration option is set to True, but the
1756 required calibration data does not exist.
1759 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1761 ccdExposure = sensorRef.get(self.config.datasetType)
1763 camera = sensorRef.get(
"camera")
1764 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1766 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1768 if self.config.doWrite:
1769 sensorRef.put(result.exposure,
"postISRCCD")
1770 if result.preInterpExposure
is not None:
1771 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1772 if result.ossThumb
is not None:
1773 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1774 if result.flattenedThumb
is not None:
1775 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1780 """Retrieve a calibration dataset for removing instrument signature.
1785 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1786 DataRef of the detector data to find calibration datasets
1789 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1790 dateObs : `str`, optional
1791 Date of the observation. Used to correct butler failures
1792 when using fallback filters.
1794 If True, disable butler proxies to enable error handling
1795 within this routine.
1799 exposure : `lsst.afw.image.Exposure`
1800 Requested calibration frame.
1805 Raised if no matching calibration frame can be found.
1808 exp = dataRef.get(datasetType, immediate=immediate)
1809 except Exception
as exc1:
1810 if not self.config.fallbackFilterName:
1811 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1813 if self.config.useFallbackDate
and dateObs:
1814 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1815 dateObs=dateObs, immediate=immediate)
1817 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1818 except Exception
as exc2:
1819 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1820 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1821 self.log.warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1823 if self.config.doAssembleIsrExposures:
1824 exp = self.assembleCcd.assembleCcd(exp)
1828 """Ensure that the data returned by Butler is a fully constructed exposure.
1830 ISR requires exposure-level image data for historical reasons, so if we did
1831 not recieve that from Butler, construct it from what we have, modifying the
1836 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1837 `lsst.afw.image.ImageF`
1838 The input data structure obtained from Butler.
1839 camera : `lsst.afw.cameraGeom.camera`
1840 The camera associated with the image. Used to find the appropriate
1843 The detector this exposure should match.
1847 inputExp : `lsst.afw.image.Exposure`
1848 The re-constructed exposure, with appropriate detector parameters.
1853 Raised if the input data cannot be used to construct an exposure.
1855 if isinstance(inputExp, afwImage.DecoratedImageU):
1856 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1857 elif isinstance(inputExp, afwImage.ImageF):
1858 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1859 elif isinstance(inputExp, afwImage.MaskedImageF):
1860 inputExp = afwImage.makeExposure(inputExp)
1861 elif isinstance(inputExp, afwImage.Exposure):
1863 elif inputExp
is None:
1867 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1870 if inputExp.getDetector()
is None:
1871 inputExp.setDetector(camera[detectorNum])
1876 """Convert exposure image from uint16 to float.
1878 If the exposure does not need to be converted, the input is
1879 immediately returned. For exposures that are converted to use
1880 floating point pixels, the variance is set to unity and the
1885 exposure : `lsst.afw.image.Exposure`
1886 The raw exposure to be converted.
1890 newexposure : `lsst.afw.image.Exposure`
1891 The input ``exposure``, converted to floating point pixels.
1896 Raised if the exposure type cannot be converted to float.
1899 if isinstance(exposure, afwImage.ExposureF):
1901 self.log.debug(
"Exposure already of type float.")
1903 if not hasattr(exposure,
"convertF"):
1904 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1906 newexposure = exposure.convertF()
1907 newexposure.variance[:] = 1
1908 newexposure.mask[:] = 0x0
1913 """Identify bad amplifiers, saturated and suspect pixels.
1917 ccdExposure : `lsst.afw.image.Exposure`
1918 Input exposure to be masked.
1919 amp : `lsst.afw.table.AmpInfoCatalog`
1920 Catalog of parameters defining the amplifier on this
1922 defects : `lsst.ip.isr.Defects`
1923 List of defects. Used to determine if the entire
1929 If this is true, the entire amplifier area is covered by
1930 defects and unusable.
1933 maskedImage = ccdExposure.getMaskedImage()
1939 if defects
is not None:
1940 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1945 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1947 maskView = dataView.getMask()
1948 maskView |= maskView.getPlaneBitMask(
"BAD")
1955 if self.config.doSaturation
and not badAmp:
1956 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1957 if self.config.doSuspect
and not badAmp:
1958 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1959 if math.isfinite(self.config.saturation):
1960 limits.update({self.config.saturatedMaskName: self.config.saturation})
1962 for maskName, maskThreshold
in limits.items():
1963 if not math.isnan(maskThreshold):
1964 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1965 isrFunctions.makeThresholdMask(
1966 maskedImage=dataView,
1967 threshold=maskThreshold,
1973 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1975 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1976 self.config.suspectMaskName])
1977 if numpy.all(maskView.getArray() & maskVal > 0):
1979 maskView |= maskView.getPlaneBitMask(
"BAD")
1984 """Apply overscan correction in place.
1986 This method does initial pixel rejection of the overscan
1987 region. The overscan can also be optionally segmented to
1988 allow for discontinuous overscan responses to be fit
1989 separately. The actual overscan subtraction is performed by
1990 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1991 which is called here after the amplifier is preprocessed.
1995 ccdExposure : `lsst.afw.image.Exposure`
1996 Exposure to have overscan correction performed.
1997 amp : `lsst.afw.cameraGeom.Amplifer`
1998 The amplifier to consider while correcting the overscan.
2002 overscanResults : `lsst.pipe.base.Struct`
2003 Result struct with components:
2004 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2005 Value or fit subtracted from the amplifier image data.
2006 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2007 Value or fit subtracted from the overscan image data.
2008 - ``overscanImage`` : `lsst.afw.image.Image`
2009 Image of the overscan region with the overscan
2010 correction applied. This quantity is used to estimate
2011 the amplifier read noise empirically.
2016 Raised if the ``amp`` does not contain raw pixel information.
2020 lsst.ip.isr.isrFunctions.overscanCorrection
2022 if amp.getRawHorizontalOverscanBBox().isEmpty():
2023 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2026 statControl = afwMath.StatisticsControl()
2027 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2030 dataBBox = amp.getRawDataBBox()
2031 oscanBBox = amp.getRawHorizontalOverscanBBox()
2035 prescanBBox = amp.getRawPrescanBBox()
2036 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2037 dx0 += self.config.overscanNumLeadingColumnsToSkip
2038 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2040 dx0 += self.config.overscanNumTrailingColumnsToSkip
2041 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2047 if ((self.config.overscanBiasJump
2048 and self.config.overscanBiasJumpLocation)
2049 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2050 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2051 self.config.overscanBiasJumpDevices)):
2052 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2053 yLower = self.config.overscanBiasJumpLocation
2054 yUpper = dataBBox.getHeight() - yLower
2056 yUpper = self.config.overscanBiasJumpLocation
2057 yLower = dataBBox.getHeight() - yUpper
2075 oscanBBox.getHeight())))
2078 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2079 ampImage = ccdExposure.maskedImage[imageBBox]
2080 overscanImage = ccdExposure.maskedImage[overscanBBox]
2082 overscanArray = overscanImage.image.array
2083 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2084 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2085 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2087 statControl = afwMath.StatisticsControl()
2088 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2090 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2093 levelStat = afwMath.MEDIAN
2094 sigmaStat = afwMath.STDEVCLIP
2096 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2097 self.config.qa.flatness.nIter)
2098 metadata = ccdExposure.getMetadata()
2099 ampNum = amp.getName()
2101 if isinstance(overscanResults.overscanFit, float):
2102 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2103 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2105 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2106 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2107 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2109 return overscanResults
2112 """Set the variance plane using the gain and read noise
2114 The read noise is calculated from the ``overscanImage`` if the
2115 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2116 the value from the amplifier data is used.
2120 ampExposure : `lsst.afw.image.Exposure`
2121 Exposure to process.
2122 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2123 Amplifier detector data.
2124 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2125 Image of overscan, required only for empirical read noise.
2126 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2127 PTC dataset containing the gains and read noise.
2133 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2134 are ``True``, but ptcDataset is not provided.
2136 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2137 ``overscanImage`` is ``None``.
2141 lsst.ip.isr.isrFunctions.updateVariance
2143 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2144 if self.config.usePtcGains:
2145 if ptcDataset
is None:
2146 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2148 gain = ptcDataset.gain[amp.getName()]
2149 self.log.info(
"Using gain from Photon Transfer Curve.")
2151 gain = amp.getGain()
2153 if math.isnan(gain):
2155 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2158 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2159 amp.getName(), gain, patchedGain)
2162 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2163 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2165 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2166 stats = afwMath.StatisticsControl()
2167 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2168 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2169 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2170 amp.getName(), readNoise)
2171 elif self.config.usePtcReadNoise:
2172 if ptcDataset
is None:
2173 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2175 readNoise = ptcDataset.noise[amp.getName()]
2176 self.log.info(
"Using read noise from Photon Transfer Curve.")
2178 readNoise = amp.getReadNoise()
2180 isrFunctions.updateVariance(
2181 maskedImage=ampExposure.getMaskedImage(),
2183 readNoise=readNoise,
2187 """Apply dark correction in place.
2191 exposure : `lsst.afw.image.Exposure`
2192 Exposure to process.
2193 darkExposure : `lsst.afw.image.Exposure`
2194 Dark exposure of the same size as ``exposure``.
2195 invert : `Bool`, optional
2196 If True, re-add the dark to an already corrected image.
2201 Raised if either ``exposure`` or ``darkExposure`` do not
2202 have their dark time defined.
2206 lsst.ip.isr.isrFunctions.darkCorrection
2208 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2209 if math.isnan(expScale):
2210 raise RuntimeError(
"Exposure darktime is NAN.")
2211 if darkExposure.getInfo().getVisitInfo()
is not None \
2212 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2213 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2217 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2220 isrFunctions.darkCorrection(
2221 maskedImage=exposure.getMaskedImage(),
2222 darkMaskedImage=darkExposure.getMaskedImage(),
2224 darkScale=darkScale,
2226 trimToFit=self.config.doTrimToMatchCalib
2230 """Check if linearization is needed for the detector cameraGeom.
2232 Checks config.doLinearize and the linearity type of the first
2237 detector : `lsst.afw.cameraGeom.Detector`
2238 Detector to get linearity type from.
2242 doLinearize : `Bool`
2243 If True, linearization should be performed.
2245 return self.config.doLinearize
and \
2246 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2249 """Apply flat correction in place.
2253 exposure : `lsst.afw.image.Exposure`
2254 Exposure to process.
2255 flatExposure : `lsst.afw.image.Exposure`
2256 Flat exposure of the same size as ``exposure``.
2257 invert : `Bool`, optional
2258 If True, unflatten an already flattened image.
2262 lsst.ip.isr.isrFunctions.flatCorrection
2264 isrFunctions.flatCorrection(
2265 maskedImage=exposure.getMaskedImage(),
2266 flatMaskedImage=flatExposure.getMaskedImage(),
2267 scalingType=self.config.flatScalingType,
2268 userScale=self.config.flatUserScale,
2270 trimToFit=self.config.doTrimToMatchCalib
2274 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2278 exposure : `lsst.afw.image.Exposure`
2279 Exposure to process. Only the amplifier DataSec is processed.
2280 amp : `lsst.afw.table.AmpInfoCatalog`
2281 Amplifier detector data.
2285 lsst.ip.isr.isrFunctions.makeThresholdMask
2287 if not math.isnan(amp.getSaturation()):
2288 maskedImage = exposure.getMaskedImage()
2289 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2290 isrFunctions.makeThresholdMask(
2291 maskedImage=dataView,
2292 threshold=amp.getSaturation(),
2294 maskName=self.config.saturatedMaskName,
2298 """Interpolate over saturated pixels, in place.
2300 This method should be called after `saturationDetection`, to
2301 ensure that the saturated pixels have been identified in the
2302 SAT mask. It should also be called after `assembleCcd`, since
2303 saturated regions may cross amplifier boundaries.
2307 exposure : `lsst.afw.image.Exposure`
2308 Exposure to process.
2312 lsst.ip.isr.isrTask.saturationDetection
2313 lsst.ip.isr.isrFunctions.interpolateFromMask
2315 isrFunctions.interpolateFromMask(
2316 maskedImage=exposure.getMaskedImage(),
2317 fwhm=self.config.fwhm,
2318 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2319 maskNameList=list(self.config.saturatedMaskName),
2323 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2327 exposure : `lsst.afw.image.Exposure`
2328 Exposure to process. Only the amplifier DataSec is processed.
2329 amp : `lsst.afw.table.AmpInfoCatalog`
2330 Amplifier detector data.
2334 lsst.ip.isr.isrFunctions.makeThresholdMask
2338 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2339 This is intended to indicate pixels that may be affected by unknown systematics;
2340 for example if non-linearity corrections above a certain level are unstable
2341 then that would be a useful value for suspectLevel. A value of `nan` indicates
2342 that no such level exists and no pixels are to be masked as suspicious.
2344 suspectLevel = amp.getSuspectLevel()
2345 if math.isnan(suspectLevel):
2348 maskedImage = exposure.getMaskedImage()
2349 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2350 isrFunctions.makeThresholdMask(
2351 maskedImage=dataView,
2352 threshold=suspectLevel,
2354 maskName=self.config.suspectMaskName,
2358 """Mask defects using mask plane "BAD", in place.
2362 exposure : `lsst.afw.image.Exposure`
2363 Exposure to process.
2364 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2365 `lsst.afw.image.DefectBase`.
2366 List of defects to mask.
2370 Call this after CCD assembly, since defects may cross amplifier boundaries.
2372 maskedImage = exposure.getMaskedImage()
2373 if not isinstance(defectBaseList, Defects):
2375 defectList =
Defects(defectBaseList)
2377 defectList = defectBaseList
2378 defectList.maskPixels(maskedImage, maskName=
"BAD")
2380 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2381 """Mask edge pixels with applicable mask plane.
2385 exposure : `lsst.afw.image.Exposure`
2386 Exposure to process.
2387 numEdgePixels : `int`, optional
2388 Number of edge pixels to mask.
2389 maskPlane : `str`, optional
2390 Mask plane name to use.
2391 level : `str`, optional
2392 Level at which to mask edges.
2394 maskedImage = exposure.getMaskedImage()
2395 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2397 if numEdgePixels > 0:
2398 if level ==
'DETECTOR':
2399 boxes = [maskedImage.getBBox()]
2400 elif level ==
'AMP':
2401 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2405 subImage = maskedImage[box]
2406 box.grow(-numEdgePixels)
2408 SourceDetectionTask.setEdgeBits(
2414 """Mask and interpolate defects using mask plane "BAD", in place.
2418 exposure : `lsst.afw.image.Exposure`
2419 Exposure to process.
2420 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2421 `lsst.afw.image.DefectBase`.
2422 List of defects to mask and interpolate.
2426 lsst.ip.isr.isrTask.maskDefect
2428 self.
maskDefectmaskDefect(exposure, defectBaseList)
2429 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2430 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2431 isrFunctions.interpolateFromMask(
2432 maskedImage=exposure.getMaskedImage(),
2433 fwhm=self.config.fwhm,
2434 growSaturatedFootprints=0,
2435 maskNameList=[
"BAD"],
2439 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2443 exposure : `lsst.afw.image.Exposure`
2444 Exposure to process.
2448 We mask over all non-finite values (NaN, inf), including those
2449 that are masked with other bits (because those may or may not be
2450 interpolated over later, and we want to remove all NaN/infs).
2451 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2452 preserve the historical name.
2454 maskedImage = exposure.getMaskedImage()
2457 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2458 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2459 numNans =
maskNans(maskedImage, maskVal)
2460 self.metadata.set(
"NUMNANS", numNans)
2462 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2465 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2470 exposure : `lsst.afw.image.Exposure`
2471 Exposure to process.
2475 lsst.ip.isr.isrTask.maskNan
2478 isrFunctions.interpolateFromMask(
2479 maskedImage=exposure.getMaskedImage(),
2480 fwhm=self.config.fwhm,
2481 growSaturatedFootprints=0,
2482 maskNameList=[
"UNMASKEDNAN"],
2486 """Measure the image background in subgrids, for quality control purposes.
2490 exposure : `lsst.afw.image.Exposure`
2491 Exposure to process.
2492 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2493 Configuration object containing parameters on which background
2494 statistics and subgrids to use.
2496 if IsrQaConfig
is not None:
2497 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2498 IsrQaConfig.flatness.nIter)
2499 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2500 statsControl.setAndMask(maskVal)
2501 maskedImage = exposure.getMaskedImage()
2502 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2503 skyLevel = stats.getValue(afwMath.MEDIAN)
2504 skySigma = stats.getValue(afwMath.STDEVCLIP)
2505 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2506 metadata = exposure.getMetadata()
2507 metadata.set(
'SKYLEVEL', skyLevel)
2508 metadata.set(
'SKYSIGMA', skySigma)
2511 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2512 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2513 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2514 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2515 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2516 skyLevels = numpy.zeros((nX, nY))
2519 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2521 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2523 xLLC = xc - meshXHalf
2524 yLLC = yc - meshYHalf
2525 xURC = xc + meshXHalf - 1
2526 yURC = yc + meshYHalf - 1
2529 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2531 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2533 good = numpy.where(numpy.isfinite(skyLevels))
2534 skyMedian = numpy.median(skyLevels[good])
2535 flatness = (skyLevels[good] - skyMedian) / skyMedian
2536 flatness_rms = numpy.std(flatness)
2537 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2539 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2540 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2541 nX, nY, flatness_pp, flatness_rms)
2543 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2544 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2545 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2546 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2547 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2550 """Set an approximate magnitude zero point for the exposure.
2554 exposure : `lsst.afw.image.Exposure`
2555 Exposure to process.
2557 filterLabel = exposure.getFilterLabel()
2558 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2560 if physicalFilter
in self.config.fluxMag0T1:
2561 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2563 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2564 fluxMag0 = self.config.defaultFluxMag0T1
2566 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2568 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2571 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2572 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2573 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2576 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2580 ccdExposure : `lsst.afw.image.Exposure`
2581 Exposure to process.
2582 fpPolygon : `lsst.afw.geom.Polygon`
2583 Polygon in focal plane coordinates.
2586 ccd = ccdExposure.getDetector()
2587 fpCorners = ccd.getCorners(FOCAL_PLANE)
2588 ccdPolygon = Polygon(fpCorners)
2591 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2594 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2595 validPolygon = Polygon(ccdPoints)
2596 ccdExposure.getInfo().setValidPolygon(validPolygon)
2600 """Context manager that applies and removes flats and darks,
2601 if the task is configured to apply them.
2605 exp : `lsst.afw.image.Exposure`
2606 Exposure to process.
2607 flat : `lsst.afw.image.Exposure`
2608 Flat exposure the same size as ``exp``.
2609 dark : `lsst.afw.image.Exposure`, optional
2610 Dark exposure the same size as ``exp``.
2614 exp : `lsst.afw.image.Exposure`
2615 The flat and dark corrected exposure.
2617 if self.config.doDark
and dark
is not None:
2619 if self.config.doFlat:
2624 if self.config.doFlat:
2626 if self.config.doDark
and dark
is not None:
2630 """Utility function to examine ISR exposure at different stages.
2634 exposure : `lsst.afw.image.Exposure`
2637 State of processing to view.
2639 frame = getDebugFrame(self._display, stepname)
2641 display = getDisplay(frame)
2642 display.scale(
'asinh',
'zscale')
2643 display.mtv(exposure)
2644 prompt =
"Press Enter to continue [c]... "
2646 ans = input(prompt).lower()
2647 if ans
in (
"",
"c",):
2652 """A Detector-like object that supports returning gain and saturation level
2654 This is used when the input exposure does not have a detector.
2658 exposure : `lsst.afw.image.Exposure`
2659 Exposure to generate a fake amplifier for.
2660 config : `lsst.ip.isr.isrTaskConfig`
2661 Configuration to apply to the fake amplifier.
2665 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2667 self.
_gain_gain = config.gain
2672 return self.
_bbox_bbox
2675 return self.
_bbox_bbox
2681 return self.
_gain_gain
2694 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2698 """Task to wrap the default IsrTask to allow it to be retargeted.
2700 The standard IsrTask can be called directly from a command line
2701 program, but doing so removes the ability of the task to be
2702 retargeted. As most cameras override some set of the IsrTask
2703 methods, this would remove those data-specific methods in the
2704 output post-ISR images. This wrapping class fixes the issue,
2705 allowing identical post-ISR images to be generated by both the
2706 processCcd and isrTask code.
2708 ConfigClass = RunIsrConfig
2709 _DefaultName =
"runIsr"
2713 self.makeSubtask(
"isr")
2719 dataRef : `lsst.daf.persistence.ButlerDataRef`
2720 data reference of the detector data to be processed
2724 result : `pipeBase.Struct`
2725 Result struct with component:
2727 - exposure : `lsst.afw.image.Exposure`
2728 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def ensureExposure(self, inputExp, camera, detectorNum)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.