29 import lsst.pipe.base
as pipeBase
30 import lsst.pipe.base.connectionTypes
as cT
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
134 bias = cT.PrerequisiteInput(
136 doc=
"Input bias calibration.",
137 storageClass=
"ExposureF",
138 dimensions=[
"instrument",
"detector"],
141 dark = cT.PrerequisiteInput(
143 doc=
"Input dark calibration.",
144 storageClass=
"ExposureF",
145 dimensions=[
"instrument",
"detector"],
148 flat = cT.PrerequisiteInput(
150 doc=
"Input flat calibration.",
151 storageClass=
"ExposureF",
152 dimensions=[
"instrument",
"physical_filter",
"detector"],
155 ptc = cT.PrerequisiteInput(
157 doc=
"Input Photon Transfer Curve dataset",
158 storageClass=
"PhotonTransferCurveDataset",
159 dimensions=[
"instrument",
"detector"],
162 fringes = cT.PrerequisiteInput(
164 doc=
"Input fringe calibration.",
165 storageClass=
"ExposureF",
166 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 strayLightData = cT.PrerequisiteInput(
171 doc=
"Input stray light calibration.",
172 storageClass=
"StrayLightData",
173 dimensions=[
"instrument",
"physical_filter",
"detector"],
176 bfKernel = cT.PrerequisiteInput(
178 doc=
"Input brighter-fatter kernel.",
179 storageClass=
"NumpyArray",
180 dimensions=[
"instrument"],
183 newBFKernel = cT.PrerequisiteInput(
184 name=
'brighterFatterKernel',
185 doc=
"Newer complete kernel + gain solutions.",
186 storageClass=
"BrighterFatterKernel",
187 dimensions=[
"instrument",
"detector"],
190 defects = cT.PrerequisiteInput(
192 doc=
"Input defect tables.",
193 storageClass=
"Defects",
194 dimensions=[
"instrument",
"detector"],
197 linearizer = cT.PrerequisiteInput(
199 storageClass=
"Linearizer",
200 doc=
"Linearity correction calibration.",
201 dimensions=[
"instrument",
"detector"],
204 opticsTransmission = cT.PrerequisiteInput(
205 name=
"transmission_optics",
206 storageClass=
"TransmissionCurve",
207 doc=
"Transmission curve due to the optics.",
208 dimensions=[
"instrument"],
211 filterTransmission = cT.PrerequisiteInput(
212 name=
"transmission_filter",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the filter.",
215 dimensions=[
"instrument",
"physical_filter"],
218 sensorTransmission = cT.PrerequisiteInput(
219 name=
"transmission_sensor",
220 storageClass=
"TransmissionCurve",
221 doc=
"Transmission curve due to the sensor.",
222 dimensions=[
"instrument",
"detector"],
225 atmosphereTransmission = cT.PrerequisiteInput(
226 name=
"transmission_atmosphere",
227 storageClass=
"TransmissionCurve",
228 doc=
"Transmission curve due to the atmosphere.",
229 dimensions=[
"instrument"],
232 illumMaskedImage = cT.PrerequisiteInput(
234 doc=
"Input illumination correction.",
235 storageClass=
"MaskedImageF",
236 dimensions=[
"instrument",
"physical_filter",
"detector"],
240 outputExposure = cT.Output(
242 doc=
"Output ISR processed exposure.",
243 storageClass=
"Exposure",
244 dimensions=[
"instrument",
"exposure",
"detector"],
246 preInterpExposure = cT.Output(
247 name=
'preInterpISRCCD',
248 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
249 storageClass=
"ExposureF",
250 dimensions=[
"instrument",
"exposure",
"detector"],
252 outputOssThumbnail = cT.Output(
254 doc=
"Output Overscan-subtracted thumbnail image.",
255 storageClass=
"Thumbnail",
256 dimensions=[
"instrument",
"exposure",
"detector"],
258 outputFlattenedThumbnail = cT.Output(
259 name=
"FlattenedThumb",
260 doc=
"Output flat-corrected thumbnail image.",
261 storageClass=
"Thumbnail",
262 dimensions=[
"instrument",
"exposure",
"detector"],
268 if config.doBias
is not True:
269 self.prerequisiteInputs.discard(
"bias")
270 if config.doLinearize
is not True:
271 self.prerequisiteInputs.discard(
"linearizer")
272 if config.doCrosstalk
is not True:
273 self.inputs.discard(
"crosstalkSources")
274 self.prerequisiteInputs.discard(
"crosstalk")
275 if config.doBrighterFatter
is not True:
276 self.prerequisiteInputs.discard(
"bfKernel")
277 self.prerequisiteInputs.discard(
"newBFKernel")
278 if config.doDefect
is not True:
279 self.prerequisiteInputs.discard(
"defects")
280 if config.doDark
is not True:
281 self.prerequisiteInputs.discard(
"dark")
282 if config.doFlat
is not True:
283 self.prerequisiteInputs.discard(
"flat")
284 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
285 self.prerequisiteInputs.discard(
"ptc")
286 if config.doAttachTransmissionCurve
is not True:
287 self.prerequisiteInputs.discard(
"opticsTransmission")
288 self.prerequisiteInputs.discard(
"filterTransmission")
289 self.prerequisiteInputs.discard(
"sensorTransmission")
290 self.prerequisiteInputs.discard(
"atmosphereTransmission")
291 if config.doUseOpticsTransmission
is not True:
292 self.prerequisiteInputs.discard(
"opticsTransmission")
293 if config.doUseFilterTransmission
is not True:
294 self.prerequisiteInputs.discard(
"filterTransmission")
295 if config.doUseSensorTransmission
is not True:
296 self.prerequisiteInputs.discard(
"sensorTransmission")
297 if config.doUseAtmosphereTransmission
is not True:
298 self.prerequisiteInputs.discard(
"atmosphereTransmission")
299 if config.doIlluminationCorrection
is not True:
300 self.prerequisiteInputs.discard(
"illumMaskedImage")
302 if config.doWrite
is not True:
303 self.outputs.discard(
"outputExposure")
304 self.outputs.discard(
"preInterpExposure")
305 self.outputs.discard(
"outputFlattenedThumbnail")
306 self.outputs.discard(
"outputOssThumbnail")
307 if config.doSaveInterpPixels
is not True:
308 self.outputs.discard(
"preInterpExposure")
309 if config.qa.doThumbnailOss
is not True:
310 self.outputs.discard(
"outputOssThumbnail")
311 if config.qa.doThumbnailFlattened
is not True:
312 self.outputs.discard(
"outputFlattenedThumbnail")
316 pipelineConnections=IsrTaskConnections):
317 """Configuration parameters for IsrTask.
319 Items are grouped in the order in which they are executed by the task.
321 datasetType = pexConfig.Field(
323 doc=
"Dataset type for input data; users will typically leave this alone, "
324 "but camera-specific ISR tasks will override it",
328 fallbackFilterName = pexConfig.Field(
330 doc=
"Fallback default filter name for calibrations.",
333 useFallbackDate = pexConfig.Field(
335 doc=
"Pass observation date when using fallback filter.",
338 expectWcs = pexConfig.Field(
341 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
343 fwhm = pexConfig.Field(
345 doc=
"FWHM of PSF in arcseconds.",
348 qa = pexConfig.ConfigField(
350 doc=
"QA related configuration options.",
354 doConvertIntToFloat = pexConfig.Field(
356 doc=
"Convert integer raw images to floating point values?",
361 doSaturation = pexConfig.Field(
363 doc=
"Mask saturated pixels? NB: this is totally independent of the"
364 " interpolation option - this is ONLY setting the bits in the mask."
365 " To have them interpolated make sure doSaturationInterpolation=True",
368 saturatedMaskName = pexConfig.Field(
370 doc=
"Name of mask plane to use in saturation detection and interpolation",
373 saturation = pexConfig.Field(
375 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
376 default=float(
"NaN"),
378 growSaturationFootprintSize = pexConfig.Field(
380 doc=
"Number of pixels by which to grow the saturation footprints",
385 doSuspect = pexConfig.Field(
387 doc=
"Mask suspect pixels?",
390 suspectMaskName = pexConfig.Field(
392 doc=
"Name of mask plane to use for suspect pixels",
395 numEdgeSuspect = pexConfig.Field(
397 doc=
"Number of edge pixels to be flagged as untrustworthy.",
400 edgeMaskLevel = pexConfig.ChoiceField(
402 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
405 'DETECTOR':
'Mask only the edges of the full detector.',
406 'AMP':
'Mask edges of each amplifier.',
411 doSetBadRegions = pexConfig.Field(
413 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
416 badStatistic = pexConfig.ChoiceField(
418 doc=
"How to estimate the average value for BAD regions.",
421 "MEANCLIP":
"Correct using the (clipped) mean of good data",
422 "MEDIAN":
"Correct using the median of the good data",
427 doOverscan = pexConfig.Field(
429 doc=
"Do overscan subtraction?",
432 overscan = pexConfig.ConfigurableField(
433 target=OverscanCorrectionTask,
434 doc=
"Overscan subtraction task for image segments.",
437 overscanFitType = pexConfig.ChoiceField(
439 doc=
"The method for fitting the overscan bias level.",
442 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
443 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
444 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
445 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
446 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
447 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
448 "MEAN":
"Correct using the mean of the overscan region",
449 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
450 "MEDIAN":
"Correct using the median of the overscan region",
451 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
453 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
454 " This option will no longer be used, and will be removed after v20.")
456 overscanOrder = pexConfig.Field(
458 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
459 "or number of spline knots if overscan fit type is a spline."),
461 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
462 " This option will no longer be used, and will be removed after v20.")
464 overscanNumSigmaClip = pexConfig.Field(
466 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
468 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
469 " This option will no longer be used, and will be removed after v20.")
471 overscanIsInt = pexConfig.Field(
473 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
474 " and overscan.FitType=MEDIAN_PER_ROW.",
476 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
477 " This option will no longer be used, and will be removed after v20.")
480 overscanNumLeadingColumnsToSkip = pexConfig.Field(
482 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
485 overscanNumTrailingColumnsToSkip = pexConfig.Field(
487 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
490 overscanMaxDev = pexConfig.Field(
492 doc=
"Maximum deviation from the median for overscan",
493 default=1000.0, check=
lambda x: x > 0
495 overscanBiasJump = pexConfig.Field(
497 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
500 overscanBiasJumpKeyword = pexConfig.Field(
502 doc=
"Header keyword containing information about devices.",
503 default=
"NO_SUCH_KEY",
505 overscanBiasJumpDevices = pexConfig.ListField(
507 doc=
"List of devices that need piecewise overscan correction.",
510 overscanBiasJumpLocation = pexConfig.Field(
512 doc=
"Location of bias jump along y-axis.",
517 doAssembleCcd = pexConfig.Field(
520 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
522 assembleCcd = pexConfig.ConfigurableField(
523 target=AssembleCcdTask,
524 doc=
"CCD assembly task",
528 doAssembleIsrExposures = pexConfig.Field(
531 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
533 doTrimToMatchCalib = pexConfig.Field(
536 doc=
"Trim raw data to match calibration bounding boxes?"
540 doBias = pexConfig.Field(
542 doc=
"Apply bias frame correction?",
545 biasDataProductName = pexConfig.Field(
547 doc=
"Name of the bias data product",
550 doBiasBeforeOverscan = pexConfig.Field(
552 doc=
"Reverse order of overscan and bias correction.",
557 doVariance = pexConfig.Field(
559 doc=
"Calculate variance?",
562 gain = pexConfig.Field(
564 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
565 default=float(
"NaN"),
567 readNoise = pexConfig.Field(
569 doc=
"The read noise to use if no Detector is present in the Exposure",
572 doEmpiricalReadNoise = pexConfig.Field(
575 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
577 usePtcReadNoise = pexConfig.Field(
580 doc=
"Use readnoise values from the Photon Transfer Curve?"
583 doLinearize = pexConfig.Field(
585 doc=
"Correct for nonlinearity of the detector's response?",
590 doCrosstalk = pexConfig.Field(
592 doc=
"Apply intra-CCD crosstalk correction?",
595 doCrosstalkBeforeAssemble = pexConfig.Field(
597 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
600 crosstalk = pexConfig.ConfigurableField(
601 target=CrosstalkTask,
602 doc=
"Intra-CCD crosstalk correction",
606 doDefect = pexConfig.Field(
608 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
611 doNanMasking = pexConfig.Field(
613 doc=
"Mask non-finite (NAN, inf) pixels?",
616 doWidenSaturationTrails = pexConfig.Field(
618 doc=
"Widen bleed trails based on their width?",
623 doBrighterFatter = pexConfig.Field(
626 doc=
"Apply the brighter-fatter correction?"
628 brighterFatterLevel = pexConfig.ChoiceField(
631 doc=
"The level at which to correct for brighter-fatter.",
633 "AMP":
"Every amplifier treated separately.",
634 "DETECTOR":
"One kernel per detector",
637 brighterFatterMaxIter = pexConfig.Field(
640 doc=
"Maximum number of iterations for the brighter-fatter correction"
642 brighterFatterThreshold = pexConfig.Field(
645 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
646 "absolute value of the difference between the current corrected image and the one "
647 "from the previous iteration summed over all the pixels."
649 brighterFatterApplyGain = pexConfig.Field(
652 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
654 brighterFatterMaskListToInterpolate = pexConfig.ListField(
656 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
658 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
660 brighterFatterMaskGrowSize = pexConfig.Field(
663 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
664 "when brighter-fatter correction is applied."
668 doDark = pexConfig.Field(
670 doc=
"Apply dark frame correction?",
673 darkDataProductName = pexConfig.Field(
675 doc=
"Name of the dark data product",
680 doStrayLight = pexConfig.Field(
682 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
685 strayLight = pexConfig.ConfigurableField(
686 target=StrayLightTask,
687 doc=
"y-band stray light correction"
691 doFlat = pexConfig.Field(
693 doc=
"Apply flat field correction?",
696 flatDataProductName = pexConfig.Field(
698 doc=
"Name of the flat data product",
701 flatScalingType = pexConfig.ChoiceField(
703 doc=
"The method for scaling the flat on the fly.",
706 "USER":
"Scale by flatUserScale",
707 "MEAN":
"Scale by the inverse of the mean",
708 "MEDIAN":
"Scale by the inverse of the median",
711 flatUserScale = pexConfig.Field(
713 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
716 doTweakFlat = pexConfig.Field(
718 doc=
"Tweak flats to match observed amplifier ratios?",
723 doApplyGains = pexConfig.Field(
725 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
728 usePtcGains = pexConfig.Field(
730 doc=
"Use the gain values from the Photon Transfer Curve?",
733 normalizeGains = pexConfig.Field(
735 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
740 doFringe = pexConfig.Field(
742 doc=
"Apply fringe correction?",
745 fringe = pexConfig.ConfigurableField(
747 doc=
"Fringe subtraction task",
749 fringeAfterFlat = pexConfig.Field(
751 doc=
"Do fringe subtraction after flat-fielding?",
756 doMeasureBackground = pexConfig.Field(
758 doc=
"Measure the background level on the reduced image?",
763 doCameraSpecificMasking = pexConfig.Field(
765 doc=
"Mask camera-specific bad regions?",
768 masking = pexConfig.ConfigurableField(
775 doInterpolate = pexConfig.Field(
777 doc=
"Interpolate masked pixels?",
780 doSaturationInterpolation = pexConfig.Field(
782 doc=
"Perform interpolation over pixels masked as saturated?"
783 " NB: This is independent of doSaturation; if that is False this plane"
784 " will likely be blank, resulting in a no-op here.",
787 doNanInterpolation = pexConfig.Field(
789 doc=
"Perform interpolation over pixels masked as NaN?"
790 " NB: This is independent of doNanMasking; if that is False this plane"
791 " will likely be blank, resulting in a no-op here.",
794 doNanInterpAfterFlat = pexConfig.Field(
796 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
797 "also have to interpolate them before flat-fielding."),
800 maskListToInterpolate = pexConfig.ListField(
802 doc=
"List of mask planes that should be interpolated.",
803 default=[
'SAT',
'BAD'],
805 doSaveInterpPixels = pexConfig.Field(
807 doc=
"Save a copy of the pre-interpolated pixel values?",
812 fluxMag0T1 = pexConfig.DictField(
815 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
816 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
819 defaultFluxMag0T1 = pexConfig.Field(
821 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
822 default=pow(10.0, 0.4*28.0)
826 doVignette = pexConfig.Field(
828 doc=
"Apply vignetting parameters?",
831 vignette = pexConfig.ConfigurableField(
833 doc=
"Vignetting task.",
837 doAttachTransmissionCurve = pexConfig.Field(
840 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
842 doUseOpticsTransmission = pexConfig.Field(
845 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
847 doUseFilterTransmission = pexConfig.Field(
850 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
852 doUseSensorTransmission = pexConfig.Field(
855 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
857 doUseAtmosphereTransmission = pexConfig.Field(
860 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
864 doIlluminationCorrection = pexConfig.Field(
867 doc=
"Perform illumination correction?"
869 illuminationCorrectionDataProductName = pexConfig.Field(
871 doc=
"Name of the illumination correction data product.",
874 illumScale = pexConfig.Field(
876 doc=
"Scale factor for the illumination correction.",
879 illumFilters = pexConfig.ListField(
882 doc=
"Only perform illumination correction for these filters."
886 doWrite = pexConfig.Field(
888 doc=
"Persist postISRCCD?",
895 raise ValueError(
"You may not specify both doFlat and doApplyGains")
897 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
906 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
907 """Apply common instrument signature correction algorithms to a raw frame.
909 The process for correcting imaging data is very similar from
910 camera to camera. This task provides a vanilla implementation of
911 doing these corrections, including the ability to turn certain
912 corrections off if they are not needed. The inputs to the primary
913 method, `run()`, are a raw exposure to be corrected and the
914 calibration data products. The raw input is a single chip sized
915 mosaic of all amps including overscans and other non-science
916 pixels. The method `runDataRef()` identifies and defines the
917 calibration data products, and is intended for use by a
918 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
919 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
920 subclassed for different camera, although the most camera specific
921 methods have been split into subtasks that can be redirected
924 The __init__ method sets up the subtasks for ISR processing, using
925 the defaults from `lsst.ip.isr`.
930 Positional arguments passed to the Task constructor. None used at this time.
931 kwargs : `dict`, optional
932 Keyword arguments passed on to the Task constructor. None used at this time.
934 ConfigClass = IsrTaskConfig
939 self.makeSubtask(
"assembleCcd")
940 self.makeSubtask(
"crosstalk")
941 self.makeSubtask(
"strayLight")
942 self.makeSubtask(
"fringe")
943 self.makeSubtask(
"masking")
944 self.makeSubtask(
"overscan")
945 self.makeSubtask(
"vignette")
948 inputs = butlerQC.get(inputRefs)
951 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
952 except Exception
as e:
953 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
956 inputs[
'isGen3'] =
True
958 detector = inputs[
'ccdExposure'].getDetector()
960 if self.config.doCrosstalk
is True:
963 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
964 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
965 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
967 coeffVector = (self.config.crosstalk.crosstalkValues
968 if self.config.crosstalk.useConfigCoefficients
else None)
969 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
970 inputs[
'crosstalk'] = crosstalkCalib
971 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
972 if 'crosstalkSources' not in inputs:
973 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
976 if 'linearizer' in inputs:
977 if isinstance(inputs[
'linearizer'], dict):
979 linearizer.fromYaml(inputs[
'linearizer'])
980 self.log.warn(
"Dictionary linearizers will be deprecated in DM-28741.")
981 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
985 self.log.warn(
"Bare lookup table linearizers will be deprecated in DM-28741.")
987 linearizer = inputs[
'linearizer']
988 linearizer.log = self.log
989 inputs[
'linearizer'] = linearizer
992 self.log.warn(
"Constructing linearizer from cameraGeom information.")
994 if self.config.doDefect
is True:
995 if "defects" in inputs
and inputs[
'defects']
is not None:
998 if not isinstance(inputs[
"defects"], Defects):
999 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1003 if self.config.doBrighterFatter:
1004 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1005 if brighterFatterKernel
is None:
1006 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1008 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1010 detName = detector.getName()
1011 level = brighterFatterKernel.level
1014 inputs[
'bfGains'] = brighterFatterKernel.gain
1015 if self.config.brighterFatterLevel ==
'DETECTOR':
1016 if level ==
'DETECTOR':
1017 if detName
in brighterFatterKernel.detKernels:
1018 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1020 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1021 elif level ==
'AMP':
1022 self.log.warn(
"Making DETECTOR level kernel from AMP based brighter fatter kernels.")
1023 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1024 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1025 elif self.config.brighterFatterLevel ==
'AMP':
1026 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1028 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1029 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
1030 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1032 assembler=self.assembleCcd
1033 if self.config.doAssembleIsrExposures
else None)
1035 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1037 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1038 if 'strayLightData' not in inputs:
1039 inputs[
'strayLightData'] =
None
1041 outputs = self.
runrun(**inputs)
1042 butlerQC.put(outputs, outputRefs)
1045 """Retrieve necessary frames for instrument signature removal.
1047 Pre-fetching all required ISR data products limits the IO
1048 required by the ISR. Any conflict between the calibration data
1049 available and that needed for ISR is also detected prior to
1050 doing processing, allowing it to fail quickly.
1054 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1055 Butler reference of the detector data to be processed
1056 rawExposure : `afw.image.Exposure`
1057 The raw exposure that will later be corrected with the
1058 retrieved calibration data; should not be modified in this
1063 result : `lsst.pipe.base.Struct`
1064 Result struct with components (which may be `None`):
1065 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1066 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1067 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1068 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1069 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1070 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1071 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1072 - ``fringes``: `lsst.pipe.base.Struct` with components:
1073 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1074 - ``seed``: random seed derived from the ccdExposureId for random
1075 number generator (`uint32`).
1076 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1077 A ``TransmissionCurve`` that represents the throughput of the optics,
1078 to be evaluated in focal-plane coordinates.
1079 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1080 A ``TransmissionCurve`` that represents the throughput of the filter
1081 itself, to be evaluated in focal-plane coordinates.
1082 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1083 A ``TransmissionCurve`` that represents the throughput of the sensor
1084 itself, to be evaluated in post-assembly trimmed detector coordinates.
1085 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1086 A ``TransmissionCurve`` that represents the throughput of the
1087 atmosphere, assumed to be spatially constant.
1088 - ``strayLightData`` : `object`
1089 An opaque object containing calibration information for
1090 stray-light correction. If `None`, no correction will be
1092 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1096 NotImplementedError :
1097 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1100 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1101 dateObs = dateObs.toPython().isoformat()
1102 except RuntimeError:
1103 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1106 ccd = rawExposure.getDetector()
1107 filterLabel = rawExposure.getFilterLabel()
1108 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1109 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1110 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1111 if self.config.doBias
else None)
1113 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1115 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1116 linearizer.log = self.log
1117 if isinstance(linearizer, numpy.ndarray):
1120 crosstalkCalib =
None
1121 if self.config.doCrosstalk:
1123 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1125 coeffVector = (self.config.crosstalk.crosstalkValues
1126 if self.config.crosstalk.useConfigCoefficients
else None)
1127 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1128 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1129 if self.config.doCrosstalk
else None)
1131 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1132 if self.config.doDark
else None)
1133 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1135 if self.config.doFlat
else None)
1137 brighterFatterKernel =
None
1138 brighterFatterGains =
None
1139 if self.config.doBrighterFatter
is True:
1144 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1145 brighterFatterGains = brighterFatterKernel.gain
1146 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1149 brighterFatterKernel = dataRef.get(
"bfKernel")
1150 self.log.info(
"Old style brighter-fatter kernel (np.array) loaded")
1152 brighterFatterKernel =
None
1153 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1156 if self.config.brighterFatterLevel ==
'DETECTOR':
1157 if brighterFatterKernel.detectorKernel:
1158 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1160 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1163 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1165 defectList = (dataRef.get(
"defects")
1166 if self.config.doDefect
else None)
1167 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1168 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1169 if self.config.doAssembleIsrExposures
else None)
1170 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1171 else pipeBase.Struct(fringes=
None))
1173 if self.config.doAttachTransmissionCurve:
1174 opticsTransmission = (dataRef.get(
"transmission_optics")
1175 if self.config.doUseOpticsTransmission
else None)
1176 filterTransmission = (dataRef.get(
"transmission_filter")
1177 if self.config.doUseFilterTransmission
else None)
1178 sensorTransmission = (dataRef.get(
"transmission_sensor")
1179 if self.config.doUseSensorTransmission
else None)
1180 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1181 if self.config.doUseAtmosphereTransmission
else None)
1183 opticsTransmission =
None
1184 filterTransmission =
None
1185 sensorTransmission =
None
1186 atmosphereTransmission =
None
1188 if self.config.doStrayLight:
1189 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1191 strayLightData =
None
1194 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1195 if (self.config.doIlluminationCorrection
1196 and physicalFilter
in self.config.illumFilters)
1200 return pipeBase.Struct(bias=biasExposure,
1201 linearizer=linearizer,
1202 crosstalk=crosstalkCalib,
1203 crosstalkSources=crosstalkSources,
1206 bfKernel=brighterFatterKernel,
1207 bfGains=brighterFatterGains,
1209 fringes=fringeStruct,
1210 opticsTransmission=opticsTransmission,
1211 filterTransmission=filterTransmission,
1212 sensorTransmission=sensorTransmission,
1213 atmosphereTransmission=atmosphereTransmission,
1214 strayLightData=strayLightData,
1215 illumMaskedImage=illumMaskedImage
1218 @pipeBase.timeMethod
1219 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1220 crosstalk=None, crosstalkSources=None,
1221 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1222 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1223 sensorTransmission=
None, atmosphereTransmission=
None,
1224 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1227 """Perform instrument signature removal on an exposure.
1229 Steps included in the ISR processing, in order performed, are:
1230 - saturation and suspect pixel masking
1231 - overscan subtraction
1232 - CCD assembly of individual amplifiers
1234 - variance image construction
1235 - linearization of non-linear response
1237 - brighter-fatter correction
1240 - stray light subtraction
1242 - masking of known defects and camera specific features
1243 - vignette calculation
1244 - appending transmission curve and distortion model
1248 ccdExposure : `lsst.afw.image.Exposure`
1249 The raw exposure that is to be run through ISR. The
1250 exposure is modified by this method.
1251 camera : `lsst.afw.cameraGeom.Camera`, optional
1252 The camera geometry for this exposure. Required if ``isGen3`` is
1253 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1254 ``flat`` does not have an associated detector.
1255 bias : `lsst.afw.image.Exposure`, optional
1256 Bias calibration frame.
1257 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1258 Functor for linearization.
1259 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1260 Calibration for crosstalk.
1261 crosstalkSources : `list`, optional
1262 List of possible crosstalk sources.
1263 dark : `lsst.afw.image.Exposure`, optional
1264 Dark calibration frame.
1265 flat : `lsst.afw.image.Exposure`, optional
1266 Flat calibration frame.
1267 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1268 Photon transfer curve dataset, with, e.g., gains
1270 bfKernel : `numpy.ndarray`, optional
1271 Brighter-fatter kernel.
1272 bfGains : `dict` of `float`, optional
1273 Gains used to override the detector's nominal gains for the
1274 brighter-fatter correction. A dict keyed by amplifier name for
1275 the detector in question.
1276 defects : `lsst.ip.isr.Defects`, optional
1278 fringes : `lsst.pipe.base.Struct`, optional
1279 Struct containing the fringe correction data, with
1281 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1282 - ``seed``: random seed derived from the ccdExposureId for random
1283 number generator (`uint32`)
1284 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1285 A ``TransmissionCurve`` that represents the throughput of the optics,
1286 to be evaluated in focal-plane coordinates.
1287 filterTransmission : `lsst.afw.image.TransmissionCurve`
1288 A ``TransmissionCurve`` that represents the throughput of the filter
1289 itself, to be evaluated in focal-plane coordinates.
1290 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1291 A ``TransmissionCurve`` that represents the throughput of the sensor
1292 itself, to be evaluated in post-assembly trimmed detector coordinates.
1293 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1294 A ``TransmissionCurve`` that represents the throughput of the
1295 atmosphere, assumed to be spatially constant.
1296 detectorNum : `int`, optional
1297 The integer number for the detector to process.
1298 isGen3 : bool, optional
1299 Flag this call to run() as using the Gen3 butler environment.
1300 strayLightData : `object`, optional
1301 Opaque object containing calibration information for stray-light
1302 correction. If `None`, no correction will be performed.
1303 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1304 Illumination correction image.
1308 result : `lsst.pipe.base.Struct`
1309 Result struct with component:
1310 - ``exposure`` : `afw.image.Exposure`
1311 The fully ISR corrected exposure.
1312 - ``outputExposure`` : `afw.image.Exposure`
1313 An alias for `exposure`
1314 - ``ossThumb`` : `numpy.ndarray`
1315 Thumbnail image of the exposure after overscan subtraction.
1316 - ``flattenedThumb`` : `numpy.ndarray`
1317 Thumbnail image of the exposure after flat-field correction.
1322 Raised if a configuration option is set to True, but the
1323 required calibration data has not been specified.
1327 The current processed exposure can be viewed by setting the
1328 appropriate lsstDebug entries in the `debug.display`
1329 dictionary. The names of these entries correspond to some of
1330 the IsrTaskConfig Boolean options, with the value denoting the
1331 frame to use. The exposure is shown inside the matching
1332 option check and after the processing of that step has
1333 finished. The steps with debug points are:
1344 In addition, setting the "postISRCCD" entry displays the
1345 exposure after all ISR processing has finished.
1353 if detectorNum
is None:
1354 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1356 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1357 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1358 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1359 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1361 if isinstance(ccdExposure, ButlerDataRef):
1362 return self.
runDataRefrunDataRef(ccdExposure)
1364 ccd = ccdExposure.getDetector()
1365 filterLabel = ccdExposure.getFilterLabel()
1366 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1369 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1370 ccd = [
FakeAmp(ccdExposure, self.config)]
1373 if self.config.doBias
and bias
is None:
1374 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1375 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1376 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1377 if self.config.doBrighterFatter
and bfKernel
is None:
1378 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1379 if self.config.doDark
and dark
is None:
1380 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1381 if self.config.doFlat
and flat
is None:
1382 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1383 if self.config.doDefect
and defects
is None:
1384 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1385 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1386 and fringes.fringes
is None):
1391 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1392 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1393 and illumMaskedImage
is None):
1394 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1397 if self.config.doConvertIntToFloat:
1398 self.log.info(
"Converting exposure to floating point values.")
1401 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1402 self.log.info(
"Applying bias correction.")
1403 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1404 trimToFit=self.config.doTrimToMatchCalib)
1405 self.
debugViewdebugView(ccdExposure,
"doBias")
1411 if ccdExposure.getBBox().contains(amp.getBBox()):
1413 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1415 if self.config.doOverscan
and not badAmp:
1418 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1419 if overscanResults
is not None and \
1420 self.config.qa
is not None and self.config.qa.saveStats
is True:
1421 if isinstance(overscanResults.overscanFit, float):
1422 qaMedian = overscanResults.overscanFit
1423 qaStdev = float(
"NaN")
1425 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1426 afwMath.MEDIAN | afwMath.STDEVCLIP)
1427 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1428 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1430 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1431 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1432 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1433 amp.getName(), qaMedian, qaStdev)
1436 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1437 afwMath.MEDIAN | afwMath.STDEVCLIP)
1438 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1439 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1441 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1442 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1443 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1444 amp.getName(), qaMedianAfter, qaStdevAfter)
1446 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1449 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1450 overscanResults =
None
1452 overscans.append(overscanResults
if overscanResults
is not None else None)
1454 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1456 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1457 self.log.info(
"Applying crosstalk correction.")
1458 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1459 crosstalkSources=crosstalkSources, camera=camera)
1460 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1462 if self.config.doAssembleCcd:
1463 self.log.info(
"Assembling CCD from amplifiers.")
1464 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1466 if self.config.expectWcs
and not ccdExposure.getWcs():
1467 self.log.warn(
"No WCS found in input exposure.")
1468 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1471 if self.config.qa.doThumbnailOss:
1472 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1474 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1475 self.log.info(
"Applying bias correction.")
1476 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1477 trimToFit=self.config.doTrimToMatchCalib)
1478 self.
debugViewdebugView(ccdExposure,
"doBias")
1480 if self.config.doVariance:
1481 for amp, overscanResults
in zip(ccd, overscans):
1482 if ccdExposure.getBBox().contains(amp.getBBox()):
1483 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1484 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1485 if overscanResults
is not None:
1487 overscanImage=overscanResults.overscanImage,
1493 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1494 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1495 afwMath.MEDIAN | afwMath.STDEVCLIP)
1496 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1497 qaStats.getValue(afwMath.MEDIAN))
1498 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1499 qaStats.getValue(afwMath.STDEVCLIP))
1500 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1501 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1502 qaStats.getValue(afwMath.STDEVCLIP))
1505 self.log.info(
"Applying linearizer.")
1506 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1507 detector=ccd, log=self.log)
1509 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1510 self.log.info(
"Applying crosstalk correction.")
1511 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1512 crosstalkSources=crosstalkSources, isTrimmed=
True)
1513 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1517 if self.config.doDefect:
1518 self.log.info(
"Masking defects.")
1519 self.
maskDefectmaskDefect(ccdExposure, defects)
1521 if self.config.numEdgeSuspect > 0:
1522 self.log.info(
"Masking edges as SUSPECT.")
1523 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1524 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1526 if self.config.doNanMasking:
1527 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1528 self.
maskNanmaskNan(ccdExposure)
1530 if self.config.doWidenSaturationTrails:
1531 self.log.info(
"Widening saturation trails.")
1532 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1534 if self.config.doCameraSpecificMasking:
1535 self.log.info(
"Masking regions for camera specific reasons.")
1536 self.masking.
run(ccdExposure)
1538 if self.config.doBrighterFatter:
1547 interpExp = ccdExposure.clone()
1548 with self.
flatContextflatContext(interpExp, flat, dark):
1549 isrFunctions.interpolateFromMask(
1550 maskedImage=interpExp.getMaskedImage(),
1551 fwhm=self.config.fwhm,
1552 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1553 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1555 bfExp = interpExp.clone()
1557 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1558 type(bfKernel), type(bfGains))
1559 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1560 self.config.brighterFatterMaxIter,
1561 self.config.brighterFatterThreshold,
1562 self.config.brighterFatterApplyGain,
1564 if bfResults[1] == self.config.brighterFatterMaxIter:
1565 self.log.warn(
"Brighter-fatter correction did not converge, final difference %f.",
1568 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1570 image = ccdExposure.getMaskedImage().getImage()
1571 bfCorr = bfExp.getMaskedImage().getImage()
1572 bfCorr -= interpExp.getMaskedImage().getImage()
1581 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1582 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1585 if self.config.brighterFatterMaskGrowSize > 0:
1586 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1587 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1588 isrFunctions.growMasks(ccdExposure.getMask(),
1589 radius=self.config.brighterFatterMaskGrowSize,
1590 maskNameList=maskPlane,
1591 maskValue=maskPlane)
1593 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1595 if self.config.doDark:
1596 self.log.info(
"Applying dark correction.")
1598 self.
debugViewdebugView(ccdExposure,
"doDark")
1600 if self.config.doFringe
and not self.config.fringeAfterFlat:
1601 self.log.info(
"Applying fringe correction before flat.")
1602 self.fringe.
run(ccdExposure, **fringes.getDict())
1603 self.
debugViewdebugView(ccdExposure,
"doFringe")
1605 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1606 self.log.info(
"Checking strayLight correction.")
1607 self.strayLight.
run(ccdExposure, strayLightData)
1608 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1610 if self.config.doFlat:
1611 self.log.info(
"Applying flat correction.")
1613 self.
debugViewdebugView(ccdExposure,
"doFlat")
1615 if self.config.doApplyGains:
1616 self.log.info(
"Applying gain correction instead of flat.")
1617 if self.config.usePtcGains:
1618 self.log.info(
"Using gains from the Photon Transfer Curve.")
1619 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1622 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1624 if self.config.doFringe
and self.config.fringeAfterFlat:
1625 self.log.info(
"Applying fringe correction after flat.")
1626 self.fringe.
run(ccdExposure, **fringes.getDict())
1628 if self.config.doVignette:
1629 self.log.info(
"Constructing Vignette polygon.")
1632 if self.config.vignette.doWriteVignettePolygon:
1635 if self.config.doAttachTransmissionCurve:
1636 self.log.info(
"Adding transmission curves.")
1637 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1638 filterTransmission=filterTransmission,
1639 sensorTransmission=sensorTransmission,
1640 atmosphereTransmission=atmosphereTransmission)
1642 flattenedThumb =
None
1643 if self.config.qa.doThumbnailFlattened:
1644 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1646 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1647 self.log.info(
"Performing illumination correction.")
1648 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1649 illumMaskedImage, illumScale=self.config.illumScale,
1650 trimToFit=self.config.doTrimToMatchCalib)
1653 if self.config.doSaveInterpPixels:
1654 preInterpExp = ccdExposure.clone()
1669 if self.config.doSetBadRegions:
1670 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1671 if badPixelCount > 0:
1672 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1674 if self.config.doInterpolate:
1675 self.log.info(
"Interpolating masked pixels.")
1676 isrFunctions.interpolateFromMask(
1677 maskedImage=ccdExposure.getMaskedImage(),
1678 fwhm=self.config.fwhm,
1679 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1680 maskNameList=list(self.config.maskListToInterpolate)
1685 if self.config.doMeasureBackground:
1686 self.log.info(
"Measuring background level.")
1689 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1691 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1692 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1693 afwMath.MEDIAN | afwMath.STDEVCLIP)
1694 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1695 qaStats.getValue(afwMath.MEDIAN))
1696 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1697 qaStats.getValue(afwMath.STDEVCLIP))
1698 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1699 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1700 qaStats.getValue(afwMath.STDEVCLIP))
1702 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1704 return pipeBase.Struct(
1705 exposure=ccdExposure,
1707 flattenedThumb=flattenedThumb,
1709 preInterpolatedExposure=preInterpExp,
1710 outputExposure=ccdExposure,
1711 outputOssThumbnail=ossThumb,
1712 outputFlattenedThumbnail=flattenedThumb,
1715 @pipeBase.timeMethod
1717 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1719 This method contains the `CmdLineTask` interface to the ISR
1720 processing. All IO is handled here, freeing the `run()` method
1721 to manage only pixel-level calculations. The steps performed
1723 - Read in necessary detrending/isr/calibration data.
1724 - Process raw exposure in `run()`.
1725 - Persist the ISR-corrected exposure as "postISRCCD" if
1726 config.doWrite=True.
1730 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1731 DataRef of the detector data to be processed
1735 result : `lsst.pipe.base.Struct`
1736 Result struct with component:
1737 - ``exposure`` : `afw.image.Exposure`
1738 The fully ISR corrected exposure.
1743 Raised if a configuration option is set to True, but the
1744 required calibration data does not exist.
1747 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1749 ccdExposure = sensorRef.get(self.config.datasetType)
1751 camera = sensorRef.get(
"camera")
1752 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1754 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1756 if self.config.doWrite:
1757 sensorRef.put(result.exposure,
"postISRCCD")
1758 if result.preInterpolatedExposure
is not None:
1759 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1760 if result.ossThumb
is not None:
1761 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1762 if result.flattenedThumb
is not None:
1763 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1768 """Retrieve a calibration dataset for removing instrument signature.
1773 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1774 DataRef of the detector data to find calibration datasets
1777 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1778 dateObs : `str`, optional
1779 Date of the observation. Used to correct butler failures
1780 when using fallback filters.
1782 If True, disable butler proxies to enable error handling
1783 within this routine.
1787 exposure : `lsst.afw.image.Exposure`
1788 Requested calibration frame.
1793 Raised if no matching calibration frame can be found.
1796 exp = dataRef.get(datasetType, immediate=immediate)
1797 except Exception
as exc1:
1798 if not self.config.fallbackFilterName:
1799 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1801 if self.config.useFallbackDate
and dateObs:
1802 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1803 dateObs=dateObs, immediate=immediate)
1805 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1806 except Exception
as exc2:
1807 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1808 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1809 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1811 if self.config.doAssembleIsrExposures:
1812 exp = self.assembleCcd.assembleCcd(exp)
1816 """Ensure that the data returned by Butler is a fully constructed exposure.
1818 ISR requires exposure-level image data for historical reasons, so if we did
1819 not recieve that from Butler, construct it from what we have, modifying the
1824 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1825 `lsst.afw.image.ImageF`
1826 The input data structure obtained from Butler.
1827 camera : `lsst.afw.cameraGeom.camera`
1828 The camera associated with the image. Used to find the appropriate
1831 The detector this exposure should match.
1835 inputExp : `lsst.afw.image.Exposure`
1836 The re-constructed exposure, with appropriate detector parameters.
1841 Raised if the input data cannot be used to construct an exposure.
1843 if isinstance(inputExp, afwImage.DecoratedImageU):
1844 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1845 elif isinstance(inputExp, afwImage.ImageF):
1846 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1847 elif isinstance(inputExp, afwImage.MaskedImageF):
1848 inputExp = afwImage.makeExposure(inputExp)
1849 elif isinstance(inputExp, afwImage.Exposure):
1851 elif inputExp
is None:
1855 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1858 if inputExp.getDetector()
is None:
1859 inputExp.setDetector(camera[detectorNum])
1864 """Convert exposure image from uint16 to float.
1866 If the exposure does not need to be converted, the input is
1867 immediately returned. For exposures that are converted to use
1868 floating point pixels, the variance is set to unity and the
1873 exposure : `lsst.afw.image.Exposure`
1874 The raw exposure to be converted.
1878 newexposure : `lsst.afw.image.Exposure`
1879 The input ``exposure``, converted to floating point pixels.
1884 Raised if the exposure type cannot be converted to float.
1887 if isinstance(exposure, afwImage.ExposureF):
1889 self.log.debug(
"Exposure already of type float.")
1891 if not hasattr(exposure,
"convertF"):
1892 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1894 newexposure = exposure.convertF()
1895 newexposure.variance[:] = 1
1896 newexposure.mask[:] = 0x0
1901 """Identify bad amplifiers, saturated and suspect pixels.
1905 ccdExposure : `lsst.afw.image.Exposure`
1906 Input exposure to be masked.
1907 amp : `lsst.afw.table.AmpInfoCatalog`
1908 Catalog of parameters defining the amplifier on this
1910 defects : `lsst.ip.isr.Defects`
1911 List of defects. Used to determine if the entire
1917 If this is true, the entire amplifier area is covered by
1918 defects and unusable.
1921 maskedImage = ccdExposure.getMaskedImage()
1927 if defects
is not None:
1928 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1933 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1935 maskView = dataView.getMask()
1936 maskView |= maskView.getPlaneBitMask(
"BAD")
1943 if self.config.doSaturation
and not badAmp:
1944 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1945 if self.config.doSuspect
and not badAmp:
1946 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1947 if math.isfinite(self.config.saturation):
1948 limits.update({self.config.saturatedMaskName: self.config.saturation})
1950 for maskName, maskThreshold
in limits.items():
1951 if not math.isnan(maskThreshold):
1952 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1953 isrFunctions.makeThresholdMask(
1954 maskedImage=dataView,
1955 threshold=maskThreshold,
1961 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1963 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1964 self.config.suspectMaskName])
1965 if numpy.all(maskView.getArray() & maskVal > 0):
1967 maskView |= maskView.getPlaneBitMask(
"BAD")
1972 """Apply overscan correction in place.
1974 This method does initial pixel rejection of the overscan
1975 region. The overscan can also be optionally segmented to
1976 allow for discontinuous overscan responses to be fit
1977 separately. The actual overscan subtraction is performed by
1978 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1979 which is called here after the amplifier is preprocessed.
1983 ccdExposure : `lsst.afw.image.Exposure`
1984 Exposure to have overscan correction performed.
1985 amp : `lsst.afw.cameraGeom.Amplifer`
1986 The amplifier to consider while correcting the overscan.
1990 overscanResults : `lsst.pipe.base.Struct`
1991 Result struct with components:
1992 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1993 Value or fit subtracted from the amplifier image data.
1994 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1995 Value or fit subtracted from the overscan image data.
1996 - ``overscanImage`` : `lsst.afw.image.Image`
1997 Image of the overscan region with the overscan
1998 correction applied. This quantity is used to estimate
1999 the amplifier read noise empirically.
2004 Raised if the ``amp`` does not contain raw pixel information.
2008 lsst.ip.isr.isrFunctions.overscanCorrection
2010 if amp.getRawHorizontalOverscanBBox().isEmpty():
2011 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2014 statControl = afwMath.StatisticsControl()
2015 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2018 dataBBox = amp.getRawDataBBox()
2019 oscanBBox = amp.getRawHorizontalOverscanBBox()
2023 prescanBBox = amp.getRawPrescanBBox()
2024 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2025 dx0 += self.config.overscanNumLeadingColumnsToSkip
2026 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2028 dx0 += self.config.overscanNumTrailingColumnsToSkip
2029 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2035 if ((self.config.overscanBiasJump
2036 and self.config.overscanBiasJumpLocation)
2037 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2038 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2039 self.config.overscanBiasJumpDevices)):
2040 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2041 yLower = self.config.overscanBiasJumpLocation
2042 yUpper = dataBBox.getHeight() - yLower
2044 yUpper = self.config.overscanBiasJumpLocation
2045 yLower = dataBBox.getHeight() - yUpper
2063 oscanBBox.getHeight())))
2066 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2067 ampImage = ccdExposure.maskedImage[imageBBox]
2068 overscanImage = ccdExposure.maskedImage[overscanBBox]
2070 overscanArray = overscanImage.image.array
2071 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2072 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2073 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2075 statControl = afwMath.StatisticsControl()
2076 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2078 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2081 levelStat = afwMath.MEDIAN
2082 sigmaStat = afwMath.STDEVCLIP
2084 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2085 self.config.qa.flatness.nIter)
2086 metadata = ccdExposure.getMetadata()
2087 ampNum = amp.getName()
2089 if isinstance(overscanResults.overscanFit, float):
2090 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2091 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2093 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2094 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2095 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2097 return overscanResults
2100 """Set the variance plane using the gain and read noise
2102 The read noise is calculated from the ``overscanImage`` if the
2103 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2104 the value from the amplifier data is used.
2108 ampExposure : `lsst.afw.image.Exposure`
2109 Exposure to process.
2110 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2111 Amplifier detector data.
2112 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2113 Image of overscan, required only for empirical read noise.
2114 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2115 PTC dataset containing the gains and read noise.
2121 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2122 are ``True``, but ptcDataset is not provided.
2124 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2125 ``overscanImage`` is ``None``.
2129 lsst.ip.isr.isrFunctions.updateVariance
2131 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2132 if self.config.usePtcGains:
2133 if ptcDataset
is None:
2134 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2136 gain = ptcDataset.gain[amp.getName()]
2137 self.log.info(
"Using gain from Photon Transfer Curve.")
2139 gain = amp.getGain()
2141 if math.isnan(gain):
2143 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2146 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2147 amp.getName(), gain, patchedGain)
2150 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2151 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2153 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2154 stats = afwMath.StatisticsControl()
2155 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2156 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2157 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2158 amp.getName(), readNoise)
2159 elif self.config.usePtcReadNoise:
2160 if ptcDataset
is None:
2161 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2163 readNoise = ptcDataset.noise[amp.getName()]
2164 self.log.info(
"Using read noise from Photon Transfer Curve.")
2166 readNoise = amp.getReadNoise()
2168 isrFunctions.updateVariance(
2169 maskedImage=ampExposure.getMaskedImage(),
2171 readNoise=readNoise,
2175 """Apply dark correction in place.
2179 exposure : `lsst.afw.image.Exposure`
2180 Exposure to process.
2181 darkExposure : `lsst.afw.image.Exposure`
2182 Dark exposure of the same size as ``exposure``.
2183 invert : `Bool`, optional
2184 If True, re-add the dark to an already corrected image.
2189 Raised if either ``exposure`` or ``darkExposure`` do not
2190 have their dark time defined.
2194 lsst.ip.isr.isrFunctions.darkCorrection
2196 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2197 if math.isnan(expScale):
2198 raise RuntimeError(
"Exposure darktime is NAN.")
2199 if darkExposure.getInfo().getVisitInfo()
is not None \
2200 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2201 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2205 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2208 isrFunctions.darkCorrection(
2209 maskedImage=exposure.getMaskedImage(),
2210 darkMaskedImage=darkExposure.getMaskedImage(),
2212 darkScale=darkScale,
2214 trimToFit=self.config.doTrimToMatchCalib
2218 """Check if linearization is needed for the detector cameraGeom.
2220 Checks config.doLinearize and the linearity type of the first
2225 detector : `lsst.afw.cameraGeom.Detector`
2226 Detector to get linearity type from.
2230 doLinearize : `Bool`
2231 If True, linearization should be performed.
2233 return self.config.doLinearize
and \
2234 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2237 """Apply flat correction in place.
2241 exposure : `lsst.afw.image.Exposure`
2242 Exposure to process.
2243 flatExposure : `lsst.afw.image.Exposure`
2244 Flat exposure of the same size as ``exposure``.
2245 invert : `Bool`, optional
2246 If True, unflatten an already flattened image.
2250 lsst.ip.isr.isrFunctions.flatCorrection
2252 isrFunctions.flatCorrection(
2253 maskedImage=exposure.getMaskedImage(),
2254 flatMaskedImage=flatExposure.getMaskedImage(),
2255 scalingType=self.config.flatScalingType,
2256 userScale=self.config.flatUserScale,
2258 trimToFit=self.config.doTrimToMatchCalib
2262 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2266 exposure : `lsst.afw.image.Exposure`
2267 Exposure to process. Only the amplifier DataSec is processed.
2268 amp : `lsst.afw.table.AmpInfoCatalog`
2269 Amplifier detector data.
2273 lsst.ip.isr.isrFunctions.makeThresholdMask
2275 if not math.isnan(amp.getSaturation()):
2276 maskedImage = exposure.getMaskedImage()
2277 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2278 isrFunctions.makeThresholdMask(
2279 maskedImage=dataView,
2280 threshold=amp.getSaturation(),
2282 maskName=self.config.saturatedMaskName,
2286 """Interpolate over saturated pixels, in place.
2288 This method should be called after `saturationDetection`, to
2289 ensure that the saturated pixels have been identified in the
2290 SAT mask. It should also be called after `assembleCcd`, since
2291 saturated regions may cross amplifier boundaries.
2295 exposure : `lsst.afw.image.Exposure`
2296 Exposure to process.
2300 lsst.ip.isr.isrTask.saturationDetection
2301 lsst.ip.isr.isrFunctions.interpolateFromMask
2303 isrFunctions.interpolateFromMask(
2304 maskedImage=exposure.getMaskedImage(),
2305 fwhm=self.config.fwhm,
2306 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2307 maskNameList=list(self.config.saturatedMaskName),
2311 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2315 exposure : `lsst.afw.image.Exposure`
2316 Exposure to process. Only the amplifier DataSec is processed.
2317 amp : `lsst.afw.table.AmpInfoCatalog`
2318 Amplifier detector data.
2322 lsst.ip.isr.isrFunctions.makeThresholdMask
2326 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2327 This is intended to indicate pixels that may be affected by unknown systematics;
2328 for example if non-linearity corrections above a certain level are unstable
2329 then that would be a useful value for suspectLevel. A value of `nan` indicates
2330 that no such level exists and no pixels are to be masked as suspicious.
2332 suspectLevel = amp.getSuspectLevel()
2333 if math.isnan(suspectLevel):
2336 maskedImage = exposure.getMaskedImage()
2337 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2338 isrFunctions.makeThresholdMask(
2339 maskedImage=dataView,
2340 threshold=suspectLevel,
2342 maskName=self.config.suspectMaskName,
2346 """Mask defects using mask plane "BAD", in place.
2350 exposure : `lsst.afw.image.Exposure`
2351 Exposure to process.
2352 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2353 `lsst.afw.image.DefectBase`.
2354 List of defects to mask.
2358 Call this after CCD assembly, since defects may cross amplifier boundaries.
2360 maskedImage = exposure.getMaskedImage()
2361 if not isinstance(defectBaseList, Defects):
2363 defectList =
Defects(defectBaseList)
2365 defectList = defectBaseList
2366 defectList.maskPixels(maskedImage, maskName=
"BAD")
2368 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2369 """Mask edge pixels with applicable mask plane.
2373 exposure : `lsst.afw.image.Exposure`
2374 Exposure to process.
2375 numEdgePixels : `int`, optional
2376 Number of edge pixels to mask.
2377 maskPlane : `str`, optional
2378 Mask plane name to use.
2379 level : `str`, optional
2380 Level at which to mask edges.
2382 maskedImage = exposure.getMaskedImage()
2383 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2385 if numEdgePixels > 0:
2386 if level ==
'DETECTOR':
2387 boxes = [maskedImage.getBBox()]
2388 elif level ==
'AMP':
2389 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2393 subImage = maskedImage[box]
2394 box.grow(-numEdgePixels)
2396 SourceDetectionTask.setEdgeBits(
2402 """Mask and interpolate defects using mask plane "BAD", in place.
2406 exposure : `lsst.afw.image.Exposure`
2407 Exposure to process.
2408 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2409 `lsst.afw.image.DefectBase`.
2410 List of defects to mask and interpolate.
2414 lsst.ip.isr.isrTask.maskDefect
2416 self.
maskDefectmaskDefect(exposure, defectBaseList)
2417 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2418 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2419 isrFunctions.interpolateFromMask(
2420 maskedImage=exposure.getMaskedImage(),
2421 fwhm=self.config.fwhm,
2422 growSaturatedFootprints=0,
2423 maskNameList=[
"BAD"],
2427 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2431 exposure : `lsst.afw.image.Exposure`
2432 Exposure to process.
2436 We mask over all non-finite values (NaN, inf), including those
2437 that are masked with other bits (because those may or may not be
2438 interpolated over later, and we want to remove all NaN/infs).
2439 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2440 preserve the historical name.
2442 maskedImage = exposure.getMaskedImage()
2445 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2446 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2447 numNans =
maskNans(maskedImage, maskVal)
2448 self.metadata.set(
"NUMNANS", numNans)
2450 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2453 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2458 exposure : `lsst.afw.image.Exposure`
2459 Exposure to process.
2463 lsst.ip.isr.isrTask.maskNan
2466 isrFunctions.interpolateFromMask(
2467 maskedImage=exposure.getMaskedImage(),
2468 fwhm=self.config.fwhm,
2469 growSaturatedFootprints=0,
2470 maskNameList=[
"UNMASKEDNAN"],
2474 """Measure the image background in subgrids, for quality control purposes.
2478 exposure : `lsst.afw.image.Exposure`
2479 Exposure to process.
2480 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2481 Configuration object containing parameters on which background
2482 statistics and subgrids to use.
2484 if IsrQaConfig
is not None:
2485 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2486 IsrQaConfig.flatness.nIter)
2487 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2488 statsControl.setAndMask(maskVal)
2489 maskedImage = exposure.getMaskedImage()
2490 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2491 skyLevel = stats.getValue(afwMath.MEDIAN)
2492 skySigma = stats.getValue(afwMath.STDEVCLIP)
2493 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2494 metadata = exposure.getMetadata()
2495 metadata.set(
'SKYLEVEL', skyLevel)
2496 metadata.set(
'SKYSIGMA', skySigma)
2499 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2500 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2501 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2502 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2503 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2504 skyLevels = numpy.zeros((nX, nY))
2507 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2509 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2511 xLLC = xc - meshXHalf
2512 yLLC = yc - meshYHalf
2513 xURC = xc + meshXHalf - 1
2514 yURC = yc + meshYHalf - 1
2517 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2519 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2521 good = numpy.where(numpy.isfinite(skyLevels))
2522 skyMedian = numpy.median(skyLevels[good])
2523 flatness = (skyLevels[good] - skyMedian) / skyMedian
2524 flatness_rms = numpy.std(flatness)
2525 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2527 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2528 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2529 nX, nY, flatness_pp, flatness_rms)
2531 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2532 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2533 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2534 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2535 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2538 """Set an approximate magnitude zero point for the exposure.
2542 exposure : `lsst.afw.image.Exposure`
2543 Exposure to process.
2545 filterLabel = exposure.getFilterLabel()
2546 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2548 if physicalFilter
in self.config.fluxMag0T1:
2549 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2551 self.log.warn(
"No rough magnitude zero point defined for filter {}.".format(physicalFilter))
2552 fluxMag0 = self.config.defaultFluxMag0T1
2554 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2556 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2559 self.log.info(
"Setting rough magnitude zero point for filter {}: {}".
2560 format(physicalFilter, 2.5*math.log10(fluxMag0*expTime)))
2561 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2564 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2568 ccdExposure : `lsst.afw.image.Exposure`
2569 Exposure to process.
2570 fpPolygon : `lsst.afw.geom.Polygon`
2571 Polygon in focal plane coordinates.
2574 ccd = ccdExposure.getDetector()
2575 fpCorners = ccd.getCorners(FOCAL_PLANE)
2576 ccdPolygon = Polygon(fpCorners)
2579 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2582 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2583 validPolygon = Polygon(ccdPoints)
2584 ccdExposure.getInfo().setValidPolygon(validPolygon)
2588 """Context manager that applies and removes flats and darks,
2589 if the task is configured to apply them.
2593 exp : `lsst.afw.image.Exposure`
2594 Exposure to process.
2595 flat : `lsst.afw.image.Exposure`
2596 Flat exposure the same size as ``exp``.
2597 dark : `lsst.afw.image.Exposure`, optional
2598 Dark exposure the same size as ``exp``.
2602 exp : `lsst.afw.image.Exposure`
2603 The flat and dark corrected exposure.
2605 if self.config.doDark
and dark
is not None:
2607 if self.config.doFlat:
2612 if self.config.doFlat:
2614 if self.config.doDark
and dark
is not None:
2618 """Utility function to examine ISR exposure at different stages.
2622 exposure : `lsst.afw.image.Exposure`
2625 State of processing to view.
2627 frame = getDebugFrame(self._display, stepname)
2629 display = getDisplay(frame)
2630 display.scale(
'asinh',
'zscale')
2631 display.mtv(exposure)
2632 prompt =
"Press Enter to continue [c]... "
2634 ans = input(prompt).lower()
2635 if ans
in (
"",
"c",):
2640 """A Detector-like object that supports returning gain and saturation level
2642 This is used when the input exposure does not have a detector.
2646 exposure : `lsst.afw.image.Exposure`
2647 Exposure to generate a fake amplifier for.
2648 config : `lsst.ip.isr.isrTaskConfig`
2649 Configuration to apply to the fake amplifier.
2653 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2655 self.
_gain_gain = config.gain
2660 return self.
_bbox_bbox
2663 return self.
_bbox_bbox
2669 return self.
_gain_gain
2682 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2686 """Task to wrap the default IsrTask to allow it to be retargeted.
2688 The standard IsrTask can be called directly from a command line
2689 program, but doing so removes the ability of the task to be
2690 retargeted. As most cameras override some set of the IsrTask
2691 methods, this would remove those data-specific methods in the
2692 output post-ISR images. This wrapping class fixes the issue,
2693 allowing identical post-ISR images to be generated by both the
2694 processCcd and isrTask code.
2696 ConfigClass = RunIsrConfig
2697 _DefaultName =
"runIsr"
2701 self.makeSubtask(
"isr")
2707 dataRef : `lsst.daf.persistence.ButlerDataRef`
2708 data reference of the detector data to be processed
2712 result : `pipeBase.Struct`
2713 Result struct with component:
2715 - exposure : `lsst.afw.image.Exposure`
2716 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def ensureExposure(self, inputExp, camera, detectorNum)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.