29 import lsst.pipe.base
as pipeBase
30 import lsst.pipe.base.connectionTypes
as cT
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
126 crosstalkSources = cT.PrerequisiteInput(
127 name=
"isrOverscanCorrected",
128 doc=
"Overscan corrected input images.",
129 storageClass=
"Exposure",
130 dimensions=[
"instrument",
"exposure",
"detector"],
133 lookupFunction=crosstalkSourceLookup,
136 bias = cT.PrerequisiteInput(
138 doc=
"Input bias calibration.",
139 storageClass=
"ExposureF",
140 dimensions=[
"instrument",
"detector"],
143 dark = cT.PrerequisiteInput(
145 doc=
"Input dark calibration.",
146 storageClass=
"ExposureF",
147 dimensions=[
"instrument",
"detector"],
150 flat = cT.PrerequisiteInput(
152 doc=
"Input flat calibration.",
153 storageClass=
"ExposureF",
154 dimensions=[
"instrument",
"physical_filter",
"detector"],
157 ptc = cT.PrerequisiteInput(
159 doc=
"Input Photon Transfer Curve dataset",
160 storageClass=
"PhotonTransferCurveDataset",
161 dimensions=[
"instrument",
"detector"],
164 fringes = cT.PrerequisiteInput(
166 doc=
"Input fringe calibration.",
167 storageClass=
"ExposureF",
168 dimensions=[
"instrument",
"physical_filter",
"detector"],
172 strayLightData = cT.PrerequisiteInput(
174 doc=
"Input stray light calibration.",
175 storageClass=
"StrayLightData",
176 dimensions=[
"instrument",
"physical_filter",
"detector"],
181 bfKernel = cT.PrerequisiteInput(
183 doc=
"Input brighter-fatter kernel.",
184 storageClass=
"NumpyArray",
185 dimensions=[
"instrument"],
189 newBFKernel = cT.PrerequisiteInput(
190 name=
'brighterFatterKernel',
191 doc=
"Newer complete kernel + gain solutions.",
192 storageClass=
"BrighterFatterKernel",
193 dimensions=[
"instrument",
"detector"],
197 defects = cT.PrerequisiteInput(
199 doc=
"Input defect tables.",
200 storageClass=
"Defects",
201 dimensions=[
"instrument",
"detector"],
204 linearizer = cT.PrerequisiteInput(
206 storageClass=
"Linearizer",
207 doc=
"Linearity correction calibration.",
208 dimensions=[
"instrument",
"detector"],
212 opticsTransmission = cT.PrerequisiteInput(
213 name=
"transmission_optics",
214 storageClass=
"TransmissionCurve",
215 doc=
"Transmission curve due to the optics.",
216 dimensions=[
"instrument"],
219 filterTransmission = cT.PrerequisiteInput(
220 name=
"transmission_filter",
221 storageClass=
"TransmissionCurve",
222 doc=
"Transmission curve due to the filter.",
223 dimensions=[
"instrument",
"physical_filter"],
226 sensorTransmission = cT.PrerequisiteInput(
227 name=
"transmission_sensor",
228 storageClass=
"TransmissionCurve",
229 doc=
"Transmission curve due to the sensor.",
230 dimensions=[
"instrument",
"detector"],
233 atmosphereTransmission = cT.PrerequisiteInput(
234 name=
"transmission_atmosphere",
235 storageClass=
"TransmissionCurve",
236 doc=
"Transmission curve due to the atmosphere.",
237 dimensions=[
"instrument"],
240 illumMaskedImage = cT.PrerequisiteInput(
242 doc=
"Input illumination correction.",
243 storageClass=
"MaskedImageF",
244 dimensions=[
"instrument",
"physical_filter",
"detector"],
248 outputExposure = cT.Output(
250 doc=
"Output ISR processed exposure.",
251 storageClass=
"Exposure",
252 dimensions=[
"instrument",
"exposure",
"detector"],
254 preInterpExposure = cT.Output(
255 name=
'preInterpISRCCD',
256 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
257 storageClass=
"ExposureF",
258 dimensions=[
"instrument",
"exposure",
"detector"],
260 outputOssThumbnail = cT.Output(
262 doc=
"Output Overscan-subtracted thumbnail image.",
263 storageClass=
"Thumbnail",
264 dimensions=[
"instrument",
"exposure",
"detector"],
266 outputFlattenedThumbnail = cT.Output(
267 name=
"FlattenedThumb",
268 doc=
"Output flat-corrected thumbnail image.",
269 storageClass=
"Thumbnail",
270 dimensions=[
"instrument",
"exposure",
"detector"],
276 if config.doBias
is not True:
277 self.prerequisiteInputs.discard(
"bias")
278 if config.doLinearize
is not True:
279 self.prerequisiteInputs.discard(
"linearizer")
280 if config.doCrosstalk
is not True:
281 self.inputs.discard(
"crosstalkSources")
282 self.prerequisiteInputs.discard(
"crosstalk")
283 if config.doBrighterFatter
is not True:
284 self.prerequisiteInputs.discard(
"bfKernel")
285 self.prerequisiteInputs.discard(
"newBFKernel")
286 if config.doDefect
is not True:
287 self.prerequisiteInputs.discard(
"defects")
288 if config.doDark
is not True:
289 self.prerequisiteInputs.discard(
"dark")
290 if config.doFlat
is not True:
291 self.prerequisiteInputs.discard(
"flat")
292 if config.doFringe
is not True:
293 self.prerequisiteInputs.discard(
"fringe")
294 if config.doStrayLight
is not True:
295 self.prerequisiteInputs.discard(
"strayLightData")
296 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
297 self.prerequisiteInputs.discard(
"ptc")
298 if config.doAttachTransmissionCurve
is not True:
299 self.prerequisiteInputs.discard(
"opticsTransmission")
300 self.prerequisiteInputs.discard(
"filterTransmission")
301 self.prerequisiteInputs.discard(
"sensorTransmission")
302 self.prerequisiteInputs.discard(
"atmosphereTransmission")
303 if config.doUseOpticsTransmission
is not True:
304 self.prerequisiteInputs.discard(
"opticsTransmission")
305 if config.doUseFilterTransmission
is not True:
306 self.prerequisiteInputs.discard(
"filterTransmission")
307 if config.doUseSensorTransmission
is not True:
308 self.prerequisiteInputs.discard(
"sensorTransmission")
309 if config.doUseAtmosphereTransmission
is not True:
310 self.prerequisiteInputs.discard(
"atmosphereTransmission")
311 if config.doIlluminationCorrection
is not True:
312 self.prerequisiteInputs.discard(
"illumMaskedImage")
314 if config.doWrite
is not True:
315 self.outputs.discard(
"outputExposure")
316 self.outputs.discard(
"preInterpExposure")
317 self.outputs.discard(
"outputFlattenedThumbnail")
318 self.outputs.discard(
"outputOssThumbnail")
319 if config.doSaveInterpPixels
is not True:
320 self.outputs.discard(
"preInterpExposure")
321 if config.qa.doThumbnailOss
is not True:
322 self.outputs.discard(
"outputOssThumbnail")
323 if config.qa.doThumbnailFlattened
is not True:
324 self.outputs.discard(
"outputFlattenedThumbnail")
328 pipelineConnections=IsrTaskConnections):
329 """Configuration parameters for IsrTask.
331 Items are grouped in the order in which they are executed by the task.
333 datasetType = pexConfig.Field(
335 doc=
"Dataset type for input data; users will typically leave this alone, "
336 "but camera-specific ISR tasks will override it",
340 fallbackFilterName = pexConfig.Field(
342 doc=
"Fallback default filter name for calibrations.",
345 useFallbackDate = pexConfig.Field(
347 doc=
"Pass observation date when using fallback filter.",
350 expectWcs = pexConfig.Field(
353 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
355 fwhm = pexConfig.Field(
357 doc=
"FWHM of PSF in arcseconds.",
360 qa = pexConfig.ConfigField(
362 doc=
"QA related configuration options.",
366 doConvertIntToFloat = pexConfig.Field(
368 doc=
"Convert integer raw images to floating point values?",
373 doSaturation = pexConfig.Field(
375 doc=
"Mask saturated pixels? NB: this is totally independent of the"
376 " interpolation option - this is ONLY setting the bits in the mask."
377 " To have them interpolated make sure doSaturationInterpolation=True",
380 saturatedMaskName = pexConfig.Field(
382 doc=
"Name of mask plane to use in saturation detection and interpolation",
385 saturation = pexConfig.Field(
387 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
388 default=float(
"NaN"),
390 growSaturationFootprintSize = pexConfig.Field(
392 doc=
"Number of pixels by which to grow the saturation footprints",
397 doSuspect = pexConfig.Field(
399 doc=
"Mask suspect pixels?",
402 suspectMaskName = pexConfig.Field(
404 doc=
"Name of mask plane to use for suspect pixels",
407 numEdgeSuspect = pexConfig.Field(
409 doc=
"Number of edge pixels to be flagged as untrustworthy.",
412 edgeMaskLevel = pexConfig.ChoiceField(
414 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
417 'DETECTOR':
'Mask only the edges of the full detector.',
418 'AMP':
'Mask edges of each amplifier.',
423 doSetBadRegions = pexConfig.Field(
425 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
428 badStatistic = pexConfig.ChoiceField(
430 doc=
"How to estimate the average value for BAD regions.",
433 "MEANCLIP":
"Correct using the (clipped) mean of good data",
434 "MEDIAN":
"Correct using the median of the good data",
439 doOverscan = pexConfig.Field(
441 doc=
"Do overscan subtraction?",
444 overscan = pexConfig.ConfigurableField(
445 target=OverscanCorrectionTask,
446 doc=
"Overscan subtraction task for image segments.",
449 overscanFitType = pexConfig.ChoiceField(
451 doc=
"The method for fitting the overscan bias level.",
454 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
455 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
456 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
457 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
458 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
459 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
460 "MEAN":
"Correct using the mean of the overscan region",
461 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
462 "MEDIAN":
"Correct using the median of the overscan region",
463 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
465 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
466 " This option will no longer be used, and will be removed after v20.")
468 overscanOrder = pexConfig.Field(
470 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
471 "or number of spline knots if overscan fit type is a spline."),
473 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
474 " This option will no longer be used, and will be removed after v20.")
476 overscanNumSigmaClip = pexConfig.Field(
478 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
480 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
481 " This option will no longer be used, and will be removed after v20.")
483 overscanIsInt = pexConfig.Field(
485 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
486 " and overscan.FitType=MEDIAN_PER_ROW.",
488 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
489 " This option will no longer be used, and will be removed after v20.")
492 overscanNumLeadingColumnsToSkip = pexConfig.Field(
494 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
497 overscanNumTrailingColumnsToSkip = pexConfig.Field(
499 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
502 overscanMaxDev = pexConfig.Field(
504 doc=
"Maximum deviation from the median for overscan",
505 default=1000.0, check=
lambda x: x > 0
507 overscanBiasJump = pexConfig.Field(
509 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
512 overscanBiasJumpKeyword = pexConfig.Field(
514 doc=
"Header keyword containing information about devices.",
515 default=
"NO_SUCH_KEY",
517 overscanBiasJumpDevices = pexConfig.ListField(
519 doc=
"List of devices that need piecewise overscan correction.",
522 overscanBiasJumpLocation = pexConfig.Field(
524 doc=
"Location of bias jump along y-axis.",
529 doAssembleCcd = pexConfig.Field(
532 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
534 assembleCcd = pexConfig.ConfigurableField(
535 target=AssembleCcdTask,
536 doc=
"CCD assembly task",
540 doAssembleIsrExposures = pexConfig.Field(
543 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
545 doTrimToMatchCalib = pexConfig.Field(
548 doc=
"Trim raw data to match calibration bounding boxes?"
552 doBias = pexConfig.Field(
554 doc=
"Apply bias frame correction?",
557 biasDataProductName = pexConfig.Field(
559 doc=
"Name of the bias data product",
562 doBiasBeforeOverscan = pexConfig.Field(
564 doc=
"Reverse order of overscan and bias correction.",
569 doVariance = pexConfig.Field(
571 doc=
"Calculate variance?",
574 gain = pexConfig.Field(
576 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
577 default=float(
"NaN"),
579 readNoise = pexConfig.Field(
581 doc=
"The read noise to use if no Detector is present in the Exposure",
584 doEmpiricalReadNoise = pexConfig.Field(
587 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
589 usePtcReadNoise = pexConfig.Field(
592 doc=
"Use readnoise values from the Photon Transfer Curve?"
594 maskNegativeVariance = pexConfig.Field(
597 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
598 "in the measurement of the overscan at an edge due to the data falling off faster "
599 "than the overscan model can account for it."
601 negativeVarianceMaskName = pexConfig.Field(
604 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
607 doLinearize = pexConfig.Field(
609 doc=
"Correct for nonlinearity of the detector's response?",
614 doCrosstalk = pexConfig.Field(
616 doc=
"Apply intra-CCD crosstalk correction?",
619 doCrosstalkBeforeAssemble = pexConfig.Field(
621 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
624 crosstalk = pexConfig.ConfigurableField(
625 target=CrosstalkTask,
626 doc=
"Intra-CCD crosstalk correction",
630 doDefect = pexConfig.Field(
632 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
635 doNanMasking = pexConfig.Field(
637 doc=
"Mask non-finite (NAN, inf) pixels?",
640 doWidenSaturationTrails = pexConfig.Field(
642 doc=
"Widen bleed trails based on their width?",
647 doBrighterFatter = pexConfig.Field(
650 doc=
"Apply the brighter-fatter correction?"
652 brighterFatterLevel = pexConfig.ChoiceField(
655 doc=
"The level at which to correct for brighter-fatter.",
657 "AMP":
"Every amplifier treated separately.",
658 "DETECTOR":
"One kernel per detector",
661 brighterFatterMaxIter = pexConfig.Field(
664 doc=
"Maximum number of iterations for the brighter-fatter correction"
666 brighterFatterThreshold = pexConfig.Field(
669 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
670 "absolute value of the difference between the current corrected image and the one "
671 "from the previous iteration summed over all the pixels."
673 brighterFatterApplyGain = pexConfig.Field(
676 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
678 brighterFatterMaskListToInterpolate = pexConfig.ListField(
680 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
682 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
684 brighterFatterMaskGrowSize = pexConfig.Field(
687 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
688 "when brighter-fatter correction is applied."
692 doDark = pexConfig.Field(
694 doc=
"Apply dark frame correction?",
697 darkDataProductName = pexConfig.Field(
699 doc=
"Name of the dark data product",
704 doStrayLight = pexConfig.Field(
706 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
709 strayLight = pexConfig.ConfigurableField(
710 target=StrayLightTask,
711 doc=
"y-band stray light correction"
715 doFlat = pexConfig.Field(
717 doc=
"Apply flat field correction?",
720 flatDataProductName = pexConfig.Field(
722 doc=
"Name of the flat data product",
725 flatScalingType = pexConfig.ChoiceField(
727 doc=
"The method for scaling the flat on the fly.",
730 "USER":
"Scale by flatUserScale",
731 "MEAN":
"Scale by the inverse of the mean",
732 "MEDIAN":
"Scale by the inverse of the median",
735 flatUserScale = pexConfig.Field(
737 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
740 doTweakFlat = pexConfig.Field(
742 doc=
"Tweak flats to match observed amplifier ratios?",
747 doApplyGains = pexConfig.Field(
749 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
752 usePtcGains = pexConfig.Field(
754 doc=
"Use the gain values from the Photon Transfer Curve?",
757 normalizeGains = pexConfig.Field(
759 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
764 doFringe = pexConfig.Field(
766 doc=
"Apply fringe correction?",
769 fringe = pexConfig.ConfigurableField(
771 doc=
"Fringe subtraction task",
773 fringeAfterFlat = pexConfig.Field(
775 doc=
"Do fringe subtraction after flat-fielding?",
780 doMeasureBackground = pexConfig.Field(
782 doc=
"Measure the background level on the reduced image?",
787 doCameraSpecificMasking = pexConfig.Field(
789 doc=
"Mask camera-specific bad regions?",
792 masking = pexConfig.ConfigurableField(
799 doInterpolate = pexConfig.Field(
801 doc=
"Interpolate masked pixels?",
804 doSaturationInterpolation = pexConfig.Field(
806 doc=
"Perform interpolation over pixels masked as saturated?"
807 " NB: This is independent of doSaturation; if that is False this plane"
808 " will likely be blank, resulting in a no-op here.",
811 doNanInterpolation = pexConfig.Field(
813 doc=
"Perform interpolation over pixels masked as NaN?"
814 " NB: This is independent of doNanMasking; if that is False this plane"
815 " will likely be blank, resulting in a no-op here.",
818 doNanInterpAfterFlat = pexConfig.Field(
820 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
821 "also have to interpolate them before flat-fielding."),
824 maskListToInterpolate = pexConfig.ListField(
826 doc=
"List of mask planes that should be interpolated.",
827 default=[
'SAT',
'BAD'],
829 doSaveInterpPixels = pexConfig.Field(
831 doc=
"Save a copy of the pre-interpolated pixel values?",
836 fluxMag0T1 = pexConfig.DictField(
839 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
840 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
843 defaultFluxMag0T1 = pexConfig.Field(
845 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
846 default=pow(10.0, 0.4*28.0)
850 doVignette = pexConfig.Field(
852 doc=
"Apply vignetting parameters?",
855 vignette = pexConfig.ConfigurableField(
857 doc=
"Vignetting task.",
861 doAttachTransmissionCurve = pexConfig.Field(
864 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
866 doUseOpticsTransmission = pexConfig.Field(
869 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
871 doUseFilterTransmission = pexConfig.Field(
874 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
876 doUseSensorTransmission = pexConfig.Field(
879 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
881 doUseAtmosphereTransmission = pexConfig.Field(
884 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
888 doIlluminationCorrection = pexConfig.Field(
891 doc=
"Perform illumination correction?"
893 illuminationCorrectionDataProductName = pexConfig.Field(
895 doc=
"Name of the illumination correction data product.",
898 illumScale = pexConfig.Field(
900 doc=
"Scale factor for the illumination correction.",
903 illumFilters = pexConfig.ListField(
906 doc=
"Only perform illumination correction for these filters."
910 doWrite = pexConfig.Field(
912 doc=
"Persist postISRCCD?",
919 raise ValueError(
"You may not specify both doFlat and doApplyGains")
921 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
930 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
931 """Apply common instrument signature correction algorithms to a raw frame.
933 The process for correcting imaging data is very similar from
934 camera to camera. This task provides a vanilla implementation of
935 doing these corrections, including the ability to turn certain
936 corrections off if they are not needed. The inputs to the primary
937 method, `run()`, are a raw exposure to be corrected and the
938 calibration data products. The raw input is a single chip sized
939 mosaic of all amps including overscans and other non-science
940 pixels. The method `runDataRef()` identifies and defines the
941 calibration data products, and is intended for use by a
942 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
943 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
944 subclassed for different camera, although the most camera specific
945 methods have been split into subtasks that can be redirected
948 The __init__ method sets up the subtasks for ISR processing, using
949 the defaults from `lsst.ip.isr`.
954 Positional arguments passed to the Task constructor. None used at this time.
955 kwargs : `dict`, optional
956 Keyword arguments passed on to the Task constructor. None used at this time.
958 ConfigClass = IsrTaskConfig
963 self.makeSubtask(
"assembleCcd")
964 self.makeSubtask(
"crosstalk")
965 self.makeSubtask(
"strayLight")
966 self.makeSubtask(
"fringe")
967 self.makeSubtask(
"masking")
968 self.makeSubtask(
"overscan")
969 self.makeSubtask(
"vignette")
972 inputs = butlerQC.get(inputRefs)
975 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
976 except Exception
as e:
977 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
980 inputs[
'isGen3'] =
True
982 detector = inputs[
'ccdExposure'].getDetector()
984 if self.config.doCrosstalk
is True:
987 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
988 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
989 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
991 coeffVector = (self.config.crosstalk.crosstalkValues
992 if self.config.crosstalk.useConfigCoefficients
else None)
993 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
994 inputs[
'crosstalk'] = crosstalkCalib
995 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
996 if 'crosstalkSources' not in inputs:
997 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
1000 if 'linearizer' in inputs:
1001 if isinstance(inputs[
'linearizer'], dict):
1003 linearizer.fromYaml(inputs[
'linearizer'])
1004 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1005 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1009 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1011 linearizer = inputs[
'linearizer']
1012 linearizer.log = self.log
1013 inputs[
'linearizer'] = linearizer
1016 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1018 if self.config.doDefect
is True:
1019 if "defects" in inputs
and inputs[
'defects']
is not None:
1022 if not isinstance(inputs[
"defects"], Defects):
1023 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1027 if self.config.doBrighterFatter:
1028 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1029 if brighterFatterKernel
is None:
1030 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1032 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1034 detName = detector.getName()
1035 level = brighterFatterKernel.level
1038 inputs[
'bfGains'] = brighterFatterKernel.gain
1039 if self.config.brighterFatterLevel ==
'DETECTOR':
1040 if level ==
'DETECTOR':
1041 if detName
in brighterFatterKernel.detKernels:
1042 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1044 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1045 elif level ==
'AMP':
1046 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1048 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1049 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1050 elif self.config.brighterFatterLevel ==
'AMP':
1051 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1053 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1054 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
1055 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1057 assembler=self.assembleCcd
1058 if self.config.doAssembleIsrExposures
else None)
1060 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1062 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1063 if 'strayLightData' not in inputs:
1064 inputs[
'strayLightData'] =
None
1066 outputs = self.
runrun(**inputs)
1067 butlerQC.put(outputs, outputRefs)
1070 """Retrieve necessary frames for instrument signature removal.
1072 Pre-fetching all required ISR data products limits the IO
1073 required by the ISR. Any conflict between the calibration data
1074 available and that needed for ISR is also detected prior to
1075 doing processing, allowing it to fail quickly.
1079 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1080 Butler reference of the detector data to be processed
1081 rawExposure : `afw.image.Exposure`
1082 The raw exposure that will later be corrected with the
1083 retrieved calibration data; should not be modified in this
1088 result : `lsst.pipe.base.Struct`
1089 Result struct with components (which may be `None`):
1090 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1091 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1092 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1093 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1094 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1095 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1096 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1097 - ``fringes``: `lsst.pipe.base.Struct` with components:
1098 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1099 - ``seed``: random seed derived from the ccdExposureId for random
1100 number generator (`uint32`).
1101 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1102 A ``TransmissionCurve`` that represents the throughput of the optics,
1103 to be evaluated in focal-plane coordinates.
1104 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1105 A ``TransmissionCurve`` that represents the throughput of the filter
1106 itself, to be evaluated in focal-plane coordinates.
1107 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1108 A ``TransmissionCurve`` that represents the throughput of the sensor
1109 itself, to be evaluated in post-assembly trimmed detector coordinates.
1110 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1111 A ``TransmissionCurve`` that represents the throughput of the
1112 atmosphere, assumed to be spatially constant.
1113 - ``strayLightData`` : `object`
1114 An opaque object containing calibration information for
1115 stray-light correction. If `None`, no correction will be
1117 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1121 NotImplementedError :
1122 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1125 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1126 dateObs = dateObs.toPython().isoformat()
1127 except RuntimeError:
1128 self.log.warning(
"Unable to identify dateObs for rawExposure.")
1131 ccd = rawExposure.getDetector()
1132 filterLabel = rawExposure.getFilterLabel()
1133 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1134 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1135 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1136 if self.config.doBias
else None)
1138 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1140 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1141 linearizer.log = self.log
1142 if isinstance(linearizer, numpy.ndarray):
1145 crosstalkCalib =
None
1146 if self.config.doCrosstalk:
1148 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1150 coeffVector = (self.config.crosstalk.crosstalkValues
1151 if self.config.crosstalk.useConfigCoefficients
else None)
1152 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1153 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1154 if self.config.doCrosstalk
else None)
1156 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1157 if self.config.doDark
else None)
1158 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1160 if self.config.doFlat
else None)
1162 brighterFatterKernel =
None
1163 brighterFatterGains =
None
1164 if self.config.doBrighterFatter
is True:
1169 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1170 brighterFatterGains = brighterFatterKernel.gain
1171 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1174 brighterFatterKernel = dataRef.get(
"bfKernel")
1175 self.log.info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1177 brighterFatterKernel =
None
1178 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1181 if self.config.brighterFatterLevel ==
'DETECTOR':
1182 if brighterFatterKernel.detKernels:
1183 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1185 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1188 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1190 defectList = (dataRef.get(
"defects")
1191 if self.config.doDefect
else None)
1192 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1193 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1194 if self.config.doAssembleIsrExposures
else None)
1195 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1196 else pipeBase.Struct(fringes=
None))
1198 if self.config.doAttachTransmissionCurve:
1199 opticsTransmission = (dataRef.get(
"transmission_optics")
1200 if self.config.doUseOpticsTransmission
else None)
1201 filterTransmission = (dataRef.get(
"transmission_filter")
1202 if self.config.doUseFilterTransmission
else None)
1203 sensorTransmission = (dataRef.get(
"transmission_sensor")
1204 if self.config.doUseSensorTransmission
else None)
1205 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1206 if self.config.doUseAtmosphereTransmission
else None)
1208 opticsTransmission =
None
1209 filterTransmission =
None
1210 sensorTransmission =
None
1211 atmosphereTransmission =
None
1213 if self.config.doStrayLight:
1214 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1216 strayLightData =
None
1219 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1220 if (self.config.doIlluminationCorrection
1221 and physicalFilter
in self.config.illumFilters)
1225 return pipeBase.Struct(bias=biasExposure,
1226 linearizer=linearizer,
1227 crosstalk=crosstalkCalib,
1228 crosstalkSources=crosstalkSources,
1231 bfKernel=brighterFatterKernel,
1232 bfGains=brighterFatterGains,
1234 fringes=fringeStruct,
1235 opticsTransmission=opticsTransmission,
1236 filterTransmission=filterTransmission,
1237 sensorTransmission=sensorTransmission,
1238 atmosphereTransmission=atmosphereTransmission,
1239 strayLightData=strayLightData,
1240 illumMaskedImage=illumMaskedImage
1243 @pipeBase.timeMethod
1244 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1245 crosstalk=None, crosstalkSources=None,
1246 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1247 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1248 sensorTransmission=
None, atmosphereTransmission=
None,
1249 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1252 """Perform instrument signature removal on an exposure.
1254 Steps included in the ISR processing, in order performed, are:
1255 - saturation and suspect pixel masking
1256 - overscan subtraction
1257 - CCD assembly of individual amplifiers
1259 - variance image construction
1260 - linearization of non-linear response
1262 - brighter-fatter correction
1265 - stray light subtraction
1267 - masking of known defects and camera specific features
1268 - vignette calculation
1269 - appending transmission curve and distortion model
1273 ccdExposure : `lsst.afw.image.Exposure`
1274 The raw exposure that is to be run through ISR. The
1275 exposure is modified by this method.
1276 camera : `lsst.afw.cameraGeom.Camera`, optional
1277 The camera geometry for this exposure. Required if ``isGen3`` is
1278 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1279 ``flat`` does not have an associated detector.
1280 bias : `lsst.afw.image.Exposure`, optional
1281 Bias calibration frame.
1282 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1283 Functor for linearization.
1284 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1285 Calibration for crosstalk.
1286 crosstalkSources : `list`, optional
1287 List of possible crosstalk sources.
1288 dark : `lsst.afw.image.Exposure`, optional
1289 Dark calibration frame.
1290 flat : `lsst.afw.image.Exposure`, optional
1291 Flat calibration frame.
1292 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1293 Photon transfer curve dataset, with, e.g., gains
1295 bfKernel : `numpy.ndarray`, optional
1296 Brighter-fatter kernel.
1297 bfGains : `dict` of `float`, optional
1298 Gains used to override the detector's nominal gains for the
1299 brighter-fatter correction. A dict keyed by amplifier name for
1300 the detector in question.
1301 defects : `lsst.ip.isr.Defects`, optional
1303 fringes : `lsst.pipe.base.Struct`, optional
1304 Struct containing the fringe correction data, with
1306 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1307 - ``seed``: random seed derived from the ccdExposureId for random
1308 number generator (`uint32`)
1309 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1310 A ``TransmissionCurve`` that represents the throughput of the optics,
1311 to be evaluated in focal-plane coordinates.
1312 filterTransmission : `lsst.afw.image.TransmissionCurve`
1313 A ``TransmissionCurve`` that represents the throughput of the filter
1314 itself, to be evaluated in focal-plane coordinates.
1315 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1316 A ``TransmissionCurve`` that represents the throughput of the sensor
1317 itself, to be evaluated in post-assembly trimmed detector coordinates.
1318 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1319 A ``TransmissionCurve`` that represents the throughput of the
1320 atmosphere, assumed to be spatially constant.
1321 detectorNum : `int`, optional
1322 The integer number for the detector to process.
1323 isGen3 : bool, optional
1324 Flag this call to run() as using the Gen3 butler environment.
1325 strayLightData : `object`, optional
1326 Opaque object containing calibration information for stray-light
1327 correction. If `None`, no correction will be performed.
1328 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1329 Illumination correction image.
1333 result : `lsst.pipe.base.Struct`
1334 Result struct with component:
1335 - ``exposure`` : `afw.image.Exposure`
1336 The fully ISR corrected exposure.
1337 - ``outputExposure`` : `afw.image.Exposure`
1338 An alias for `exposure`
1339 - ``ossThumb`` : `numpy.ndarray`
1340 Thumbnail image of the exposure after overscan subtraction.
1341 - ``flattenedThumb`` : `numpy.ndarray`
1342 Thumbnail image of the exposure after flat-field correction.
1347 Raised if a configuration option is set to True, but the
1348 required calibration data has not been specified.
1352 The current processed exposure can be viewed by setting the
1353 appropriate lsstDebug entries in the `debug.display`
1354 dictionary. The names of these entries correspond to some of
1355 the IsrTaskConfig Boolean options, with the value denoting the
1356 frame to use. The exposure is shown inside the matching
1357 option check and after the processing of that step has
1358 finished. The steps with debug points are:
1369 In addition, setting the "postISRCCD" entry displays the
1370 exposure after all ISR processing has finished.
1378 if detectorNum
is None:
1379 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1381 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1382 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1383 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1384 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1386 if isinstance(ccdExposure, ButlerDataRef):
1387 return self.
runDataRefrunDataRef(ccdExposure)
1389 ccd = ccdExposure.getDetector()
1390 filterLabel = ccdExposure.getFilterLabel()
1391 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1394 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1395 ccd = [
FakeAmp(ccdExposure, self.config)]
1398 if self.config.doBias
and bias
is None:
1399 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1400 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1401 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1402 if self.config.doBrighterFatter
and bfKernel
is None:
1403 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1404 if self.config.doDark
and dark
is None:
1405 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1406 if self.config.doFlat
and flat
is None:
1407 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1408 if self.config.doDefect
and defects
is None:
1409 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1410 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1411 and fringes.fringes
is None):
1416 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1417 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1418 and illumMaskedImage
is None):
1419 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1422 if self.config.doConvertIntToFloat:
1423 self.log.info(
"Converting exposure to floating point values.")
1426 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1427 self.log.info(
"Applying bias correction.")
1428 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1429 trimToFit=self.config.doTrimToMatchCalib)
1430 self.
debugViewdebugView(ccdExposure,
"doBias")
1436 if ccdExposure.getBBox().contains(amp.getBBox()):
1438 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1440 if self.config.doOverscan
and not badAmp:
1443 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1444 if overscanResults
is not None and \
1445 self.config.qa
is not None and self.config.qa.saveStats
is True:
1446 if isinstance(overscanResults.overscanFit, float):
1447 qaMedian = overscanResults.overscanFit
1448 qaStdev = float(
"NaN")
1450 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1451 afwMath.MEDIAN | afwMath.STDEVCLIP)
1452 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1453 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1455 self.metadata.set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1456 self.metadata.set(f
"FIT STDEV {amp.getName()}", qaStdev)
1457 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1458 amp.getName(), qaMedian, qaStdev)
1461 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1462 afwMath.MEDIAN | afwMath.STDEVCLIP)
1463 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1464 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1466 self.metadata.set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1467 self.metadata.set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1468 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1469 amp.getName(), qaMedianAfter, qaStdevAfter)
1471 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1474 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1475 overscanResults =
None
1477 overscans.append(overscanResults
if overscanResults
is not None else None)
1479 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1481 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1482 self.log.info(
"Applying crosstalk correction.")
1483 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1484 crosstalkSources=crosstalkSources, camera=camera)
1485 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1487 if self.config.doAssembleCcd:
1488 self.log.info(
"Assembling CCD from amplifiers.")
1489 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1491 if self.config.expectWcs
and not ccdExposure.getWcs():
1492 self.log.warning(
"No WCS found in input exposure.")
1493 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1496 if self.config.qa.doThumbnailOss:
1497 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1499 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1500 self.log.info(
"Applying bias correction.")
1501 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1502 trimToFit=self.config.doTrimToMatchCalib)
1503 self.
debugViewdebugView(ccdExposure,
"doBias")
1505 if self.config.doVariance:
1506 for amp, overscanResults
in zip(ccd, overscans):
1507 if ccdExposure.getBBox().contains(amp.getBBox()):
1508 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1509 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1510 if overscanResults
is not None:
1512 overscanImage=overscanResults.overscanImage,
1518 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1519 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1520 afwMath.MEDIAN | afwMath.STDEVCLIP)
1521 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1522 qaStats.getValue(afwMath.MEDIAN))
1523 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1524 qaStats.getValue(afwMath.STDEVCLIP))
1525 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1526 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1527 qaStats.getValue(afwMath.STDEVCLIP))
1528 if self.config.maskNegativeVariance:
1532 self.log.info(
"Applying linearizer.")
1533 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1534 detector=ccd, log=self.log)
1536 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1537 self.log.info(
"Applying crosstalk correction.")
1538 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1539 crosstalkSources=crosstalkSources, isTrimmed=
True)
1540 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1544 if self.config.doDefect:
1545 self.log.info(
"Masking defects.")
1546 self.
maskDefectmaskDefect(ccdExposure, defects)
1548 if self.config.numEdgeSuspect > 0:
1549 self.log.info(
"Masking edges as SUSPECT.")
1550 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1551 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1553 if self.config.doNanMasking:
1554 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1555 self.
maskNanmaskNan(ccdExposure)
1557 if self.config.doWidenSaturationTrails:
1558 self.log.info(
"Widening saturation trails.")
1559 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1561 if self.config.doCameraSpecificMasking:
1562 self.log.info(
"Masking regions for camera specific reasons.")
1563 self.masking.
run(ccdExposure)
1565 if self.config.doBrighterFatter:
1574 interpExp = ccdExposure.clone()
1575 with self.
flatContextflatContext(interpExp, flat, dark):
1576 isrFunctions.interpolateFromMask(
1577 maskedImage=interpExp.getMaskedImage(),
1578 fwhm=self.config.fwhm,
1579 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1580 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1582 bfExp = interpExp.clone()
1584 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1585 type(bfKernel), type(bfGains))
1586 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1587 self.config.brighterFatterMaxIter,
1588 self.config.brighterFatterThreshold,
1589 self.config.brighterFatterApplyGain,
1591 if bfResults[1] == self.config.brighterFatterMaxIter:
1592 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1595 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1597 image = ccdExposure.getMaskedImage().getImage()
1598 bfCorr = bfExp.getMaskedImage().getImage()
1599 bfCorr -= interpExp.getMaskedImage().getImage()
1608 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1609 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1612 if self.config.brighterFatterMaskGrowSize > 0:
1613 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1614 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1615 isrFunctions.growMasks(ccdExposure.getMask(),
1616 radius=self.config.brighterFatterMaskGrowSize,
1617 maskNameList=maskPlane,
1618 maskValue=maskPlane)
1620 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1622 if self.config.doDark:
1623 self.log.info(
"Applying dark correction.")
1625 self.
debugViewdebugView(ccdExposure,
"doDark")
1627 if self.config.doFringe
and not self.config.fringeAfterFlat:
1628 self.log.info(
"Applying fringe correction before flat.")
1629 self.fringe.
run(ccdExposure, **fringes.getDict())
1630 self.
debugViewdebugView(ccdExposure,
"doFringe")
1632 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1633 self.log.info(
"Checking strayLight correction.")
1634 self.strayLight.
run(ccdExposure, strayLightData)
1635 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1637 if self.config.doFlat:
1638 self.log.info(
"Applying flat correction.")
1640 self.
debugViewdebugView(ccdExposure,
"doFlat")
1642 if self.config.doApplyGains:
1643 self.log.info(
"Applying gain correction instead of flat.")
1644 if self.config.usePtcGains:
1645 self.log.info(
"Using gains from the Photon Transfer Curve.")
1646 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1649 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1651 if self.config.doFringe
and self.config.fringeAfterFlat:
1652 self.log.info(
"Applying fringe correction after flat.")
1653 self.fringe.
run(ccdExposure, **fringes.getDict())
1655 if self.config.doVignette:
1656 self.log.info(
"Constructing Vignette polygon.")
1659 if self.config.vignette.doWriteVignettePolygon:
1662 if self.config.doAttachTransmissionCurve:
1663 self.log.info(
"Adding transmission curves.")
1664 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1665 filterTransmission=filterTransmission,
1666 sensorTransmission=sensorTransmission,
1667 atmosphereTransmission=atmosphereTransmission)
1669 flattenedThumb =
None
1670 if self.config.qa.doThumbnailFlattened:
1671 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1673 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1674 self.log.info(
"Performing illumination correction.")
1675 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1676 illumMaskedImage, illumScale=self.config.illumScale,
1677 trimToFit=self.config.doTrimToMatchCalib)
1680 if self.config.doSaveInterpPixels:
1681 preInterpExp = ccdExposure.clone()
1696 if self.config.doSetBadRegions:
1697 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1698 if badPixelCount > 0:
1699 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1701 if self.config.doInterpolate:
1702 self.log.info(
"Interpolating masked pixels.")
1703 isrFunctions.interpolateFromMask(
1704 maskedImage=ccdExposure.getMaskedImage(),
1705 fwhm=self.config.fwhm,
1706 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1707 maskNameList=list(self.config.maskListToInterpolate)
1712 if self.config.doMeasureBackground:
1713 self.log.info(
"Measuring background level.")
1716 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1718 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1719 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1720 afwMath.MEDIAN | afwMath.STDEVCLIP)
1721 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1722 qaStats.getValue(afwMath.MEDIAN))
1723 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1724 qaStats.getValue(afwMath.STDEVCLIP))
1725 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1726 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1727 qaStats.getValue(afwMath.STDEVCLIP))
1729 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1731 return pipeBase.Struct(
1732 exposure=ccdExposure,
1734 flattenedThumb=flattenedThumb,
1736 preInterpExposure=preInterpExp,
1737 outputExposure=ccdExposure,
1738 outputOssThumbnail=ossThumb,
1739 outputFlattenedThumbnail=flattenedThumb,
1742 @pipeBase.timeMethod
1744 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1746 This method contains the `CmdLineTask` interface to the ISR
1747 processing. All IO is handled here, freeing the `run()` method
1748 to manage only pixel-level calculations. The steps performed
1750 - Read in necessary detrending/isr/calibration data.
1751 - Process raw exposure in `run()`.
1752 - Persist the ISR-corrected exposure as "postISRCCD" if
1753 config.doWrite=True.
1757 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1758 DataRef of the detector data to be processed
1762 result : `lsst.pipe.base.Struct`
1763 Result struct with component:
1764 - ``exposure`` : `afw.image.Exposure`
1765 The fully ISR corrected exposure.
1770 Raised if a configuration option is set to True, but the
1771 required calibration data does not exist.
1774 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1776 ccdExposure = sensorRef.get(self.config.datasetType)
1778 camera = sensorRef.get(
"camera")
1779 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1781 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1783 if self.config.doWrite:
1784 sensorRef.put(result.exposure,
"postISRCCD")
1785 if result.preInterpExposure
is not None:
1786 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1787 if result.ossThumb
is not None:
1788 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1789 if result.flattenedThumb
is not None:
1790 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1795 """Retrieve a calibration dataset for removing instrument signature.
1800 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1801 DataRef of the detector data to find calibration datasets
1804 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1805 dateObs : `str`, optional
1806 Date of the observation. Used to correct butler failures
1807 when using fallback filters.
1809 If True, disable butler proxies to enable error handling
1810 within this routine.
1814 exposure : `lsst.afw.image.Exposure`
1815 Requested calibration frame.
1820 Raised if no matching calibration frame can be found.
1823 exp = dataRef.get(datasetType, immediate=immediate)
1824 except Exception
as exc1:
1825 if not self.config.fallbackFilterName:
1826 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1828 if self.config.useFallbackDate
and dateObs:
1829 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1830 dateObs=dateObs, immediate=immediate)
1832 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1833 except Exception
as exc2:
1834 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1835 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1836 self.log.warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1838 if self.config.doAssembleIsrExposures:
1839 exp = self.assembleCcd.assembleCcd(exp)
1843 """Ensure that the data returned by Butler is a fully constructed exposure.
1845 ISR requires exposure-level image data for historical reasons, so if we did
1846 not recieve that from Butler, construct it from what we have, modifying the
1851 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1852 `lsst.afw.image.ImageF`
1853 The input data structure obtained from Butler.
1854 camera : `lsst.afw.cameraGeom.camera`
1855 The camera associated with the image. Used to find the appropriate
1858 The detector this exposure should match.
1862 inputExp : `lsst.afw.image.Exposure`
1863 The re-constructed exposure, with appropriate detector parameters.
1868 Raised if the input data cannot be used to construct an exposure.
1870 if isinstance(inputExp, afwImage.DecoratedImageU):
1871 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1872 elif isinstance(inputExp, afwImage.ImageF):
1873 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1874 elif isinstance(inputExp, afwImage.MaskedImageF):
1875 inputExp = afwImage.makeExposure(inputExp)
1876 elif isinstance(inputExp, afwImage.Exposure):
1878 elif inputExp
is None:
1882 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1885 if inputExp.getDetector()
is None:
1886 inputExp.setDetector(camera[detectorNum])
1891 """Convert exposure image from uint16 to float.
1893 If the exposure does not need to be converted, the input is
1894 immediately returned. For exposures that are converted to use
1895 floating point pixels, the variance is set to unity and the
1900 exposure : `lsst.afw.image.Exposure`
1901 The raw exposure to be converted.
1905 newexposure : `lsst.afw.image.Exposure`
1906 The input ``exposure``, converted to floating point pixels.
1911 Raised if the exposure type cannot be converted to float.
1914 if isinstance(exposure, afwImage.ExposureF):
1916 self.log.debug(
"Exposure already of type float.")
1918 if not hasattr(exposure,
"convertF"):
1919 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1921 newexposure = exposure.convertF()
1922 newexposure.variance[:] = 1
1923 newexposure.mask[:] = 0x0
1928 """Identify bad amplifiers, saturated and suspect pixels.
1932 ccdExposure : `lsst.afw.image.Exposure`
1933 Input exposure to be masked.
1934 amp : `lsst.afw.table.AmpInfoCatalog`
1935 Catalog of parameters defining the amplifier on this
1937 defects : `lsst.ip.isr.Defects`
1938 List of defects. Used to determine if the entire
1944 If this is true, the entire amplifier area is covered by
1945 defects and unusable.
1948 maskedImage = ccdExposure.getMaskedImage()
1954 if defects
is not None:
1955 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1960 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1962 maskView = dataView.getMask()
1963 maskView |= maskView.getPlaneBitMask(
"BAD")
1970 if self.config.doSaturation
and not badAmp:
1971 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1972 if self.config.doSuspect
and not badAmp:
1973 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1974 if math.isfinite(self.config.saturation):
1975 limits.update({self.config.saturatedMaskName: self.config.saturation})
1977 for maskName, maskThreshold
in limits.items():
1978 if not math.isnan(maskThreshold):
1979 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1980 isrFunctions.makeThresholdMask(
1981 maskedImage=dataView,
1982 threshold=maskThreshold,
1988 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1990 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1991 self.config.suspectMaskName])
1992 if numpy.all(maskView.getArray() & maskVal > 0):
1994 maskView |= maskView.getPlaneBitMask(
"BAD")
1999 """Apply overscan correction in place.
2001 This method does initial pixel rejection of the overscan
2002 region. The overscan can also be optionally segmented to
2003 allow for discontinuous overscan responses to be fit
2004 separately. The actual overscan subtraction is performed by
2005 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2006 which is called here after the amplifier is preprocessed.
2010 ccdExposure : `lsst.afw.image.Exposure`
2011 Exposure to have overscan correction performed.
2012 amp : `lsst.afw.cameraGeom.Amplifer`
2013 The amplifier to consider while correcting the overscan.
2017 overscanResults : `lsst.pipe.base.Struct`
2018 Result struct with components:
2019 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2020 Value or fit subtracted from the amplifier image data.
2021 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2022 Value or fit subtracted from the overscan image data.
2023 - ``overscanImage`` : `lsst.afw.image.Image`
2024 Image of the overscan region with the overscan
2025 correction applied. This quantity is used to estimate
2026 the amplifier read noise empirically.
2031 Raised if the ``amp`` does not contain raw pixel information.
2035 lsst.ip.isr.isrFunctions.overscanCorrection
2037 if amp.getRawHorizontalOverscanBBox().isEmpty():
2038 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2041 statControl = afwMath.StatisticsControl()
2042 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2045 dataBBox = amp.getRawDataBBox()
2046 oscanBBox = amp.getRawHorizontalOverscanBBox()
2050 prescanBBox = amp.getRawPrescanBBox()
2051 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2052 dx0 += self.config.overscanNumLeadingColumnsToSkip
2053 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2055 dx0 += self.config.overscanNumTrailingColumnsToSkip
2056 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2062 if ((self.config.overscanBiasJump
2063 and self.config.overscanBiasJumpLocation)
2064 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2065 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2066 self.config.overscanBiasJumpDevices)):
2067 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2068 yLower = self.config.overscanBiasJumpLocation
2069 yUpper = dataBBox.getHeight() - yLower
2071 yUpper = self.config.overscanBiasJumpLocation
2072 yLower = dataBBox.getHeight() - yUpper
2090 oscanBBox.getHeight())))
2093 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2094 ampImage = ccdExposure.maskedImage[imageBBox]
2095 overscanImage = ccdExposure.maskedImage[overscanBBox]
2097 overscanArray = overscanImage.image.array
2098 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2099 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2100 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2102 statControl = afwMath.StatisticsControl()
2103 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2105 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2108 levelStat = afwMath.MEDIAN
2109 sigmaStat = afwMath.STDEVCLIP
2111 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2112 self.config.qa.flatness.nIter)
2113 metadata = ccdExposure.getMetadata()
2114 ampNum = amp.getName()
2116 if isinstance(overscanResults.overscanFit, float):
2117 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2118 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2120 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2121 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2122 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2124 return overscanResults
2127 """Set the variance plane using the gain and read noise
2129 The read noise is calculated from the ``overscanImage`` if the
2130 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2131 the value from the amplifier data is used.
2135 ampExposure : `lsst.afw.image.Exposure`
2136 Exposure to process.
2137 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2138 Amplifier detector data.
2139 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2140 Image of overscan, required only for empirical read noise.
2141 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2142 PTC dataset containing the gains and read noise.
2148 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2149 are ``True``, but ptcDataset is not provided.
2151 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2152 ``overscanImage`` is ``None``.
2156 lsst.ip.isr.isrFunctions.updateVariance
2158 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2159 if self.config.usePtcGains:
2160 if ptcDataset
is None:
2161 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2163 gain = ptcDataset.gain[amp.getName()]
2164 self.log.info(
"Using gain from Photon Transfer Curve.")
2166 gain = amp.getGain()
2168 if math.isnan(gain):
2170 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2173 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2174 amp.getName(), gain, patchedGain)
2177 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2178 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2180 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2181 stats = afwMath.StatisticsControl()
2182 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2183 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2184 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2185 amp.getName(), readNoise)
2186 elif self.config.usePtcReadNoise:
2187 if ptcDataset
is None:
2188 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2190 readNoise = ptcDataset.noise[amp.getName()]
2191 self.log.info(
"Using read noise from Photon Transfer Curve.")
2193 readNoise = amp.getReadNoise()
2195 isrFunctions.updateVariance(
2196 maskedImage=ampExposure.getMaskedImage(),
2198 readNoise=readNoise,
2202 """Identify and mask pixels with negative variance values.
2206 exposure : `lsst.afw.image.Exposure`
2207 Exposure to process.
2211 lsst.ip.isr.isrFunctions.updateVariance
2213 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2214 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2215 exposure.mask.array[bad] |= maskPlane
2218 """Apply dark correction in place.
2222 exposure : `lsst.afw.image.Exposure`
2223 Exposure to process.
2224 darkExposure : `lsst.afw.image.Exposure`
2225 Dark exposure of the same size as ``exposure``.
2226 invert : `Bool`, optional
2227 If True, re-add the dark to an already corrected image.
2232 Raised if either ``exposure`` or ``darkExposure`` do not
2233 have their dark time defined.
2237 lsst.ip.isr.isrFunctions.darkCorrection
2239 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2240 if math.isnan(expScale):
2241 raise RuntimeError(
"Exposure darktime is NAN.")
2242 if darkExposure.getInfo().getVisitInfo()
is not None \
2243 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2244 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2248 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2251 isrFunctions.darkCorrection(
2252 maskedImage=exposure.getMaskedImage(),
2253 darkMaskedImage=darkExposure.getMaskedImage(),
2255 darkScale=darkScale,
2257 trimToFit=self.config.doTrimToMatchCalib
2261 """Check if linearization is needed for the detector cameraGeom.
2263 Checks config.doLinearize and the linearity type of the first
2268 detector : `lsst.afw.cameraGeom.Detector`
2269 Detector to get linearity type from.
2273 doLinearize : `Bool`
2274 If True, linearization should be performed.
2276 return self.config.doLinearize
and \
2277 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2280 """Apply flat correction in place.
2284 exposure : `lsst.afw.image.Exposure`
2285 Exposure to process.
2286 flatExposure : `lsst.afw.image.Exposure`
2287 Flat exposure of the same size as ``exposure``.
2288 invert : `Bool`, optional
2289 If True, unflatten an already flattened image.
2293 lsst.ip.isr.isrFunctions.flatCorrection
2295 isrFunctions.flatCorrection(
2296 maskedImage=exposure.getMaskedImage(),
2297 flatMaskedImage=flatExposure.getMaskedImage(),
2298 scalingType=self.config.flatScalingType,
2299 userScale=self.config.flatUserScale,
2301 trimToFit=self.config.doTrimToMatchCalib
2305 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2309 exposure : `lsst.afw.image.Exposure`
2310 Exposure to process. Only the amplifier DataSec is processed.
2311 amp : `lsst.afw.table.AmpInfoCatalog`
2312 Amplifier detector data.
2316 lsst.ip.isr.isrFunctions.makeThresholdMask
2318 if not math.isnan(amp.getSaturation()):
2319 maskedImage = exposure.getMaskedImage()
2320 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2321 isrFunctions.makeThresholdMask(
2322 maskedImage=dataView,
2323 threshold=amp.getSaturation(),
2325 maskName=self.config.saturatedMaskName,
2329 """Interpolate over saturated pixels, in place.
2331 This method should be called after `saturationDetection`, to
2332 ensure that the saturated pixels have been identified in the
2333 SAT mask. It should also be called after `assembleCcd`, since
2334 saturated regions may cross amplifier boundaries.
2338 exposure : `lsst.afw.image.Exposure`
2339 Exposure to process.
2343 lsst.ip.isr.isrTask.saturationDetection
2344 lsst.ip.isr.isrFunctions.interpolateFromMask
2346 isrFunctions.interpolateFromMask(
2347 maskedImage=exposure.getMaskedImage(),
2348 fwhm=self.config.fwhm,
2349 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2350 maskNameList=list(self.config.saturatedMaskName),
2354 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2358 exposure : `lsst.afw.image.Exposure`
2359 Exposure to process. Only the amplifier DataSec is processed.
2360 amp : `lsst.afw.table.AmpInfoCatalog`
2361 Amplifier detector data.
2365 lsst.ip.isr.isrFunctions.makeThresholdMask
2369 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2370 This is intended to indicate pixels that may be affected by unknown systematics;
2371 for example if non-linearity corrections above a certain level are unstable
2372 then that would be a useful value for suspectLevel. A value of `nan` indicates
2373 that no such level exists and no pixels are to be masked as suspicious.
2375 suspectLevel = amp.getSuspectLevel()
2376 if math.isnan(suspectLevel):
2379 maskedImage = exposure.getMaskedImage()
2380 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2381 isrFunctions.makeThresholdMask(
2382 maskedImage=dataView,
2383 threshold=suspectLevel,
2385 maskName=self.config.suspectMaskName,
2389 """Mask defects using mask plane "BAD", in place.
2393 exposure : `lsst.afw.image.Exposure`
2394 Exposure to process.
2395 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2396 `lsst.afw.image.DefectBase`.
2397 List of defects to mask.
2401 Call this after CCD assembly, since defects may cross amplifier boundaries.
2403 maskedImage = exposure.getMaskedImage()
2404 if not isinstance(defectBaseList, Defects):
2406 defectList =
Defects(defectBaseList)
2408 defectList = defectBaseList
2409 defectList.maskPixels(maskedImage, maskName=
"BAD")
2411 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2412 """Mask edge pixels with applicable mask plane.
2416 exposure : `lsst.afw.image.Exposure`
2417 Exposure to process.
2418 numEdgePixels : `int`, optional
2419 Number of edge pixels to mask.
2420 maskPlane : `str`, optional
2421 Mask plane name to use.
2422 level : `str`, optional
2423 Level at which to mask edges.
2425 maskedImage = exposure.getMaskedImage()
2426 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2428 if numEdgePixels > 0:
2429 if level ==
'DETECTOR':
2430 boxes = [maskedImage.getBBox()]
2431 elif level ==
'AMP':
2432 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2436 subImage = maskedImage[box]
2437 box.grow(-numEdgePixels)
2439 SourceDetectionTask.setEdgeBits(
2445 """Mask and interpolate defects using mask plane "BAD", in place.
2449 exposure : `lsst.afw.image.Exposure`
2450 Exposure to process.
2451 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2452 `lsst.afw.image.DefectBase`.
2453 List of defects to mask and interpolate.
2457 lsst.ip.isr.isrTask.maskDefect
2459 self.
maskDefectmaskDefect(exposure, defectBaseList)
2460 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2461 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2462 isrFunctions.interpolateFromMask(
2463 maskedImage=exposure.getMaskedImage(),
2464 fwhm=self.config.fwhm,
2465 growSaturatedFootprints=0,
2466 maskNameList=[
"BAD"],
2470 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2474 exposure : `lsst.afw.image.Exposure`
2475 Exposure to process.
2479 We mask over all non-finite values (NaN, inf), including those
2480 that are masked with other bits (because those may or may not be
2481 interpolated over later, and we want to remove all NaN/infs).
2482 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2483 preserve the historical name.
2485 maskedImage = exposure.getMaskedImage()
2488 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2489 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2490 numNans =
maskNans(maskedImage, maskVal)
2491 self.metadata.set(
"NUMNANS", numNans)
2493 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2496 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2501 exposure : `lsst.afw.image.Exposure`
2502 Exposure to process.
2506 lsst.ip.isr.isrTask.maskNan
2509 isrFunctions.interpolateFromMask(
2510 maskedImage=exposure.getMaskedImage(),
2511 fwhm=self.config.fwhm,
2512 growSaturatedFootprints=0,
2513 maskNameList=[
"UNMASKEDNAN"],
2517 """Measure the image background in subgrids, for quality control purposes.
2521 exposure : `lsst.afw.image.Exposure`
2522 Exposure to process.
2523 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2524 Configuration object containing parameters on which background
2525 statistics and subgrids to use.
2527 if IsrQaConfig
is not None:
2528 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2529 IsrQaConfig.flatness.nIter)
2530 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2531 statsControl.setAndMask(maskVal)
2532 maskedImage = exposure.getMaskedImage()
2533 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2534 skyLevel = stats.getValue(afwMath.MEDIAN)
2535 skySigma = stats.getValue(afwMath.STDEVCLIP)
2536 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2537 metadata = exposure.getMetadata()
2538 metadata.set(
'SKYLEVEL', skyLevel)
2539 metadata.set(
'SKYSIGMA', skySigma)
2542 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2543 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2544 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2545 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2546 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2547 skyLevels = numpy.zeros((nX, nY))
2550 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2552 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2554 xLLC = xc - meshXHalf
2555 yLLC = yc - meshYHalf
2556 xURC = xc + meshXHalf - 1
2557 yURC = yc + meshYHalf - 1
2560 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2562 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2564 good = numpy.where(numpy.isfinite(skyLevels))
2565 skyMedian = numpy.median(skyLevels[good])
2566 flatness = (skyLevels[good] - skyMedian) / skyMedian
2567 flatness_rms = numpy.std(flatness)
2568 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2570 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2571 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2572 nX, nY, flatness_pp, flatness_rms)
2574 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2575 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2576 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2577 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2578 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2581 """Set an approximate magnitude zero point for the exposure.
2585 exposure : `lsst.afw.image.Exposure`
2586 Exposure to process.
2588 filterLabel = exposure.getFilterLabel()
2589 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2591 if physicalFilter
in self.config.fluxMag0T1:
2592 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2594 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2595 fluxMag0 = self.config.defaultFluxMag0T1
2597 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2599 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2602 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2603 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2604 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2607 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2611 ccdExposure : `lsst.afw.image.Exposure`
2612 Exposure to process.
2613 fpPolygon : `lsst.afw.geom.Polygon`
2614 Polygon in focal plane coordinates.
2617 ccd = ccdExposure.getDetector()
2618 fpCorners = ccd.getCorners(FOCAL_PLANE)
2619 ccdPolygon = Polygon(fpCorners)
2622 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2625 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2626 validPolygon = Polygon(ccdPoints)
2627 ccdExposure.getInfo().setValidPolygon(validPolygon)
2631 """Context manager that applies and removes flats and darks,
2632 if the task is configured to apply them.
2636 exp : `lsst.afw.image.Exposure`
2637 Exposure to process.
2638 flat : `lsst.afw.image.Exposure`
2639 Flat exposure the same size as ``exp``.
2640 dark : `lsst.afw.image.Exposure`, optional
2641 Dark exposure the same size as ``exp``.
2645 exp : `lsst.afw.image.Exposure`
2646 The flat and dark corrected exposure.
2648 if self.config.doDark
and dark
is not None:
2650 if self.config.doFlat:
2655 if self.config.doFlat:
2657 if self.config.doDark
and dark
is not None:
2661 """Utility function to examine ISR exposure at different stages.
2665 exposure : `lsst.afw.image.Exposure`
2668 State of processing to view.
2670 frame = getDebugFrame(self._display, stepname)
2672 display = getDisplay(frame)
2673 display.scale(
'asinh',
'zscale')
2674 display.mtv(exposure)
2675 prompt =
"Press Enter to continue [c]... "
2677 ans = input(prompt).lower()
2678 if ans
in (
"",
"c",):
2683 """A Detector-like object that supports returning gain and saturation level
2685 This is used when the input exposure does not have a detector.
2689 exposure : `lsst.afw.image.Exposure`
2690 Exposure to generate a fake amplifier for.
2691 config : `lsst.ip.isr.isrTaskConfig`
2692 Configuration to apply to the fake amplifier.
2696 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2698 self.
_gain_gain = config.gain
2703 return self.
_bbox_bbox
2706 return self.
_bbox_bbox
2712 return self.
_gain_gain
2725 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2729 """Task to wrap the default IsrTask to allow it to be retargeted.
2731 The standard IsrTask can be called directly from a command line
2732 program, but doing so removes the ability of the task to be
2733 retargeted. As most cameras override some set of the IsrTask
2734 methods, this would remove those data-specific methods in the
2735 output post-ISR images. This wrapping class fixes the issue,
2736 allowing identical post-ISR images to be generated by both the
2737 processCcd and isrTask code.
2739 ConfigClass = RunIsrConfig
2740 _DefaultName =
"runIsr"
2744 self.makeSubtask(
"isr")
2750 dataRef : `lsst.daf.persistence.ButlerDataRef`
2751 data reference of the detector data to be processed
2755 result : `pipeBase.Struct`
2756 Result struct with component:
2758 - exposure : `lsst.afw.image.Exposure`
2759 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def ensureExposure(self, inputExp, camera, detectorNum)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.