28 import lsst.pex.config
as pexConfig
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
44 from .
import isrFunctions
46 from .
import linearize
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument",
"calibration_label"],
115 crosstalk = cT.PrerequisiteInput(
117 doc=
"Input crosstalk object",
118 storageClass=
"CrosstalkCalib",
119 dimensions=[
"instrument",
"calibration_label",
"detector"],
123 crosstalkSources = cT.PrerequisiteInput(
124 name=
"isrOverscanCorrected",
125 doc=
"Overscan corrected input images.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
130 lookupFunction=crosstalkSourceLookup,
132 bias = cT.PrerequisiteInput(
134 doc=
"Input bias calibration.",
135 storageClass=
"ExposureF",
136 dimensions=[
"instrument",
"calibration_label",
"detector"],
138 dark = cT.PrerequisiteInput(
140 doc=
"Input dark calibration.",
141 storageClass=
"ExposureF",
142 dimensions=[
"instrument",
"calibration_label",
"detector"],
144 flat = cT.PrerequisiteInput(
146 doc=
"Input flat calibration.",
147 storageClass=
"ExposureF",
148 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
150 fringes = cT.PrerequisiteInput(
152 doc=
"Input fringe calibration.",
153 storageClass=
"ExposureF",
154 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
156 strayLightData = cT.PrerequisiteInput(
158 doc=
"Input stray light calibration.",
159 storageClass=
"StrayLightData",
160 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
162 bfKernel = cT.PrerequisiteInput(
164 doc=
"Input brighter-fatter kernel.",
165 storageClass=
"NumpyArray",
166 dimensions=[
"instrument",
"calibration_label"],
168 newBFKernel = cT.PrerequisiteInput(
169 name=
'brighterFatterKernel',
170 doc=
"Newer complete kernel + gain solutions.",
171 storageClass=
"BrighterFatterKernel",
172 dimensions=[
"instrument",
"calibration_label",
"detector"],
174 defects = cT.PrerequisiteInput(
176 doc=
"Input defect tables.",
177 storageClass=
"Defects",
178 dimensions=[
"instrument",
"calibration_label",
"detector"],
180 opticsTransmission = cT.PrerequisiteInput(
181 name=
"transmission_optics",
182 storageClass=
"TransmissionCurve",
183 doc=
"Transmission curve due to the optics.",
184 dimensions=[
"instrument",
"calibration_label"],
186 filterTransmission = cT.PrerequisiteInput(
187 name=
"transmission_filter",
188 storageClass=
"TransmissionCurve",
189 doc=
"Transmission curve due to the filter.",
190 dimensions=[
"instrument",
"physical_filter",
"calibration_label"],
192 sensorTransmission = cT.PrerequisiteInput(
193 name=
"transmission_sensor",
194 storageClass=
"TransmissionCurve",
195 doc=
"Transmission curve due to the sensor.",
196 dimensions=[
"instrument",
"calibration_label",
"detector"],
198 atmosphereTransmission = cT.PrerequisiteInput(
199 name=
"transmission_atmosphere",
200 storageClass=
"TransmissionCurve",
201 doc=
"Transmission curve due to the atmosphere.",
202 dimensions=[
"instrument"],
204 illumMaskedImage = cT.PrerequisiteInput(
206 doc=
"Input illumination correction.",
207 storageClass=
"MaskedImageF",
208 dimensions=[
"instrument",
"physical_filter",
"calibration_label",
"detector"],
211 outputExposure = cT.Output(
213 doc=
"Output ISR processed exposure.",
214 storageClass=
"Exposure",
215 dimensions=[
"instrument",
"exposure",
"detector"],
217 preInterpExposure = cT.Output(
218 name=
'preInterpISRCCD',
219 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
220 storageClass=
"ExposureF",
221 dimensions=[
"instrument",
"exposure",
"detector"],
223 outputOssThumbnail = cT.Output(
225 doc=
"Output Overscan-subtracted thumbnail image.",
226 storageClass=
"Thumbnail",
227 dimensions=[
"instrument",
"exposure",
"detector"],
229 outputFlattenedThumbnail = cT.Output(
230 name=
"FlattenedThumb",
231 doc=
"Output flat-corrected thumbnail image.",
232 storageClass=
"Thumbnail",
233 dimensions=[
"instrument",
"exposure",
"detector"],
239 if config.doBias
is not True:
240 self.prerequisiteInputs.discard(
"bias")
241 if config.doLinearize
is not True:
242 self.prerequisiteInputs.discard(
"linearizer")
243 if config.doCrosstalk
is not True:
244 self.inputs.discard(
"crosstalkSources")
245 self.prerequisiteInputs.discard(
"crosstalk")
246 if config.doBrighterFatter
is not True:
247 self.prerequisiteInputs.discard(
"bfKernel")
248 self.prerequisiteInputs.discard(
"newBFKernel")
249 if config.doDefect
is not True:
250 self.prerequisiteInputs.discard(
"defects")
251 if config.doDark
is not True:
252 self.prerequisiteInputs.discard(
"dark")
253 if config.doFlat
is not True:
254 self.prerequisiteInputs.discard(
"flat")
255 if config.doAttachTransmissionCurve
is not True:
256 self.prerequisiteInputs.discard(
"opticsTransmission")
257 self.prerequisiteInputs.discard(
"filterTransmission")
258 self.prerequisiteInputs.discard(
"sensorTransmission")
259 self.prerequisiteInputs.discard(
"atmosphereTransmission")
260 if config.doUseOpticsTransmission
is not True:
261 self.prerequisiteInputs.discard(
"opticsTransmission")
262 if config.doUseFilterTransmission
is not True:
263 self.prerequisiteInputs.discard(
"filterTransmission")
264 if config.doUseSensorTransmission
is not True:
265 self.prerequisiteInputs.discard(
"sensorTransmission")
266 if config.doUseAtmosphereTransmission
is not True:
267 self.prerequisiteInputs.discard(
"atmosphereTransmission")
268 if config.doIlluminationCorrection
is not True:
269 self.prerequisiteInputs.discard(
"illumMaskedImage")
271 if config.doWrite
is not True:
272 self.outputs.discard(
"outputExposure")
273 self.outputs.discard(
"preInterpExposure")
274 self.outputs.discard(
"outputFlattenedThumbnail")
275 self.outputs.discard(
"outputOssThumbnail")
276 if config.doSaveInterpPixels
is not True:
277 self.outputs.discard(
"preInterpExposure")
278 if config.qa.doThumbnailOss
is not True:
279 self.outputs.discard(
"outputOssThumbnail")
280 if config.qa.doThumbnailFlattened
is not True:
281 self.outputs.discard(
"outputFlattenedThumbnail")
285 pipelineConnections=IsrTaskConnections):
286 """Configuration parameters for IsrTask.
288 Items are grouped in the order in which they are executed by the task.
290 datasetType = pexConfig.Field(
292 doc=
"Dataset type for input data; users will typically leave this alone, "
293 "but camera-specific ISR tasks will override it",
297 fallbackFilterName = pexConfig.Field(
299 doc=
"Fallback default filter name for calibrations.",
302 useFallbackDate = pexConfig.Field(
304 doc=
"Pass observation date when using fallback filter.",
307 expectWcs = pexConfig.Field(
310 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
312 fwhm = pexConfig.Field(
314 doc=
"FWHM of PSF in arcseconds.",
317 qa = pexConfig.ConfigField(
319 doc=
"QA related configuration options.",
323 doConvertIntToFloat = pexConfig.Field(
325 doc=
"Convert integer raw images to floating point values?",
330 doSaturation = pexConfig.Field(
332 doc=
"Mask saturated pixels? NB: this is totally independent of the"
333 " interpolation option - this is ONLY setting the bits in the mask."
334 " To have them interpolated make sure doSaturationInterpolation=True",
337 saturatedMaskName = pexConfig.Field(
339 doc=
"Name of mask plane to use in saturation detection and interpolation",
342 saturation = pexConfig.Field(
344 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
345 default=float(
"NaN"),
347 growSaturationFootprintSize = pexConfig.Field(
349 doc=
"Number of pixels by which to grow the saturation footprints",
354 doSuspect = pexConfig.Field(
356 doc=
"Mask suspect pixels?",
359 suspectMaskName = pexConfig.Field(
361 doc=
"Name of mask plane to use for suspect pixels",
364 numEdgeSuspect = pexConfig.Field(
366 doc=
"Number of edge pixels to be flagged as untrustworthy.",
371 doSetBadRegions = pexConfig.Field(
373 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
376 badStatistic = pexConfig.ChoiceField(
378 doc=
"How to estimate the average value for BAD regions.",
381 "MEANCLIP":
"Correct using the (clipped) mean of good data",
382 "MEDIAN":
"Correct using the median of the good data",
387 doOverscan = pexConfig.Field(
389 doc=
"Do overscan subtraction?",
392 overscan = pexConfig.ConfigurableField(
393 target=OverscanCorrectionTask,
394 doc=
"Overscan subtraction task for image segments.",
397 overscanFitType = pexConfig.ChoiceField(
399 doc=
"The method for fitting the overscan bias level.",
402 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
403 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
404 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
405 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
406 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
407 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
408 "MEAN":
"Correct using the mean of the overscan region",
409 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
410 "MEDIAN":
"Correct using the median of the overscan region",
411 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
413 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
414 " This option will no longer be used, and will be removed after v20.")
416 overscanOrder = pexConfig.Field(
418 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
419 "or number of spline knots if overscan fit type is a spline."),
421 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
422 " This option will no longer be used, and will be removed after v20.")
424 overscanNumSigmaClip = pexConfig.Field(
426 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
428 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
429 " This option will no longer be used, and will be removed after v20.")
431 overscanIsInt = pexConfig.Field(
433 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
434 " and overscan.FitType=MEDIAN_PER_ROW.",
436 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
437 " This option will no longer be used, and will be removed after v20.")
440 overscanNumLeadingColumnsToSkip = pexConfig.Field(
442 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
445 overscanNumTrailingColumnsToSkip = pexConfig.Field(
447 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
450 overscanMaxDev = pexConfig.Field(
452 doc=
"Maximum deviation from the median for overscan",
453 default=1000.0, check=
lambda x: x > 0
455 overscanBiasJump = pexConfig.Field(
457 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
460 overscanBiasJumpKeyword = pexConfig.Field(
462 doc=
"Header keyword containing information about devices.",
463 default=
"NO_SUCH_KEY",
465 overscanBiasJumpDevices = pexConfig.ListField(
467 doc=
"List of devices that need piecewise overscan correction.",
470 overscanBiasJumpLocation = pexConfig.Field(
472 doc=
"Location of bias jump along y-axis.",
477 doAssembleCcd = pexConfig.Field(
480 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
482 assembleCcd = pexConfig.ConfigurableField(
483 target=AssembleCcdTask,
484 doc=
"CCD assembly task",
488 doAssembleIsrExposures = pexConfig.Field(
491 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
493 doTrimToMatchCalib = pexConfig.Field(
496 doc=
"Trim raw data to match calibration bounding boxes?"
500 doBias = pexConfig.Field(
502 doc=
"Apply bias frame correction?",
505 biasDataProductName = pexConfig.Field(
507 doc=
"Name of the bias data product",
512 doVariance = pexConfig.Field(
514 doc=
"Calculate variance?",
517 gain = pexConfig.Field(
519 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
520 default=float(
"NaN"),
522 readNoise = pexConfig.Field(
524 doc=
"The read noise to use if no Detector is present in the Exposure",
527 doEmpiricalReadNoise = pexConfig.Field(
530 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
534 doLinearize = pexConfig.Field(
536 doc=
"Correct for nonlinearity of the detector's response?",
541 doCrosstalk = pexConfig.Field(
543 doc=
"Apply intra-CCD crosstalk correction?",
546 doCrosstalkBeforeAssemble = pexConfig.Field(
548 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
551 crosstalk = pexConfig.ConfigurableField(
552 target=CrosstalkTask,
553 doc=
"Intra-CCD crosstalk correction",
557 doDefect = pexConfig.Field(
559 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
562 doNanMasking = pexConfig.Field(
564 doc=
"Mask NAN pixels?",
567 doWidenSaturationTrails = pexConfig.Field(
569 doc=
"Widen bleed trails based on their width?",
574 doBrighterFatter = pexConfig.Field(
577 doc=
"Apply the brighter fatter correction"
579 brighterFatterLevel = pexConfig.ChoiceField(
582 doc=
"The level at which to correct for brighter-fatter.",
584 "AMP":
"Every amplifier treated separately.",
585 "DETECTOR":
"One kernel per detector",
588 brighterFatterMaxIter = pexConfig.Field(
591 doc=
"Maximum number of iterations for the brighter fatter correction"
593 brighterFatterThreshold = pexConfig.Field(
596 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
597 " absolute value of the difference between the current corrected image and the one"
598 " from the previous iteration summed over all the pixels."
600 brighterFatterApplyGain = pexConfig.Field(
603 doc=
"Should the gain be applied when applying the brighter fatter correction?"
605 brighterFatterMaskGrowSize = pexConfig.Field(
608 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
609 " when brighter-fatter correction is applied."
613 doDark = pexConfig.Field(
615 doc=
"Apply dark frame correction?",
618 darkDataProductName = pexConfig.Field(
620 doc=
"Name of the dark data product",
625 doStrayLight = pexConfig.Field(
627 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
630 strayLight = pexConfig.ConfigurableField(
631 target=StrayLightTask,
632 doc=
"y-band stray light correction"
636 doFlat = pexConfig.Field(
638 doc=
"Apply flat field correction?",
641 flatDataProductName = pexConfig.Field(
643 doc=
"Name of the flat data product",
646 flatScalingType = pexConfig.ChoiceField(
648 doc=
"The method for scaling the flat on the fly.",
651 "USER":
"Scale by flatUserScale",
652 "MEAN":
"Scale by the inverse of the mean",
653 "MEDIAN":
"Scale by the inverse of the median",
656 flatUserScale = pexConfig.Field(
658 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
661 doTweakFlat = pexConfig.Field(
663 doc=
"Tweak flats to match observed amplifier ratios?",
668 doApplyGains = pexConfig.Field(
670 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
673 normalizeGains = pexConfig.Field(
675 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
680 doFringe = pexConfig.Field(
682 doc=
"Apply fringe correction?",
685 fringe = pexConfig.ConfigurableField(
687 doc=
"Fringe subtraction task",
689 fringeAfterFlat = pexConfig.Field(
691 doc=
"Do fringe subtraction after flat-fielding?",
696 doMeasureBackground = pexConfig.Field(
698 doc=
"Measure the background level on the reduced image?",
703 doCameraSpecificMasking = pexConfig.Field(
705 doc=
"Mask camera-specific bad regions?",
708 masking = pexConfig.ConfigurableField(
715 doInterpolate = pexConfig.Field(
717 doc=
"Interpolate masked pixels?",
720 doSaturationInterpolation = pexConfig.Field(
722 doc=
"Perform interpolation over pixels masked as saturated?"
723 " NB: This is independent of doSaturation; if that is False this plane"
724 " will likely be blank, resulting in a no-op here.",
727 doNanInterpolation = pexConfig.Field(
729 doc=
"Perform interpolation over pixels masked as NaN?"
730 " NB: This is independent of doNanMasking; if that is False this plane"
731 " will likely be blank, resulting in a no-op here.",
734 doNanInterpAfterFlat = pexConfig.Field(
736 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
737 "also have to interpolate them before flat-fielding."),
740 maskListToInterpolate = pexConfig.ListField(
742 doc=
"List of mask planes that should be interpolated.",
743 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
745 doSaveInterpPixels = pexConfig.Field(
747 doc=
"Save a copy of the pre-interpolated pixel values?",
752 fluxMag0T1 = pexConfig.DictField(
755 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
756 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
759 defaultFluxMag0T1 = pexConfig.Field(
761 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
762 default=pow(10.0, 0.4*28.0)
766 doVignette = pexConfig.Field(
768 doc=
"Apply vignetting parameters?",
771 vignette = pexConfig.ConfigurableField(
773 doc=
"Vignetting task.",
777 doAttachTransmissionCurve = pexConfig.Field(
780 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
782 doUseOpticsTransmission = pexConfig.Field(
785 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
787 doUseFilterTransmission = pexConfig.Field(
790 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
792 doUseSensorTransmission = pexConfig.Field(
795 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
797 doUseAtmosphereTransmission = pexConfig.Field(
800 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
804 doIlluminationCorrection = pexConfig.Field(
807 doc=
"Perform illumination correction?"
809 illuminationCorrectionDataProductName = pexConfig.Field(
811 doc=
"Name of the illumination correction data product.",
814 illumScale = pexConfig.Field(
816 doc=
"Scale factor for the illumination correction.",
819 illumFilters = pexConfig.ListField(
822 doc=
"Only perform illumination correction for these filters."
826 doWrite = pexConfig.Field(
828 doc=
"Persist postISRCCD?",
835 raise ValueError(
"You may not specify both doFlat and doApplyGains")
837 self.config.maskListToInterpolate.append(
"SAT")
839 self.config.maskListToInterpolate.append(
"UNMASKEDNAN")
842 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
843 """Apply common instrument signature correction algorithms to a raw frame.
845 The process for correcting imaging data is very similar from
846 camera to camera. This task provides a vanilla implementation of
847 doing these corrections, including the ability to turn certain
848 corrections off if they are not needed. The inputs to the primary
849 method, `run()`, are a raw exposure to be corrected and the
850 calibration data products. The raw input is a single chip sized
851 mosaic of all amps including overscans and other non-science
852 pixels. The method `runDataRef()` identifies and defines the
853 calibration data products, and is intended for use by a
854 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
855 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
856 subclassed for different camera, although the most camera specific
857 methods have been split into subtasks that can be redirected
860 The __init__ method sets up the subtasks for ISR processing, using
861 the defaults from `lsst.ip.isr`.
866 Positional arguments passed to the Task constructor. None used at this time.
867 kwargs : `dict`, optional
868 Keyword arguments passed on to the Task constructor. None used at this time.
870 ConfigClass = IsrTaskConfig
875 self.makeSubtask(
"assembleCcd")
876 self.makeSubtask(
"crosstalk")
877 self.makeSubtask(
"strayLight")
878 self.makeSubtask(
"fringe")
879 self.makeSubtask(
"masking")
880 self.makeSubtask(
"overscan")
881 self.makeSubtask(
"vignette")
884 inputs = butlerQC.get(inputRefs)
887 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
888 except Exception
as e:
889 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
892 inputs[
'isGen3'] =
True
894 detector = inputs[
'ccdExposure'].getDetector()
896 if self.config.doCrosstalk
is True:
899 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
900 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
901 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
903 coeffVector = (self.config.crosstalk.crosstalkValues
904 if self.config.crosstalk.useConfigCoefficients
else None)
905 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
906 inputs[
'crosstalk'] = crosstalkCalib
907 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
908 if 'crosstalkSources' not in inputs:
909 self.log.warn(
"No crosstalkSources found for chip with interChip terms!")
912 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
914 linearizer.fromYaml(inputs[
'linearizer'])
918 inputs[
'linearizer'] = linearizer
920 if self.config.doDefect
is True:
921 if "defects" in inputs
and inputs[
'defects']
is not None:
924 if not isinstance(inputs[
"defects"], Defects):
925 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
929 if self.config.doBrighterFatter:
930 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
931 if brighterFatterKernel
is None:
932 brighterFatterKernel = inputs.get(
'bfKernel',
None)
934 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
935 detId = detector.getId()
936 inputs[
'bfGains'] = brighterFatterKernel.gain
939 if self.config.brighterFatterLevel ==
'DETECTOR':
940 if brighterFatterKernel.detectorKernel:
941 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
942 elif brighterFatterKernel.detectorKernelFromAmpKernels:
943 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
945 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
948 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
950 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
951 expId = inputs[
'ccdExposure'].getInfo().getVisitInfo().getExposureId()
952 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
954 assembler=self.assembleCcd
955 if self.config.doAssembleIsrExposures
else None)
957 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
959 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
960 if 'strayLightData' not in inputs:
961 inputs[
'strayLightData'] =
None
963 outputs = self.
run(**inputs)
964 butlerQC.put(outputs, outputRefs)
967 """!Retrieve necessary frames for instrument signature removal.
969 Pre-fetching all required ISR data products limits the IO
970 required by the ISR. Any conflict between the calibration data
971 available and that needed for ISR is also detected prior to
972 doing processing, allowing it to fail quickly.
976 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
977 Butler reference of the detector data to be processed
978 rawExposure : `afw.image.Exposure`
979 The raw exposure that will later be corrected with the
980 retrieved calibration data; should not be modified in this
985 result : `lsst.pipe.base.Struct`
986 Result struct with components (which may be `None`):
987 - ``bias``: bias calibration frame (`afw.image.Exposure`)
988 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
989 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
990 - ``dark``: dark calibration frame (`afw.image.Exposure`)
991 - ``flat``: flat calibration frame (`afw.image.Exposure`)
992 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
993 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
994 - ``fringes``: `lsst.pipe.base.Struct` with components:
995 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
996 - ``seed``: random seed derived from the ccdExposureId for random
997 number generator (`uint32`).
998 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
999 A ``TransmissionCurve`` that represents the throughput of the optics,
1000 to be evaluated in focal-plane coordinates.
1001 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1002 A ``TransmissionCurve`` that represents the throughput of the filter
1003 itself, to be evaluated in focal-plane coordinates.
1004 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1005 A ``TransmissionCurve`` that represents the throughput of the sensor
1006 itself, to be evaluated in post-assembly trimmed detector coordinates.
1007 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1008 A ``TransmissionCurve`` that represents the throughput of the
1009 atmosphere, assumed to be spatially constant.
1010 - ``strayLightData`` : `object`
1011 An opaque object containing calibration information for
1012 stray-light correction. If `None`, no correction will be
1014 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1018 NotImplementedError :
1019 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1022 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1023 dateObs = dateObs.toPython().isoformat()
1024 except RuntimeError:
1025 self.log.warn(
"Unable to identify dateObs for rawExposure.")
1028 ccd = rawExposure.getDetector()
1029 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName()
1030 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1031 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
1032 if self.config.doBias
else None)
1034 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1036 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1037 linearizer.log = self.log
1038 if isinstance(linearizer, numpy.ndarray):
1041 crosstalkCalib =
None
1042 if self.config.doCrosstalk:
1044 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1046 coeffVector = (self.config.crosstalk.crosstalkValues
1047 if self.config.crosstalk.useConfigCoefficients
else None)
1048 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1049 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1050 if self.config.doCrosstalk
else None)
1052 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
1053 if self.config.doDark
else None)
1054 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
1056 if self.config.doFlat
else None)
1058 brighterFatterKernel =
None
1059 brighterFatterGains =
None
1060 if self.config.doBrighterFatter
is True:
1065 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1066 brighterFatterGains = brighterFatterKernel.gain
1067 self.log.info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1070 brighterFatterKernel = dataRef.get(
"bfKernel")
1071 self.log.info(
"Old style bright-fatter kernel (np.array) loaded")
1073 brighterFatterKernel =
None
1074 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1077 if self.config.brighterFatterLevel ==
'DETECTOR':
1078 if brighterFatterKernel.detectorKernel:
1079 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1080 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1081 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1083 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1086 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1088 defectList = (dataRef.get(
"defects")
1089 if self.config.doDefect
else None)
1090 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1091 if self.config.doAssembleIsrExposures
else None)
1092 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1093 else pipeBase.Struct(fringes=
None))
1095 if self.config.doAttachTransmissionCurve:
1096 opticsTransmission = (dataRef.get(
"transmission_optics")
1097 if self.config.doUseOpticsTransmission
else None)
1098 filterTransmission = (dataRef.get(
"transmission_filter")
1099 if self.config.doUseFilterTransmission
else None)
1100 sensorTransmission = (dataRef.get(
"transmission_sensor")
1101 if self.config.doUseSensorTransmission
else None)
1102 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1103 if self.config.doUseAtmosphereTransmission
else None)
1105 opticsTransmission =
None
1106 filterTransmission =
None
1107 sensorTransmission =
None
1108 atmosphereTransmission =
None
1110 if self.config.doStrayLight:
1111 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1113 strayLightData =
None
1116 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1117 if (self.config.doIlluminationCorrection
1118 and filterName
in self.config.illumFilters)
1122 return pipeBase.Struct(bias=biasExposure,
1123 linearizer=linearizer,
1124 crosstalk=crosstalkCalib,
1125 crosstalkSources=crosstalkSources,
1128 bfKernel=brighterFatterKernel,
1129 bfGains=brighterFatterGains,
1131 fringes=fringeStruct,
1132 opticsTransmission=opticsTransmission,
1133 filterTransmission=filterTransmission,
1134 sensorTransmission=sensorTransmission,
1135 atmosphereTransmission=atmosphereTransmission,
1136 strayLightData=strayLightData,
1137 illumMaskedImage=illumMaskedImage
1140 @pipeBase.timeMethod
1141 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1142 crosstalk=None, crosstalkSources=None,
1143 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1144 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1145 sensorTransmission=
None, atmosphereTransmission=
None,
1146 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1149 """!Perform instrument signature removal on an exposure.
1151 Steps included in the ISR processing, in order performed, are:
1152 - saturation and suspect pixel masking
1153 - overscan subtraction
1154 - CCD assembly of individual amplifiers
1156 - variance image construction
1157 - linearization of non-linear response
1159 - brighter-fatter correction
1162 - stray light subtraction
1164 - masking of known defects and camera specific features
1165 - vignette calculation
1166 - appending transmission curve and distortion model
1170 ccdExposure : `lsst.afw.image.Exposure`
1171 The raw exposure that is to be run through ISR. The
1172 exposure is modified by this method.
1173 camera : `lsst.afw.cameraGeom.Camera`, optional
1174 The camera geometry for this exposure. Required if ``isGen3`` is
1175 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1176 ``flat`` does not have an associated detector.
1177 bias : `lsst.afw.image.Exposure`, optional
1178 Bias calibration frame.
1179 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1180 Functor for linearization.
1181 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1182 Calibration for crosstalk.
1183 crosstalkSources : `list`, optional
1184 List of possible crosstalk sources.
1185 dark : `lsst.afw.image.Exposure`, optional
1186 Dark calibration frame.
1187 flat : `lsst.afw.image.Exposure`, optional
1188 Flat calibration frame.
1189 bfKernel : `numpy.ndarray`, optional
1190 Brighter-fatter kernel.
1191 bfGains : `dict` of `float`, optional
1192 Gains used to override the detector's nominal gains for the
1193 brighter-fatter correction. A dict keyed by amplifier name for
1194 the detector in question.
1195 defects : `lsst.meas.algorithms.Defects`, optional
1197 fringes : `lsst.pipe.base.Struct`, optional
1198 Struct containing the fringe correction data, with
1200 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1201 - ``seed``: random seed derived from the ccdExposureId for random
1202 number generator (`uint32`)
1203 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1204 A ``TransmissionCurve`` that represents the throughput of the optics,
1205 to be evaluated in focal-plane coordinates.
1206 filterTransmission : `lsst.afw.image.TransmissionCurve`
1207 A ``TransmissionCurve`` that represents the throughput of the filter
1208 itself, to be evaluated in focal-plane coordinates.
1209 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1210 A ``TransmissionCurve`` that represents the throughput of the sensor
1211 itself, to be evaluated in post-assembly trimmed detector coordinates.
1212 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1213 A ``TransmissionCurve`` that represents the throughput of the
1214 atmosphere, assumed to be spatially constant.
1215 detectorNum : `int`, optional
1216 The integer number for the detector to process.
1217 isGen3 : bool, optional
1218 Flag this call to run() as using the Gen3 butler environment.
1219 strayLightData : `object`, optional
1220 Opaque object containing calibration information for stray-light
1221 correction. If `None`, no correction will be performed.
1222 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1223 Illumination correction image.
1227 result : `lsst.pipe.base.Struct`
1228 Result struct with component:
1229 - ``exposure`` : `afw.image.Exposure`
1230 The fully ISR corrected exposure.
1231 - ``outputExposure`` : `afw.image.Exposure`
1232 An alias for `exposure`
1233 - ``ossThumb`` : `numpy.ndarray`
1234 Thumbnail image of the exposure after overscan subtraction.
1235 - ``flattenedThumb`` : `numpy.ndarray`
1236 Thumbnail image of the exposure after flat-field correction.
1241 Raised if a configuration option is set to True, but the
1242 required calibration data has not been specified.
1246 The current processed exposure can be viewed by setting the
1247 appropriate lsstDebug entries in the `debug.display`
1248 dictionary. The names of these entries correspond to some of
1249 the IsrTaskConfig Boolean options, with the value denoting the
1250 frame to use. The exposure is shown inside the matching
1251 option check and after the processing of that step has
1252 finished. The steps with debug points are:
1263 In addition, setting the "postISRCCD" entry displays the
1264 exposure after all ISR processing has finished.
1272 if detectorNum
is None:
1273 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1275 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1280 if isinstance(ccdExposure, ButlerDataRef):
1283 ccd = ccdExposure.getDetector()
1284 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1287 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1288 ccd = [
FakeAmp(ccdExposure, self.config)]
1291 if self.config.doBias
and bias
is None:
1292 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1294 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1295 if self.config.doBrighterFatter
and bfKernel
is None:
1296 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1297 if self.config.doDark
and dark
is None:
1298 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1299 if self.config.doFlat
and flat
is None:
1300 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1301 if self.config.doDefect
and defects
is None:
1302 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1303 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1304 and fringes.fringes
is None):
1309 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1310 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1311 and illumMaskedImage
is None):
1312 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1315 if self.config.doConvertIntToFloat:
1316 self.log.info(
"Converting exposure to floating point values.")
1323 if ccdExposure.getBBox().contains(amp.getBBox()):
1327 if self.config.doOverscan
and not badAmp:
1330 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1331 if overscanResults
is not None and \
1332 self.config.qa
is not None and self.config.qa.saveStats
is True:
1333 if isinstance(overscanResults.overscanFit, float):
1334 qaMedian = overscanResults.overscanFit
1335 qaStdev = float(
"NaN")
1337 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1338 afwMath.MEDIAN | afwMath.STDEVCLIP)
1339 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1340 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1342 self.metadata.set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1343 self.metadata.set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1344 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1345 amp.getName(), qaMedian, qaStdev)
1346 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1349 self.log.warn(
"Amplifier %s is bad.", amp.getName())
1350 overscanResults =
None
1352 overscans.append(overscanResults
if overscanResults
is not None else None)
1354 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1356 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1357 self.log.info(
"Applying crosstalk correction.")
1358 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1359 crosstalkSources=crosstalkSources)
1360 self.
debugView(ccdExposure,
"doCrosstalk")
1362 if self.config.doAssembleCcd:
1363 self.log.info(
"Assembling CCD from amplifiers.")
1364 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1366 if self.config.expectWcs
and not ccdExposure.getWcs():
1367 self.log.warn(
"No WCS found in input exposure.")
1368 self.
debugView(ccdExposure,
"doAssembleCcd")
1371 if self.config.qa.doThumbnailOss:
1372 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1374 if self.config.doBias:
1375 self.log.info(
"Applying bias correction.")
1376 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1377 trimToFit=self.config.doTrimToMatchCalib)
1380 if self.config.doVariance:
1381 for amp, overscanResults
in zip(ccd, overscans):
1382 if ccdExposure.getBBox().contains(amp.getBBox()):
1383 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1384 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1385 if overscanResults
is not None:
1387 overscanImage=overscanResults.overscanImage)
1391 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1392 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1393 afwMath.MEDIAN | afwMath.STDEVCLIP)
1394 self.metadata.set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1395 qaStats.getValue(afwMath.MEDIAN))
1396 self.metadata.set(f
"ISR VARIANCE {amp.getName()} STDEV",
1397 qaStats.getValue(afwMath.STDEVCLIP))
1398 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1399 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1400 qaStats.getValue(afwMath.STDEVCLIP))
1403 self.log.info(
"Applying linearizer.")
1404 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1405 detector=ccd, log=self.log)
1407 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1408 self.log.info(
"Applying crosstalk correction.")
1409 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1410 crosstalkSources=crosstalkSources, isTrimmed=
True)
1411 self.
debugView(ccdExposure,
"doCrosstalk")
1415 if self.config.doDefect:
1416 self.log.info(
"Masking defects.")
1419 if self.config.numEdgeSuspect > 0:
1420 self.log.info(
"Masking edges as SUSPECT.")
1421 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1422 maskPlane=
"SUSPECT")
1424 if self.config.doNanMasking:
1425 self.log.info(
"Masking NAN value pixels.")
1428 if self.config.doWidenSaturationTrails:
1429 self.log.info(
"Widening saturation trails.")
1430 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1432 if self.config.doCameraSpecificMasking:
1433 self.log.info(
"Masking regions for camera specific reasons.")
1434 self.masking.
run(ccdExposure)
1436 if self.config.doBrighterFatter:
1445 interpExp = ccdExposure.clone()
1447 isrFunctions.interpolateFromMask(
1448 maskedImage=interpExp.getMaskedImage(),
1449 fwhm=self.config.fwhm,
1450 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1451 maskNameList=self.config.maskListToInterpolate
1453 bfExp = interpExp.clone()
1455 self.log.info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1456 type(bfKernel), type(bfGains))
1457 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1458 self.config.brighterFatterMaxIter,
1459 self.config.brighterFatterThreshold,
1460 self.config.brighterFatterApplyGain,
1462 if bfResults[1] == self.config.brighterFatterMaxIter:
1463 self.log.warn(
"Brighter fatter correction did not converge, final difference %f.",
1466 self.log.info(
"Finished brighter fatter correction in %d iterations.",
1468 image = ccdExposure.getMaskedImage().getImage()
1469 bfCorr = bfExp.getMaskedImage().getImage()
1470 bfCorr -= interpExp.getMaskedImage().getImage()
1479 self.log.info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1480 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1483 if self.config.brighterFatterMaskGrowSize > 0:
1484 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1485 for maskPlane
in self.config.maskListToInterpolate:
1486 isrFunctions.growMasks(ccdExposure.getMask(),
1487 radius=self.config.brighterFatterMaskGrowSize,
1488 maskNameList=maskPlane,
1489 maskValue=maskPlane)
1491 self.
debugView(ccdExposure,
"doBrighterFatter")
1493 if self.config.doDark:
1494 self.log.info(
"Applying dark correction.")
1498 if self.config.doFringe
and not self.config.fringeAfterFlat:
1499 self.log.info(
"Applying fringe correction before flat.")
1500 self.fringe.
run(ccdExposure, **fringes.getDict())
1503 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1504 self.log.info(
"Checking strayLight correction.")
1505 self.strayLight.
run(ccdExposure, strayLightData)
1506 self.
debugView(ccdExposure,
"doStrayLight")
1508 if self.config.doFlat:
1509 self.log.info(
"Applying flat correction.")
1513 if self.config.doApplyGains:
1514 self.log.info(
"Applying gain correction instead of flat.")
1515 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1517 if self.config.doFringe
and self.config.fringeAfterFlat:
1518 self.log.info(
"Applying fringe correction after flat.")
1519 self.fringe.
run(ccdExposure, **fringes.getDict())
1521 if self.config.doVignette:
1522 self.log.info(
"Constructing Vignette polygon.")
1525 if self.config.vignette.doWriteVignettePolygon:
1528 if self.config.doAttachTransmissionCurve:
1529 self.log.info(
"Adding transmission curves.")
1530 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1531 filterTransmission=filterTransmission,
1532 sensorTransmission=sensorTransmission,
1533 atmosphereTransmission=atmosphereTransmission)
1535 flattenedThumb =
None
1536 if self.config.qa.doThumbnailFlattened:
1537 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1539 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1540 self.log.info(
"Performing illumination correction.")
1541 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1542 illumMaskedImage, illumScale=self.config.illumScale,
1543 trimToFit=self.config.doTrimToMatchCalib)
1546 if self.config.doSaveInterpPixels:
1547 preInterpExp = ccdExposure.clone()
1562 if self.config.doSetBadRegions:
1563 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1564 if badPixelCount > 0:
1565 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1567 if self.config.doInterpolate:
1568 self.log.info(
"Interpolating masked pixels.")
1569 isrFunctions.interpolateFromMask(
1570 maskedImage=ccdExposure.getMaskedImage(),
1571 fwhm=self.config.fwhm,
1572 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1573 maskNameList=list(self.config.maskListToInterpolate)
1578 if self.config.doMeasureBackground:
1579 self.log.info(
"Measuring background level.")
1582 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1584 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1585 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1586 afwMath.MEDIAN | afwMath.STDEVCLIP)
1587 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1588 qaStats.getValue(afwMath.MEDIAN))
1589 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1590 qaStats.getValue(afwMath.STDEVCLIP))
1591 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1592 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1593 qaStats.getValue(afwMath.STDEVCLIP))
1595 self.
debugView(ccdExposure,
"postISRCCD")
1597 return pipeBase.Struct(
1598 exposure=ccdExposure,
1600 flattenedThumb=flattenedThumb,
1602 preInterpolatedExposure=preInterpExp,
1603 outputExposure=ccdExposure,
1604 outputOssThumbnail=ossThumb,
1605 outputFlattenedThumbnail=flattenedThumb,
1608 @pipeBase.timeMethod
1610 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1612 This method contains the `CmdLineTask` interface to the ISR
1613 processing. All IO is handled here, freeing the `run()` method
1614 to manage only pixel-level calculations. The steps performed
1616 - Read in necessary detrending/isr/calibration data.
1617 - Process raw exposure in `run()`.
1618 - Persist the ISR-corrected exposure as "postISRCCD" if
1619 config.doWrite=True.
1623 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1624 DataRef of the detector data to be processed
1628 result : `lsst.pipe.base.Struct`
1629 Result struct with component:
1630 - ``exposure`` : `afw.image.Exposure`
1631 The fully ISR corrected exposure.
1636 Raised if a configuration option is set to True, but the
1637 required calibration data does not exist.
1640 self.log.info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1642 ccdExposure = sensorRef.get(self.config.datasetType)
1644 camera = sensorRef.get(
"camera")
1645 isrData = self.
readIsrData(sensorRef, ccdExposure)
1647 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1649 if self.config.doWrite:
1650 sensorRef.put(result.exposure,
"postISRCCD")
1651 if result.preInterpolatedExposure
is not None:
1652 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1653 if result.ossThumb
is not None:
1654 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1655 if result.flattenedThumb
is not None:
1656 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1661 """!Retrieve a calibration dataset for removing instrument signature.
1666 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1667 DataRef of the detector data to find calibration datasets
1670 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1671 dateObs : `str`, optional
1672 Date of the observation. Used to correct butler failures
1673 when using fallback filters.
1675 If True, disable butler proxies to enable error handling
1676 within this routine.
1680 exposure : `lsst.afw.image.Exposure`
1681 Requested calibration frame.
1686 Raised if no matching calibration frame can be found.
1689 exp = dataRef.get(datasetType, immediate=immediate)
1690 except Exception
as exc1:
1691 if not self.config.fallbackFilterName:
1692 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1694 if self.config.useFallbackDate
and dateObs:
1695 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1696 dateObs=dateObs, immediate=immediate)
1698 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1699 except Exception
as exc2:
1700 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1701 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1702 self.log.warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1704 if self.config.doAssembleIsrExposures:
1705 exp = self.assembleCcd.assembleCcd(exp)
1709 """Ensure that the data returned by Butler is a fully constructed exposure.
1711 ISR requires exposure-level image data for historical reasons, so if we did
1712 not recieve that from Butler, construct it from what we have, modifying the
1717 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1718 `lsst.afw.image.ImageF`
1719 The input data structure obtained from Butler.
1720 camera : `lsst.afw.cameraGeom.camera`
1721 The camera associated with the image. Used to find the appropriate
1724 The detector this exposure should match.
1728 inputExp : `lsst.afw.image.Exposure`
1729 The re-constructed exposure, with appropriate detector parameters.
1734 Raised if the input data cannot be used to construct an exposure.
1736 if isinstance(inputExp, afwImage.DecoratedImageU):
1737 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1738 elif isinstance(inputExp, afwImage.ImageF):
1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1740 elif isinstance(inputExp, afwImage.MaskedImageF):
1741 inputExp = afwImage.makeExposure(inputExp)
1742 elif isinstance(inputExp, afwImage.Exposure):
1744 elif inputExp
is None:
1748 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1751 if inputExp.getDetector()
is None:
1752 inputExp.setDetector(camera[detectorNum])
1757 """Convert exposure image from uint16 to float.
1759 If the exposure does not need to be converted, the input is
1760 immediately returned. For exposures that are converted to use
1761 floating point pixels, the variance is set to unity and the
1766 exposure : `lsst.afw.image.Exposure`
1767 The raw exposure to be converted.
1771 newexposure : `lsst.afw.image.Exposure`
1772 The input ``exposure``, converted to floating point pixels.
1777 Raised if the exposure type cannot be converted to float.
1780 if isinstance(exposure, afwImage.ExposureF):
1782 self.log.debug(
"Exposure already of type float.")
1784 if not hasattr(exposure,
"convertF"):
1785 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1787 newexposure = exposure.convertF()
1788 newexposure.variance[:] = 1
1789 newexposure.mask[:] = 0x0
1794 """Identify bad amplifiers, saturated and suspect pixels.
1798 ccdExposure : `lsst.afw.image.Exposure`
1799 Input exposure to be masked.
1800 amp : `lsst.afw.table.AmpInfoCatalog`
1801 Catalog of parameters defining the amplifier on this
1803 defects : `lsst.meas.algorithms.Defects`
1804 List of defects. Used to determine if the entire
1810 If this is true, the entire amplifier area is covered by
1811 defects and unusable.
1814 maskedImage = ccdExposure.getMaskedImage()
1820 if defects
is not None:
1821 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1826 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1828 maskView = dataView.getMask()
1829 maskView |= maskView.getPlaneBitMask(
"BAD")
1836 if self.config.doSaturation
and not badAmp:
1837 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1838 if self.config.doSuspect
and not badAmp:
1839 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1840 if math.isfinite(self.config.saturation):
1841 limits.update({self.config.saturatedMaskName: self.config.saturation})
1843 for maskName, maskThreshold
in limits.items():
1844 if not math.isnan(maskThreshold):
1845 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1846 isrFunctions.makeThresholdMask(
1847 maskedImage=dataView,
1848 threshold=maskThreshold,
1854 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1856 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1857 self.config.suspectMaskName])
1858 if numpy.all(maskView.getArray() & maskVal > 0):
1860 maskView |= maskView.getPlaneBitMask(
"BAD")
1865 """Apply overscan correction in place.
1867 This method does initial pixel rejection of the overscan
1868 region. The overscan can also be optionally segmented to
1869 allow for discontinuous overscan responses to be fit
1870 separately. The actual overscan subtraction is performed by
1871 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1872 which is called here after the amplifier is preprocessed.
1876 ccdExposure : `lsst.afw.image.Exposure`
1877 Exposure to have overscan correction performed.
1878 amp : `lsst.afw.table.AmpInfoCatalog`
1879 The amplifier to consider while correcting the overscan.
1883 overscanResults : `lsst.pipe.base.Struct`
1884 Result struct with components:
1885 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1886 Value or fit subtracted from the amplifier image data.
1887 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1888 Value or fit subtracted from the overscan image data.
1889 - ``overscanImage`` : `lsst.afw.image.Image`
1890 Image of the overscan region with the overscan
1891 correction applied. This quantity is used to estimate
1892 the amplifier read noise empirically.
1897 Raised if the ``amp`` does not contain raw pixel information.
1901 lsst.ip.isr.isrFunctions.overscanCorrection
1903 if amp.getRawHorizontalOverscanBBox().isEmpty():
1904 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1907 statControl = afwMath.StatisticsControl()
1908 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1911 dataBBox = amp.getRawDataBBox()
1912 oscanBBox = amp.getRawHorizontalOverscanBBox()
1916 prescanBBox = amp.getRawPrescanBBox()
1917 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1918 dx0 += self.config.overscanNumLeadingColumnsToSkip
1919 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1921 dx0 += self.config.overscanNumTrailingColumnsToSkip
1922 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1928 if ((self.config.overscanBiasJump
1929 and self.config.overscanBiasJumpLocation)
1930 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1931 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
1932 self.config.overscanBiasJumpDevices)):
1933 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1934 yLower = self.config.overscanBiasJumpLocation
1935 yUpper = dataBBox.getHeight() - yLower
1937 yUpper = self.config.overscanBiasJumpLocation
1938 yLower = dataBBox.getHeight() - yUpper
1956 oscanBBox.getHeight())))
1959 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1960 ampImage = ccdExposure.maskedImage[imageBBox]
1961 overscanImage = ccdExposure.maskedImage[overscanBBox]
1963 overscanArray = overscanImage.image.array
1964 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1965 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1966 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
1968 statControl = afwMath.StatisticsControl()
1969 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1971 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage)
1974 levelStat = afwMath.MEDIAN
1975 sigmaStat = afwMath.STDEVCLIP
1977 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1978 self.config.qa.flatness.nIter)
1979 metadata = ccdExposure.getMetadata()
1980 ampNum = amp.getName()
1982 if isinstance(overscanResults.overscanFit, float):
1983 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1984 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1986 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1987 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1988 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1990 return overscanResults
1993 """Set the variance plane using the amplifier gain and read noise
1995 The read noise is calculated from the ``overscanImage`` if the
1996 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1997 the value from the amplifier data is used.
2001 ampExposure : `lsst.afw.image.Exposure`
2002 Exposure to process.
2003 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2004 Amplifier detector data.
2005 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2006 Image of overscan, required only for empirical read noise.
2010 lsst.ip.isr.isrFunctions.updateVariance
2012 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2013 gain = amp.getGain()
2015 if math.isnan(gain):
2017 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2020 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2021 amp.getName(), gain, patchedGain)
2024 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2025 self.log.info(
"Overscan is none for EmpiricalReadNoise.")
2027 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2028 stats = afwMath.StatisticsControl()
2029 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2030 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2031 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2032 amp.getName(), readNoise)
2034 readNoise = amp.getReadNoise()
2036 isrFunctions.updateVariance(
2037 maskedImage=ampExposure.getMaskedImage(),
2039 readNoise=readNoise,
2043 """!Apply dark correction in place.
2047 exposure : `lsst.afw.image.Exposure`
2048 Exposure to process.
2049 darkExposure : `lsst.afw.image.Exposure`
2050 Dark exposure of the same size as ``exposure``.
2051 invert : `Bool`, optional
2052 If True, re-add the dark to an already corrected image.
2057 Raised if either ``exposure`` or ``darkExposure`` do not
2058 have their dark time defined.
2062 lsst.ip.isr.isrFunctions.darkCorrection
2064 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2065 if math.isnan(expScale):
2066 raise RuntimeError(
"Exposure darktime is NAN.")
2067 if darkExposure.getInfo().getVisitInfo()
is not None \
2068 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2069 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2073 self.log.warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2076 isrFunctions.darkCorrection(
2077 maskedImage=exposure.getMaskedImage(),
2078 darkMaskedImage=darkExposure.getMaskedImage(),
2080 darkScale=darkScale,
2082 trimToFit=self.config.doTrimToMatchCalib
2086 """!Check if linearization is needed for the detector cameraGeom.
2088 Checks config.doLinearize and the linearity type of the first
2093 detector : `lsst.afw.cameraGeom.Detector`
2094 Detector to get linearity type from.
2098 doLinearize : `Bool`
2099 If True, linearization should be performed.
2101 return self.config.doLinearize
and \
2102 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2105 """!Apply flat correction in place.
2109 exposure : `lsst.afw.image.Exposure`
2110 Exposure to process.
2111 flatExposure : `lsst.afw.image.Exposure`
2112 Flat exposure of the same size as ``exposure``.
2113 invert : `Bool`, optional
2114 If True, unflatten an already flattened image.
2118 lsst.ip.isr.isrFunctions.flatCorrection
2120 isrFunctions.flatCorrection(
2121 maskedImage=exposure.getMaskedImage(),
2122 flatMaskedImage=flatExposure.getMaskedImage(),
2123 scalingType=self.config.flatScalingType,
2124 userScale=self.config.flatUserScale,
2126 trimToFit=self.config.doTrimToMatchCalib
2130 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2134 exposure : `lsst.afw.image.Exposure`
2135 Exposure to process. Only the amplifier DataSec is processed.
2136 amp : `lsst.afw.table.AmpInfoCatalog`
2137 Amplifier detector data.
2141 lsst.ip.isr.isrFunctions.makeThresholdMask
2143 if not math.isnan(amp.getSaturation()):
2144 maskedImage = exposure.getMaskedImage()
2145 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2146 isrFunctions.makeThresholdMask(
2147 maskedImage=dataView,
2148 threshold=amp.getSaturation(),
2150 maskName=self.config.saturatedMaskName,
2154 """!Interpolate over saturated pixels, in place.
2156 This method should be called after `saturationDetection`, to
2157 ensure that the saturated pixels have been identified in the
2158 SAT mask. It should also be called after `assembleCcd`, since
2159 saturated regions may cross amplifier boundaries.
2163 exposure : `lsst.afw.image.Exposure`
2164 Exposure to process.
2168 lsst.ip.isr.isrTask.saturationDetection
2169 lsst.ip.isr.isrFunctions.interpolateFromMask
2171 isrFunctions.interpolateFromMask(
2172 maskedImage=exposure.getMaskedImage(),
2173 fwhm=self.config.fwhm,
2174 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2175 maskNameList=list(self.config.saturatedMaskName),
2179 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2183 exposure : `lsst.afw.image.Exposure`
2184 Exposure to process. Only the amplifier DataSec is processed.
2185 amp : `lsst.afw.table.AmpInfoCatalog`
2186 Amplifier detector data.
2190 lsst.ip.isr.isrFunctions.makeThresholdMask
2194 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2195 This is intended to indicate pixels that may be affected by unknown systematics;
2196 for example if non-linearity corrections above a certain level are unstable
2197 then that would be a useful value for suspectLevel. A value of `nan` indicates
2198 that no such level exists and no pixels are to be masked as suspicious.
2200 suspectLevel = amp.getSuspectLevel()
2201 if math.isnan(suspectLevel):
2204 maskedImage = exposure.getMaskedImage()
2205 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2206 isrFunctions.makeThresholdMask(
2207 maskedImage=dataView,
2208 threshold=suspectLevel,
2210 maskName=self.config.suspectMaskName,
2214 """!Mask defects using mask plane "BAD", in place.
2218 exposure : `lsst.afw.image.Exposure`
2219 Exposure to process.
2220 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2221 `lsst.afw.image.DefectBase`.
2222 List of defects to mask.
2226 Call this after CCD assembly, since defects may cross amplifier boundaries.
2228 maskedImage = exposure.getMaskedImage()
2229 if not isinstance(defectBaseList, Defects):
2231 defectList = Defects(defectBaseList)
2233 defectList = defectBaseList
2234 defectList.maskPixels(maskedImage, maskName=
"BAD")
2236 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2237 """!Mask edge pixels with applicable mask plane.
2241 exposure : `lsst.afw.image.Exposure`
2242 Exposure to process.
2243 numEdgePixels : `int`, optional
2244 Number of edge pixels to mask.
2245 maskPlane : `str`, optional
2246 Mask plane name to use.
2248 maskedImage = exposure.getMaskedImage()
2249 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2251 if numEdgePixels > 0:
2252 goodBBox = maskedImage.getBBox()
2254 goodBBox.grow(-numEdgePixels)
2256 SourceDetectionTask.setEdgeBits(
2263 """Mask and interpolate defects using mask plane "BAD", in place.
2267 exposure : `lsst.afw.image.Exposure`
2268 Exposure to process.
2269 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2270 `lsst.afw.image.DefectBase`.
2271 List of defects to mask and interpolate.
2275 lsst.ip.isr.isrTask.maskDefect()
2278 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2279 maskPlane=
"SUSPECT")
2280 isrFunctions.interpolateFromMask(
2281 maskedImage=exposure.getMaskedImage(),
2282 fwhm=self.config.fwhm,
2283 growSaturatedFootprints=0,
2284 maskNameList=[
"BAD"],
2288 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2292 exposure : `lsst.afw.image.Exposure`
2293 Exposure to process.
2297 We mask over all NaNs, including those that are masked with
2298 other bits (because those may or may not be interpolated over
2299 later, and we want to remove all NaNs). Despite this
2300 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2301 the historical name.
2303 maskedImage = exposure.getMaskedImage()
2306 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2307 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2308 numNans =
maskNans(maskedImage, maskVal)
2309 self.metadata.set(
"NUMNANS", numNans)
2311 self.log.warn(
"There were %d unmasked NaNs.", numNans)
2314 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2318 exposure : `lsst.afw.image.Exposure`
2319 Exposure to process.
2323 lsst.ip.isr.isrTask.maskNan()
2326 isrFunctions.interpolateFromMask(
2327 maskedImage=exposure.getMaskedImage(),
2328 fwhm=self.config.fwhm,
2329 growSaturatedFootprints=0,
2330 maskNameList=[
"UNMASKEDNAN"],
2334 """Measure the image background in subgrids, for quality control purposes.
2338 exposure : `lsst.afw.image.Exposure`
2339 Exposure to process.
2340 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2341 Configuration object containing parameters on which background
2342 statistics and subgrids to use.
2344 if IsrQaConfig
is not None:
2345 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2346 IsrQaConfig.flatness.nIter)
2347 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2348 statsControl.setAndMask(maskVal)
2349 maskedImage = exposure.getMaskedImage()
2350 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2351 skyLevel = stats.getValue(afwMath.MEDIAN)
2352 skySigma = stats.getValue(afwMath.STDEVCLIP)
2353 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2354 metadata = exposure.getMetadata()
2355 metadata.set(
'SKYLEVEL', skyLevel)
2356 metadata.set(
'SKYSIGMA', skySigma)
2359 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2360 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2361 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2362 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2363 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2364 skyLevels = numpy.zeros((nX, nY))
2367 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2369 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2371 xLLC = xc - meshXHalf
2372 yLLC = yc - meshYHalf
2373 xURC = xc + meshXHalf - 1
2374 yURC = yc + meshYHalf - 1
2377 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2379 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2381 good = numpy.where(numpy.isfinite(skyLevels))
2382 skyMedian = numpy.median(skyLevels[good])
2383 flatness = (skyLevels[good] - skyMedian) / skyMedian
2384 flatness_rms = numpy.std(flatness)
2385 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2387 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2388 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2389 nX, nY, flatness_pp, flatness_rms)
2391 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2392 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2393 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2394 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2395 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2398 """Set an approximate magnitude zero point for the exposure.
2402 exposure : `lsst.afw.image.Exposure`
2403 Exposure to process.
2405 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
2406 if filterName
in self.config.fluxMag0T1:
2407 fluxMag0 = self.config.fluxMag0T1[filterName]
2409 self.log.warn(
"No rough magnitude zero point set for filter %s.", filterName)
2410 fluxMag0 = self.config.defaultFluxMag0T1
2412 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2414 self.log.warn(
"Non-positive exposure time; skipping rough zero point.")
2417 self.log.info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2418 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2421 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2425 ccdExposure : `lsst.afw.image.Exposure`
2426 Exposure to process.
2427 fpPolygon : `lsst.afw.geom.Polygon`
2428 Polygon in focal plane coordinates.
2431 ccd = ccdExposure.getDetector()
2432 fpCorners = ccd.getCorners(FOCAL_PLANE)
2433 ccdPolygon = Polygon(fpCorners)
2436 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2439 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2440 validPolygon = Polygon(ccdPoints)
2441 ccdExposure.getInfo().setValidPolygon(validPolygon)
2445 """Context manager that applies and removes flats and darks,
2446 if the task is configured to apply them.
2450 exp : `lsst.afw.image.Exposure`
2451 Exposure to process.
2452 flat : `lsst.afw.image.Exposure`
2453 Flat exposure the same size as ``exp``.
2454 dark : `lsst.afw.image.Exposure`, optional
2455 Dark exposure the same size as ``exp``.
2459 exp : `lsst.afw.image.Exposure`
2460 The flat and dark corrected exposure.
2462 if self.config.doDark
and dark
is not None:
2464 if self.config.doFlat:
2469 if self.config.doFlat:
2471 if self.config.doDark
and dark
is not None:
2475 """Utility function to examine ISR exposure at different stages.
2479 exposure : `lsst.afw.image.Exposure`
2482 State of processing to view.
2484 frame = getDebugFrame(self._display, stepname)
2486 display = getDisplay(frame)
2487 display.scale(
'asinh',
'zscale')
2488 display.mtv(exposure)
2489 prompt =
"Press Enter to continue [c]... "
2491 ans = input(prompt).lower()
2492 if ans
in (
"",
"c",):
2497 """A Detector-like object that supports returning gain and saturation level
2499 This is used when the input exposure does not have a detector.
2503 exposure : `lsst.afw.image.Exposure`
2504 Exposure to generate a fake amplifier for.
2505 config : `lsst.ip.isr.isrTaskConfig`
2506 Configuration to apply to the fake amplifier.
2510 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2512 self.
_gain = config.gain
2539 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2543 """Task to wrap the default IsrTask to allow it to be retargeted.
2545 The standard IsrTask can be called directly from a command line
2546 program, but doing so removes the ability of the task to be
2547 retargeted. As most cameras override some set of the IsrTask
2548 methods, this would remove those data-specific methods in the
2549 output post-ISR images. This wrapping class fixes the issue,
2550 allowing identical post-ISR images to be generated by both the
2551 processCcd and isrTask code.
2553 ConfigClass = RunIsrConfig
2554 _DefaultName =
"runIsr"
2558 self.makeSubtask(
"isr")
2564 dataRef : `lsst.daf.persistence.ButlerDataRef`
2565 data reference of the detector data to be processed
2569 result : `pipeBase.Struct`
2570 Result struct with component:
2572 - exposure : `lsst.afw.image.Exposure`
2573 Post-ISR processed exposure.