29import lsst.pipe.base
as pipeBase
30import lsst.pipe.base.connectionTypes
as cT
32from contextlib
import contextmanager
33from lsstDebug
import getDebugFrame
38from lsst.daf.persistence.butler
import NoResults
40from lsst.utils.timer
import timeMethod
42from .
import isrFunctions
44from .
import linearize
45from .defects
import Defects
47from .assembleCcdTask
import AssembleCcdTask
48from .crosstalk
import CrosstalkTask, CrosstalkCalib
49from .fringe
import FringeTask
50from .isr
import maskNans
51from .masking
import MaskingTask
52from .overscan
import OverscanCorrectionTask
53from .straylight
import StrayLightTask
54from .vignette
import VignetteTask
55from .ampOffset
import AmpOffsetTask
56from lsst.daf.butler
import DimensionGraph
59__all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
73 registry : `lsst.daf.butler.Registry`
74 Butler registry to query.
75 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
76 Data id to transform to identify crosstalkSources. The
77 ``detector`` entry will be stripped.
78 collections : `lsst.daf.butler.CollectionSearch`
79 Collections to search through.
83 results : `list` [`lsst.daf.butler.DatasetRef`]
84 List of datasets that match the query that will be used
as
87 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"exposure"]))
88 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
95 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records))
for ref
in results]
99 dimensions={
"instrument",
"exposure",
"detector"},
100 defaultTemplates={}):
101 ccdExposure = cT.Input(
103 doc=
"Input exposure to process.",
104 storageClass=
"Exposure",
105 dimensions=[
"instrument",
"exposure",
"detector"],
107 camera = cT.PrerequisiteInput(
109 storageClass=
"Camera",
110 doc=
"Input camera to construct complete exposures.",
111 dimensions=[
"instrument"],
115 crosstalk = cT.PrerequisiteInput(
117 doc=
"Input crosstalk object",
118 storageClass=
"CrosstalkCalib",
119 dimensions=[
"instrument",
"detector"],
123 crosstalkSources = cT.PrerequisiteInput(
124 name=
"isrOverscanCorrected",
125 doc=
"Overscan corrected input images.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
130 lookupFunction=crosstalkSourceLookup,
133 bias = cT.PrerequisiteInput(
135 doc=
"Input bias calibration.",
136 storageClass=
"ExposureF",
137 dimensions=[
"instrument",
"detector"],
140 dark = cT.PrerequisiteInput(
142 doc=
"Input dark calibration.",
143 storageClass=
"ExposureF",
144 dimensions=[
"instrument",
"detector"],
147 flat = cT.PrerequisiteInput(
149 doc=
"Input flat calibration.",
150 storageClass=
"ExposureF",
151 dimensions=[
"instrument",
"physical_filter",
"detector"],
154 ptc = cT.PrerequisiteInput(
156 doc=
"Input Photon Transfer Curve dataset",
157 storageClass=
"PhotonTransferCurveDataset",
158 dimensions=[
"instrument",
"detector"],
161 fringes = cT.PrerequisiteInput(
163 doc=
"Input fringe calibration.",
164 storageClass=
"ExposureF",
165 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 strayLightData = cT.PrerequisiteInput(
171 doc=
"Input stray light calibration.",
172 storageClass=
"StrayLightData",
173 dimensions=[
"instrument",
"physical_filter",
"detector"],
178 bfKernel = cT.PrerequisiteInput(
180 doc=
"Input brighter-fatter kernel.",
181 storageClass=
"NumpyArray",
182 dimensions=[
"instrument"],
186 newBFKernel = cT.PrerequisiteInput(
187 name=
'brighterFatterKernel',
188 doc=
"Newer complete kernel + gain solutions.",
189 storageClass=
"BrighterFatterKernel",
190 dimensions=[
"instrument",
"detector"],
194 defects = cT.PrerequisiteInput(
196 doc=
"Input defect tables.",
197 storageClass=
"Defects",
198 dimensions=[
"instrument",
"detector"],
201 linearizer = cT.PrerequisiteInput(
203 storageClass=
"Linearizer",
204 doc=
"Linearity correction calibration.",
205 dimensions=[
"instrument",
"detector"],
209 opticsTransmission = cT.PrerequisiteInput(
210 name=
"transmission_optics",
211 storageClass=
"TransmissionCurve",
212 doc=
"Transmission curve due to the optics.",
213 dimensions=[
"instrument"],
216 filterTransmission = cT.PrerequisiteInput(
217 name=
"transmission_filter",
218 storageClass=
"TransmissionCurve",
219 doc=
"Transmission curve due to the filter.",
220 dimensions=[
"instrument",
"physical_filter"],
223 sensorTransmission = cT.PrerequisiteInput(
224 name=
"transmission_sensor",
225 storageClass=
"TransmissionCurve",
226 doc=
"Transmission curve due to the sensor.",
227 dimensions=[
"instrument",
"detector"],
230 atmosphereTransmission = cT.PrerequisiteInput(
231 name=
"transmission_atmosphere",
232 storageClass=
"TransmissionCurve",
233 doc=
"Transmission curve due to the atmosphere.",
234 dimensions=[
"instrument"],
237 illumMaskedImage = cT.PrerequisiteInput(
239 doc=
"Input illumination correction.",
240 storageClass=
"MaskedImageF",
241 dimensions=[
"instrument",
"physical_filter",
"detector"],
245 outputExposure = cT.Output(
247 doc=
"Output ISR processed exposure.",
248 storageClass=
"Exposure",
249 dimensions=[
"instrument",
"exposure",
"detector"],
251 preInterpExposure = cT.Output(
252 name=
'preInterpISRCCD',
253 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
254 storageClass=
"ExposureF",
255 dimensions=[
"instrument",
"exposure",
"detector"],
257 outputOssThumbnail = cT.Output(
259 doc=
"Output Overscan-subtracted thumbnail image.",
260 storageClass=
"Thumbnail",
261 dimensions=[
"instrument",
"exposure",
"detector"],
263 outputFlattenedThumbnail = cT.Output(
264 name=
"FlattenedThumb",
265 doc=
"Output flat-corrected thumbnail image.",
266 storageClass=
"Thumbnail",
267 dimensions=[
"instrument",
"exposure",
"detector"],
273 if config.doBias
is not True:
274 self.prerequisiteInputs.discard(
"bias")
275 if config.doLinearize
is not True:
276 self.prerequisiteInputs.discard(
"linearizer")
277 if config.doCrosstalk
is not True:
278 self.prerequisiteInputs.discard(
"crosstalkSources")
279 self.prerequisiteInputs.discard(
"crosstalk")
280 if config.doBrighterFatter
is not True:
281 self.prerequisiteInputs.discard(
"bfKernel")
282 self.prerequisiteInputs.discard(
"newBFKernel")
283 if config.doDefect
is not True:
284 self.prerequisiteInputs.discard(
"defects")
285 if config.doDark
is not True:
286 self.prerequisiteInputs.discard(
"dark")
287 if config.doFlat
is not True:
288 self.prerequisiteInputs.discard(
"flat")
289 if config.doFringe
is not True:
290 self.prerequisiteInputs.discard(
"fringe")
291 if config.doStrayLight
is not True:
292 self.prerequisiteInputs.discard(
"strayLightData")
293 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
294 self.prerequisiteInputs.discard(
"ptc")
295 if config.doAttachTransmissionCurve
is not True:
296 self.prerequisiteInputs.discard(
"opticsTransmission")
297 self.prerequisiteInputs.discard(
"filterTransmission")
298 self.prerequisiteInputs.discard(
"sensorTransmission")
299 self.prerequisiteInputs.discard(
"atmosphereTransmission")
300 if config.doUseOpticsTransmission
is not True:
301 self.prerequisiteInputs.discard(
"opticsTransmission")
302 if config.doUseFilterTransmission
is not True:
303 self.prerequisiteInputs.discard(
"filterTransmission")
304 if config.doUseSensorTransmission
is not True:
305 self.prerequisiteInputs.discard(
"sensorTransmission")
306 if config.doUseAtmosphereTransmission
is not True:
307 self.prerequisiteInputs.discard(
"atmosphereTransmission")
308 if config.doIlluminationCorrection
is not True:
309 self.prerequisiteInputs.discard(
"illumMaskedImage")
311 if config.doWrite
is not True:
312 self.outputs.discard(
"outputExposure")
313 self.outputs.discard(
"preInterpExposure")
314 self.outputs.discard(
"outputFlattenedThumbnail")
315 self.outputs.discard(
"outputOssThumbnail")
316 if config.doSaveInterpPixels
is not True:
317 self.outputs.discard(
"preInterpExposure")
318 if config.qa.doThumbnailOss
is not True:
319 self.outputs.discard(
"outputOssThumbnail")
320 if config.qa.doThumbnailFlattened
is not True:
321 self.outputs.discard(
"outputFlattenedThumbnail")
325 pipelineConnections=IsrTaskConnections):
326 """Configuration parameters for IsrTask.
328 Items are grouped in the order
in which they are executed by the task.
330 datasetType = pexConfig.Field(
332 doc="Dataset type for input data; users will typically leave this alone, "
333 "but camera-specific ISR tasks will override it",
337 fallbackFilterName = pexConfig.Field(
339 doc=
"Fallback default filter name for calibrations.",
342 useFallbackDate = pexConfig.Field(
344 doc=
"Pass observation date when using fallback filter.",
347 expectWcs = pexConfig.Field(
350 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
352 fwhm = pexConfig.Field(
354 doc=
"FWHM of PSF in arcseconds.",
357 qa = pexConfig.ConfigField(
359 doc=
"QA related configuration options.",
363 doConvertIntToFloat = pexConfig.Field(
365 doc=
"Convert integer raw images to floating point values?",
370 doSaturation = pexConfig.Field(
372 doc=
"Mask saturated pixels? NB: this is totally independent of the"
373 " interpolation option - this is ONLY setting the bits in the mask."
374 " To have them interpolated make sure doSaturationInterpolation=True",
377 saturatedMaskName = pexConfig.Field(
379 doc=
"Name of mask plane to use in saturation detection and interpolation",
382 saturation = pexConfig.Field(
384 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
385 default=float(
"NaN"),
387 growSaturationFootprintSize = pexConfig.Field(
389 doc=
"Number of pixels by which to grow the saturation footprints",
394 doSuspect = pexConfig.Field(
396 doc=
"Mask suspect pixels?",
399 suspectMaskName = pexConfig.Field(
401 doc=
"Name of mask plane to use for suspect pixels",
404 numEdgeSuspect = pexConfig.Field(
406 doc=
"Number of edge pixels to be flagged as untrustworthy.",
409 edgeMaskLevel = pexConfig.ChoiceField(
411 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414 'DETECTOR':
'Mask only the edges of the full detector.',
415 'AMP':
'Mask edges of each amplifier.',
420 doSetBadRegions = pexConfig.Field(
422 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
425 badStatistic = pexConfig.ChoiceField(
427 doc=
"How to estimate the average value for BAD regions.",
430 "MEANCLIP":
"Correct using the (clipped) mean of good data",
431 "MEDIAN":
"Correct using the median of the good data",
436 doOverscan = pexConfig.Field(
438 doc=
"Do overscan subtraction?",
441 overscan = pexConfig.ConfigurableField(
442 target=OverscanCorrectionTask,
443 doc=
"Overscan subtraction task for image segments.",
445 overscanFitType = pexConfig.ChoiceField(
447 doc=
"The method for fitting the overscan bias level.",
450 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
451 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
452 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
453 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
454 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
455 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
456 "MEAN":
"Correct using the mean of the overscan region",
457 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
458 "MEDIAN":
"Correct using the median of the overscan region",
459 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
461 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
462 " This option will no longer be used, and will be removed after v20.")
464 overscanOrder = pexConfig.Field(
466 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
467 "or number of spline knots if overscan fit type is a spline."),
469 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
470 " This option will no longer be used, and will be removed after v20.")
472 overscanNumSigmaClip = pexConfig.Field(
474 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
476 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
477 " This option will no longer be used, and will be removed after v20.")
479 overscanIsInt = pexConfig.Field(
481 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
482 " and overscan.FitType=MEDIAN_PER_ROW.",
484 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
485 " This option will no longer be used, and will be removed after v20.")
489 overscanNumLeadingColumnsToSkip = pexConfig.Field(
491 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
494 overscanNumTrailingColumnsToSkip = pexConfig.Field(
496 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
499 overscanMaxDev = pexConfig.Field(
501 doc=
"Maximum deviation from the median for overscan",
502 default=1000.0, check=
lambda x: x > 0
504 overscanBiasJump = pexConfig.Field(
506 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509 overscanBiasJumpKeyword = pexConfig.Field(
511 doc=
"Header keyword containing information about devices.",
512 default=
"NO_SUCH_KEY",
514 overscanBiasJumpDevices = pexConfig.ListField(
516 doc=
"List of devices that need piecewise overscan correction.",
519 overscanBiasJumpLocation = pexConfig.Field(
521 doc=
"Location of bias jump along y-axis.",
526 doAssembleCcd = pexConfig.Field(
529 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
531 assembleCcd = pexConfig.ConfigurableField(
532 target=AssembleCcdTask,
533 doc=
"CCD assembly task",
537 doAssembleIsrExposures = pexConfig.Field(
540 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
542 doTrimToMatchCalib = pexConfig.Field(
545 doc=
"Trim raw data to match calibration bounding boxes?"
549 doBias = pexConfig.Field(
551 doc=
"Apply bias frame correction?",
554 biasDataProductName = pexConfig.Field(
556 doc=
"Name of the bias data product",
559 doBiasBeforeOverscan = pexConfig.Field(
561 doc=
"Reverse order of overscan and bias correction.",
566 doVariance = pexConfig.Field(
568 doc=
"Calculate variance?",
571 gain = pexConfig.Field(
573 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
574 default=float(
"NaN"),
576 readNoise = pexConfig.Field(
578 doc=
"The read noise to use if no Detector is present in the Exposure",
581 doEmpiricalReadNoise = pexConfig.Field(
584 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
586 usePtcReadNoise = pexConfig.Field(
589 doc=
"Use readnoise values from the Photon Transfer Curve?"
591 maskNegativeVariance = pexConfig.Field(
594 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
595 "in the measurement of the overscan at an edge due to the data falling off faster "
596 "than the overscan model can account for it."
598 negativeVarianceMaskName = pexConfig.Field(
601 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
604 doLinearize = pexConfig.Field(
606 doc=
"Correct for nonlinearity of the detector's response?",
611 doCrosstalk = pexConfig.Field(
613 doc=
"Apply intra-CCD crosstalk correction?",
616 doCrosstalkBeforeAssemble = pexConfig.Field(
618 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
621 crosstalk = pexConfig.ConfigurableField(
622 target=CrosstalkTask,
623 doc=
"Intra-CCD crosstalk correction",
627 doDefect = pexConfig.Field(
629 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
632 doNanMasking = pexConfig.Field(
634 doc=
"Mask non-finite (NAN, inf) pixels?",
637 doWidenSaturationTrails = pexConfig.Field(
639 doc=
"Widen bleed trails based on their width?",
644 doBrighterFatter = pexConfig.Field(
647 doc=
"Apply the brighter-fatter correction?"
649 brighterFatterLevel = pexConfig.ChoiceField(
652 doc=
"The level at which to correct for brighter-fatter.",
654 "AMP":
"Every amplifier treated separately.",
655 "DETECTOR":
"One kernel per detector",
658 brighterFatterMaxIter = pexConfig.Field(
661 doc=
"Maximum number of iterations for the brighter-fatter correction"
663 brighterFatterThreshold = pexConfig.Field(
666 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
667 "absolute value of the difference between the current corrected image and the one "
668 "from the previous iteration summed over all the pixels."
670 brighterFatterApplyGain = pexConfig.Field(
673 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
675 brighterFatterMaskListToInterpolate = pexConfig.ListField(
677 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
679 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
681 brighterFatterMaskGrowSize = pexConfig.Field(
684 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
685 "when brighter-fatter correction is applied."
689 doDark = pexConfig.Field(
691 doc=
"Apply dark frame correction?",
694 darkDataProductName = pexConfig.Field(
696 doc=
"Name of the dark data product",
701 doStrayLight = pexConfig.Field(
703 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
706 strayLight = pexConfig.ConfigurableField(
707 target=StrayLightTask,
708 doc=
"y-band stray light correction"
712 doFlat = pexConfig.Field(
714 doc=
"Apply flat field correction?",
717 flatDataProductName = pexConfig.Field(
719 doc=
"Name of the flat data product",
722 flatScalingType = pexConfig.ChoiceField(
724 doc=
"The method for scaling the flat on the fly.",
727 "USER":
"Scale by flatUserScale",
728 "MEAN":
"Scale by the inverse of the mean",
729 "MEDIAN":
"Scale by the inverse of the median",
732 flatUserScale = pexConfig.Field(
734 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
737 doTweakFlat = pexConfig.Field(
739 doc=
"Tweak flats to match observed amplifier ratios?",
745 doApplyGains = pexConfig.Field(
747 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
750 usePtcGains = pexConfig.Field(
752 doc=
"Use the gain values from the Photon Transfer Curve?",
755 normalizeGains = pexConfig.Field(
757 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
762 doFringe = pexConfig.Field(
764 doc=
"Apply fringe correction?",
767 fringe = pexConfig.ConfigurableField(
769 doc=
"Fringe subtraction task",
771 fringeAfterFlat = pexConfig.Field(
773 doc=
"Do fringe subtraction after flat-fielding?",
778 doAmpOffset = pexConfig.Field(
779 doc=
"Calculate and apply amp offset corrections?",
783 ampOffset = pexConfig.ConfigurableField(
784 doc=
"Amp offset correction task.",
785 target=AmpOffsetTask,
789 doMeasureBackground = pexConfig.Field(
791 doc=
"Measure the background level on the reduced image?",
796 doCameraSpecificMasking = pexConfig.Field(
798 doc=
"Mask camera-specific bad regions?",
801 masking = pexConfig.ConfigurableField(
807 doInterpolate = pexConfig.Field(
809 doc=
"Interpolate masked pixels?",
812 doSaturationInterpolation = pexConfig.Field(
814 doc=
"Perform interpolation over pixels masked as saturated?"
815 " NB: This is independent of doSaturation; if that is False this plane"
816 " will likely be blank, resulting in a no-op here.",
819 doNanInterpolation = pexConfig.Field(
821 doc=
"Perform interpolation over pixels masked as NaN?"
822 " NB: This is independent of doNanMasking; if that is False this plane"
823 " will likely be blank, resulting in a no-op here.",
826 doNanInterpAfterFlat = pexConfig.Field(
828 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
829 "also have to interpolate them before flat-fielding."),
832 maskListToInterpolate = pexConfig.ListField(
834 doc=
"List of mask planes that should be interpolated.",
835 default=[
'SAT',
'BAD'],
837 doSaveInterpPixels = pexConfig.Field(
839 doc=
"Save a copy of the pre-interpolated pixel values?",
844 fluxMag0T1 = pexConfig.DictField(
847 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
848 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
851 defaultFluxMag0T1 = pexConfig.Field(
853 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
854 default=pow(10.0, 0.4*28.0)
858 doVignette = pexConfig.Field(
860 doc=(
"Compute and attach the validPolygon defining the unvignetted region to the exposure "
861 "according to vignetting parameters?"),
864 doMaskVignettePolygon = pexConfig.Field(
866 doc=(
"Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
870 vignetteValue = pexConfig.Field(
872 doc=
"Value to replace image array pixels with in the vignetted region? Ignored if None.",
876 vignette = pexConfig.ConfigurableField(
878 doc=
"Vignetting task.",
882 doAttachTransmissionCurve = pexConfig.Field(
885 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
887 doUseOpticsTransmission = pexConfig.Field(
890 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
892 doUseFilterTransmission = pexConfig.Field(
895 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
897 doUseSensorTransmission = pexConfig.Field(
900 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
902 doUseAtmosphereTransmission = pexConfig.Field(
905 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
909 doIlluminationCorrection = pexConfig.Field(
912 doc=
"Perform illumination correction?"
914 illuminationCorrectionDataProductName = pexConfig.Field(
916 doc=
"Name of the illumination correction data product.",
919 illumScale = pexConfig.Field(
921 doc=
"Scale factor for the illumination correction.",
924 illumFilters = pexConfig.ListField(
927 doc=
"Only perform illumination correction for these filters."
932 doWrite = pexConfig.Field(
934 doc=
"Persist postISRCCD?",
941 raise ValueError(
"You may not specify both doFlat and doApplyGains")
943 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
952class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
953 """Apply common instrument signature correction algorithms to a raw frame.
955 The process for correcting imaging data
is very similar
from
956 camera to camera. This task provides a vanilla implementation of
957 doing these corrections, including the ability to turn certain
958 corrections off
if they are
not needed. The inputs to the primary
959 method, `
run()`, are a raw exposure to be corrected
and the
960 calibration data products. The raw input
is a single chip sized
961 mosaic of all amps including overscans
and other non-science
962 pixels. The method `
runDataRef()` identifies
and defines the
963 calibration data products,
and is intended
for use by a
964 `lsst.pipe.base.cmdLineTask.CmdLineTask`
and takes
as input only a
965 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
966 subclassed
for different camera, although the most camera specific
967 methods have been split into subtasks that can be redirected
970 The __init__ method sets up the subtasks
for ISR processing, using
976 Positional arguments passed to the Task constructor.
977 None used at this time.
978 kwargs : `dict`, optional
979 Keyword arguments passed on to the Task constructor.
980 None used at this time.
982 ConfigClass = IsrTaskConfig
987 self.makeSubtask(
"assembleCcd")
988 self.makeSubtask(
"crosstalk")
989 self.makeSubtask(
"strayLight")
990 self.makeSubtask(
"fringe")
991 self.makeSubtask(
"masking")
992 self.makeSubtask(
"overscan")
993 self.makeSubtask(
"vignette")
994 self.makeSubtask(
"ampOffset")
997 inputs = butlerQC.get(inputRefs)
1000 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
1001 except Exception
as e:
1002 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
1005 inputs[
'isGen3'] =
True
1007 detector = inputs[
'ccdExposure'].getDetector()
1009 if self.config.doCrosstalk
is True:
1012 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
1013 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
1014 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
1016 coeffVector = (self.config.crosstalk.crosstalkValues
1017 if self.config.crosstalk.useConfigCoefficients
else None)
1018 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1019 inputs[
'crosstalk'] = crosstalkCalib
1020 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
1021 if 'crosstalkSources' not in inputs:
1022 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
1025 if 'linearizer' in inputs:
1026 if isinstance(inputs[
'linearizer'], dict):
1028 linearizer.fromYaml(inputs[
'linearizer'])
1029 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1030 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1034 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1036 linearizer = inputs[
'linearizer']
1037 linearizer.log = self.log
1038 inputs[
'linearizer'] = linearizer
1041 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1043 if self.config.doDefect
is True:
1044 if "defects" in inputs
and inputs[
'defects']
is not None:
1048 if not isinstance(inputs[
"defects"], Defects):
1049 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1053 if self.config.doBrighterFatter:
1054 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1055 if brighterFatterKernel
is None:
1056 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1058 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1060 detName = detector.getName()
1061 level = brighterFatterKernel.level
1064 inputs[
'bfGains'] = brighterFatterKernel.gain
1065 if self.config.brighterFatterLevel ==
'DETECTOR':
1066 if level ==
'DETECTOR':
1067 if detName
in brighterFatterKernel.detKernels:
1068 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1070 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1071 elif level ==
'AMP':
1072 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1074 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1075 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1076 elif self.config.brighterFatterLevel ==
'AMP':
1077 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1079 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1080 expId = inputs[
'ccdExposure'].info.id
1081 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1083 assembler=self.assembleCcd
1084 if self.config.doAssembleIsrExposures
else None)
1086 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1088 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1089 if 'strayLightData' not in inputs:
1090 inputs[
'strayLightData'] =
None
1092 outputs = self.
runrun(**inputs)
1093 butlerQC.put(outputs, outputRefs)
1096 """Retrieve necessary frames for instrument signature removal.
1098 Pre-fetching all required ISR data products limits the IO
1099 required by the ISR. Any conflict between the calibration data
1100 available and that needed
for ISR
is also detected prior to
1101 doing processing, allowing it to fail quickly.
1105 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1106 Butler reference of the detector data to be processed
1108 The raw exposure that will later be corrected
with the
1109 retrieved calibration data; should
not be modified
in this
1114 result : `lsst.pipe.base.Struct`
1115 Result struct
with components (which may be `
None`):
1117 - ``linearizer``: functor
for linearization
1119 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1122 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1124 - ``fringes``: `lsst.pipe.base.Struct`
with components:
1126 - ``seed``: random seed derived
from the ccdExposureId
for random
1127 number generator (`uint32`).
1129 A ``TransmissionCurve`` that represents the throughput of the
1130 optics, to be evaluated
in focal-plane coordinates.
1132 A ``TransmissionCurve`` that represents the throughput of the
1133 filter itself, to be evaluated
in focal-plane coordinates.
1135 A ``TransmissionCurve`` that represents the throughput of the
1136 sensor itself, to be evaluated
in post-assembly trimmed
1137 detector coordinates.
1139 A ``TransmissionCurve`` that represents the throughput of the
1140 atmosphere, assumed to be spatially constant.
1141 - ``strayLightData`` : `object`
1142 An opaque object containing calibration information
for
1143 stray-light correction. If `
None`, no correction will be
1145 - ``illumMaskedImage`` : illumination correction image
1150 NotImplementedError :
1151 Raised
if a per-amplifier brighter-fatter kernel
is requested by
1155 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1156 dateObs = dateObs.toPython().isoformat()
1157 except RuntimeError:
1158 self.log.warning(
"Unable to identify dateObs for rawExposure.")
1161 ccd = rawExposure.getDetector()
1162 filterLabel = rawExposure.getFilterLabel()
1163 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1164 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1165 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1166 if self.config.doBias
else None)
1169 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1171 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1172 linearizer.log = self.log
1173 if isinstance(linearizer, numpy.ndarray):
1176 crosstalkCalib =
None
1177 if self.config.doCrosstalk:
1179 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1181 coeffVector = (self.config.crosstalk.crosstalkValues
1182 if self.config.crosstalk.useConfigCoefficients
else None)
1183 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1184 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1185 if self.config.doCrosstalk
else None)
1187 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1188 if self.config.doDark
else None)
1189 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1191 if self.config.doFlat
else None)
1193 brighterFatterKernel =
None
1194 brighterFatterGains =
None
1195 if self.config.doBrighterFatter
is True:
1200 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1201 brighterFatterGains = brighterFatterKernel.gain
1202 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1205 brighterFatterKernel = dataRef.get(
"bfKernel")
1206 self.log.info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1208 brighterFatterKernel =
None
1209 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1212 if self.config.brighterFatterLevel ==
'DETECTOR':
1213 if brighterFatterKernel.detKernels:
1214 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1216 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1219 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1221 defectList = (dataRef.get(
"defects")
1222 if self.config.doDefect
else None)
1223 expId = rawExposure.info.id
1224 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1225 if self.config.doAssembleIsrExposures
else None)
1226 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1227 else pipeBase.Struct(fringes=
None))
1229 if self.config.doAttachTransmissionCurve:
1230 opticsTransmission = (dataRef.get(
"transmission_optics")
1231 if self.config.doUseOpticsTransmission
else None)
1232 filterTransmission = (dataRef.get(
"transmission_filter")
1233 if self.config.doUseFilterTransmission
else None)
1234 sensorTransmission = (dataRef.get(
"transmission_sensor")
1235 if self.config.doUseSensorTransmission
else None)
1236 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1237 if self.config.doUseAtmosphereTransmission
else None)
1239 opticsTransmission =
None
1240 filterTransmission =
None
1241 sensorTransmission =
None
1242 atmosphereTransmission =
None
1244 if self.config.doStrayLight:
1245 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1247 strayLightData =
None
1250 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1251 if (self.config.doIlluminationCorrection
1252 and physicalFilter
in self.config.illumFilters)
1256 return pipeBase.Struct(bias=biasExposure,
1257 linearizer=linearizer,
1258 crosstalk=crosstalkCalib,
1259 crosstalkSources=crosstalkSources,
1262 bfKernel=brighterFatterKernel,
1263 bfGains=brighterFatterGains,
1265 fringes=fringeStruct,
1266 opticsTransmission=opticsTransmission,
1267 filterTransmission=filterTransmission,
1268 sensorTransmission=sensorTransmission,
1269 atmosphereTransmission=atmosphereTransmission,
1270 strayLightData=strayLightData,
1271 illumMaskedImage=illumMaskedImage
1275 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1276 crosstalk=None, crosstalkSources=None,
1277 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1278 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1279 sensorTransmission=
None, atmosphereTransmission=
None,
1280 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1283 """Perform instrument signature removal on an exposure.
1285 Steps included in the ISR processing,
in order performed, are:
1286 - saturation
and suspect pixel masking
1287 - overscan subtraction
1288 - CCD assembly of individual amplifiers
1290 - variance image construction
1291 - linearization of non-linear response
1293 - brighter-fatter correction
1296 - stray light subtraction
1298 - masking of known defects
and camera specific features
1299 - vignette calculation
1300 - appending transmission curve
and distortion model
1305 The raw exposure that
is to be run through ISR. The
1306 exposure
is modified by this method.
1308 The camera geometry
for this exposure. Required
if
1309 one
or more of ``ccdExposure``, ``bias``, ``dark``,
or
1310 ``flat`` does
not have an associated detector.
1312 Bias calibration frame.
1314 Functor
for linearization.
1316 Calibration
for crosstalk.
1317 crosstalkSources : `list`, optional
1318 List of possible crosstalk sources.
1320 Dark calibration frame.
1322 Flat calibration frame.
1324 Photon transfer curve dataset,
with, e.g., gains
1326 bfKernel : `numpy.ndarray`, optional
1327 Brighter-fatter kernel.
1328 bfGains : `dict` of `float`, optional
1329 Gains used to override the detector
's nominal gains for the
1330 brighter-fatter correction. A dict keyed by amplifier name for
1331 the detector
in question.
1334 fringes : `lsst.pipe.base.Struct`, optional
1335 Struct containing the fringe correction data,
with
1338 - ``seed``: random seed derived
from the ccdExposureId
for random
1339 number generator (`uint32`)
1341 A ``TransmissionCurve`` that represents the throughput of the,
1342 optics, to be evaluated
in focal-plane coordinates.
1344 A ``TransmissionCurve`` that represents the throughput of the
1345 filter itself, to be evaluated
in focal-plane coordinates.
1347 A ``TransmissionCurve`` that represents the throughput of the
1348 sensor itself, to be evaluated
in post-assembly trimmed detector
1351 A ``TransmissionCurve`` that represents the throughput of the
1352 atmosphere, assumed to be spatially constant.
1353 detectorNum : `int`, optional
1354 The integer number
for the detector to process.
1355 isGen3 : bool, optional
1356 Flag this call to
run()
as using the Gen3 butler environment.
1357 strayLightData : `object`, optional
1358 Opaque object containing calibration information
for stray-light
1359 correction. If `
None`, no correction will be performed.
1361 Illumination correction image.
1365 result : `lsst.pipe.base.Struct`
1366 Result struct
with component:
1368 The fully ISR corrected exposure.
1370 An alias
for `exposure`
1371 - ``ossThumb`` : `numpy.ndarray`
1372 Thumbnail image of the exposure after overscan subtraction.
1373 - ``flattenedThumb`` : `numpy.ndarray`
1374 Thumbnail image of the exposure after flat-field correction.
1379 Raised
if a configuration option
is set to
True, but the
1380 required calibration data has
not been specified.
1384 The current processed exposure can be viewed by setting the
1385 appropriate lsstDebug entries
in the `debug.display`
1386 dictionary. The names of these entries correspond to some of
1387 the IsrTaskConfig Boolean options,
with the value denoting the
1388 frame to use. The exposure
is shown inside the matching
1389 option check
and after the processing of that step has
1390 finished. The steps
with debug points are:
1401 In addition, setting the
"postISRCCD" entry displays the
1402 exposure after all ISR processing has finished.
1411 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1412 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1413 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1414 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1416 if isinstance(ccdExposure, ButlerDataRef):
1417 return self.
runDataRefrunDataRef(ccdExposure)
1419 ccd = ccdExposure.getDetector()
1420 filterLabel = ccdExposure.getFilterLabel()
1421 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1424 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1425 ccd = [
FakeAmp(ccdExposure, self.config)]
1428 if self.config.doBias
and bias
is None:
1429 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1430 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1431 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1432 if self.config.doBrighterFatter
and bfKernel
is None:
1433 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1434 if self.config.doDark
and dark
is None:
1435 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1436 if self.config.doFlat
and flat
is None:
1437 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1438 if self.config.doDefect
and defects
is None:
1439 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1440 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1441 and fringes.fringes
is None):
1446 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1447 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1448 and illumMaskedImage
is None):
1449 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1452 if self.config.doConvertIntToFloat:
1453 self.log.info(
"Converting exposure to floating point values.")
1456 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1457 self.log.info(
"Applying bias correction.")
1458 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1459 trimToFit=self.config.doTrimToMatchCalib)
1460 self.
debugViewdebugView(ccdExposure,
"doBias")
1467 if ccdExposure.getBBox().contains(amp.getBBox()):
1470 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1472 if self.config.doOverscan
and not badAmp:
1475 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1476 if overscanResults
is not None and \
1477 self.config.qa
is not None and self.config.qa.saveStats
is True:
1478 if isinstance(overscanResults.overscanFit, float):
1479 qaMedian = overscanResults.overscanFit
1480 qaStdev = float(
"NaN")
1482 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1483 afwMath.MEDIAN | afwMath.STDEVCLIP)
1484 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1485 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1487 self.metadata[f
"FIT MEDIAN {amp.getName()}"] = qaMedian
1488 self.metadata[f
"FIT STDEV {amp.getName()}"] = qaStdev
1489 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1490 amp.getName(), qaMedian, qaStdev)
1493 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1494 afwMath.MEDIAN | afwMath.STDEVCLIP)
1495 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1496 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1498 self.metadata[f
"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter
1499 self.metadata[f
"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter
1500 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1501 amp.getName(), qaMedianAfter, qaStdevAfter)
1503 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1506 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1507 overscanResults =
None
1509 overscans.append(overscanResults
if overscanResults
is not None else None)
1511 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1513 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1514 self.log.info(
"Applying crosstalk correction.")
1515 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1516 crosstalkSources=crosstalkSources, camera=camera)
1517 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1519 if self.config.doAssembleCcd:
1520 self.log.info(
"Assembling CCD from amplifiers.")
1521 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1523 if self.config.expectWcs
and not ccdExposure.getWcs():
1524 self.log.warning(
"No WCS found in input exposure.")
1525 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1528 if self.config.qa.doThumbnailOss:
1529 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1531 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1532 self.log.info(
"Applying bias correction.")
1533 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1534 trimToFit=self.config.doTrimToMatchCalib)
1535 self.
debugViewdebugView(ccdExposure,
"doBias")
1537 if self.config.doVariance:
1538 for amp, overscanResults
in zip(ccd, overscans):
1539 if ccdExposure.getBBox().contains(amp.getBBox()):
1540 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1541 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1542 if overscanResults
is not None:
1544 overscanImage=overscanResults.overscanImage,
1550 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1551 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1552 afwMath.MEDIAN | afwMath.STDEVCLIP)
1553 self.metadata[f
"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1554 qaStats.getValue(afwMath.MEDIAN)
1555 self.metadata[f
"ISR VARIANCE {amp.getName()} STDEV"] = \
1556 qaStats.getValue(afwMath.STDEVCLIP)
1557 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1558 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1559 qaStats.getValue(afwMath.STDEVCLIP))
1560 if self.config.maskNegativeVariance:
1564 self.log.info(
"Applying linearizer.")
1565 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1566 detector=ccd, log=self.log)
1568 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1569 self.log.info(
"Applying crosstalk correction.")
1570 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1571 crosstalkSources=crosstalkSources, isTrimmed=
True)
1572 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1577 if self.config.doDefect:
1578 self.log.info(
"Masking defects.")
1579 self.
maskDefectmaskDefect(ccdExposure, defects)
1581 if self.config.numEdgeSuspect > 0:
1582 self.log.info(
"Masking edges as SUSPECT.")
1583 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1584 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1586 if self.config.doNanMasking:
1587 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1588 self.
maskNanmaskNan(ccdExposure)
1590 if self.config.doWidenSaturationTrails:
1591 self.log.info(
"Widening saturation trails.")
1592 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1594 if self.config.doCameraSpecificMasking:
1595 self.log.info(
"Masking regions for camera specific reasons.")
1596 self.masking.
run(ccdExposure)
1598 if self.config.doBrighterFatter:
1608 interpExp = ccdExposure.clone()
1609 with self.
flatContextflatContext(interpExp, flat, dark):
1610 isrFunctions.interpolateFromMask(
1611 maskedImage=interpExp.getMaskedImage(),
1612 fwhm=self.config.fwhm,
1613 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1614 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1616 bfExp = interpExp.clone()
1618 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1619 type(bfKernel), type(bfGains))
1620 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1621 self.config.brighterFatterMaxIter,
1622 self.config.brighterFatterThreshold,
1623 self.config.brighterFatterApplyGain,
1625 if bfResults[1] == self.config.brighterFatterMaxIter:
1626 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1629 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1631 image = ccdExposure.getMaskedImage().getImage()
1632 bfCorr = bfExp.getMaskedImage().getImage()
1633 bfCorr -= interpExp.getMaskedImage().getImage()
1642 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1643 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1646 if self.config.brighterFatterMaskGrowSize > 0:
1647 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1648 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1649 isrFunctions.growMasks(ccdExposure.getMask(),
1650 radius=self.config.brighterFatterMaskGrowSize,
1651 maskNameList=maskPlane,
1652 maskValue=maskPlane)
1654 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1656 if self.config.doDark:
1657 self.log.info(
"Applying dark correction.")
1659 self.
debugViewdebugView(ccdExposure,
"doDark")
1661 if self.config.doFringe
and not self.config.fringeAfterFlat:
1662 self.log.info(
"Applying fringe correction before flat.")
1663 self.fringe.
run(ccdExposure, **fringes.getDict())
1664 self.
debugViewdebugView(ccdExposure,
"doFringe")
1666 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1667 self.log.info(
"Checking strayLight correction.")
1668 self.strayLight.
run(ccdExposure, strayLightData)
1669 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1671 if self.config.doFlat:
1672 self.log.info(
"Applying flat correction.")
1674 self.
debugViewdebugView(ccdExposure,
"doFlat")
1676 if self.config.doApplyGains:
1677 self.log.info(
"Applying gain correction instead of flat.")
1678 if self.config.usePtcGains:
1679 self.log.info(
"Using gains from the Photon Transfer Curve.")
1680 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1683 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1685 if self.config.doFringe
and self.config.fringeAfterFlat:
1686 self.log.info(
"Applying fringe correction after flat.")
1687 self.fringe.
run(ccdExposure, **fringes.getDict())
1689 if self.config.doVignette:
1690 if self.config.doMaskVignettePolygon:
1691 self.log.info(
"Constructing, attaching, and masking vignette polygon.")
1693 self.log.info(
"Constructing and attaching vignette polygon.")
1695 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1696 vignetteValue=self.config.vignetteValue, log=self.log)
1698 if self.config.doAttachTransmissionCurve:
1699 self.log.info(
"Adding transmission curves.")
1700 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1701 filterTransmission=filterTransmission,
1702 sensorTransmission=sensorTransmission,
1703 atmosphereTransmission=atmosphereTransmission)
1705 flattenedThumb =
None
1706 if self.config.qa.doThumbnailFlattened:
1707 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1709 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1710 self.log.info(
"Performing illumination correction.")
1711 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1712 illumMaskedImage, illumScale=self.config.illumScale,
1713 trimToFit=self.config.doTrimToMatchCalib)
1716 if self.config.doSaveInterpPixels:
1717 preInterpExp = ccdExposure.clone()
1732 if self.config.doSetBadRegions:
1733 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1734 if badPixelCount > 0:
1735 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1737 if self.config.doInterpolate:
1738 self.log.info(
"Interpolating masked pixels.")
1739 isrFunctions.interpolateFromMask(
1740 maskedImage=ccdExposure.getMaskedImage(),
1741 fwhm=self.config.fwhm,
1742 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1743 maskNameList=list(self.config.maskListToInterpolate)
1749 if self.config.doAmpOffset:
1750 self.log.info(
"Correcting amp offsets.")
1751 self.ampOffset.
run(ccdExposure)
1753 if self.config.doMeasureBackground:
1754 self.log.info(
"Measuring background level.")
1757 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1759 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1760 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1761 afwMath.MEDIAN | afwMath.STDEVCLIP)
1762 self.metadata[f
"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1763 self.metadata[f
"ISR BACKGROUND {amp.getName()} STDEV"] = \
1764 qaStats.getValue(afwMath.STDEVCLIP)
1765 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1766 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1767 qaStats.getValue(afwMath.STDEVCLIP))
1769 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1771 return pipeBase.Struct(
1772 exposure=ccdExposure,
1774 flattenedThumb=flattenedThumb,
1776 preInterpExposure=preInterpExp,
1777 outputExposure=ccdExposure,
1778 outputOssThumbnail=ossThumb,
1779 outputFlattenedThumbnail=flattenedThumb,
1784 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1786 This method contains the `CmdLineTask` interface to the ISR
1787 processing. All IO is handled here, freeing the `
run()` method
1788 to manage only pixel-level calculations. The steps performed
1790 - Read
in necessary detrending/isr/calibration data.
1791 - Process raw exposure
in `
run()`.
1792 - Persist the ISR-corrected exposure
as "postISRCCD" if
1793 config.doWrite=
True.
1797 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1798 DataRef of the detector data to be processed
1802 result : `lsst.pipe.base.Struct`
1803 Result struct
with component:
1805 The fully ISR corrected exposure.
1810 Raised
if a configuration option
is set to
True, but the
1811 required calibration data does
not exist.
1814 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1816 ccdExposure = sensorRef.get(self.config.datasetType)
1818 camera = sensorRef.get(
"camera")
1819 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1821 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1823 if self.config.doWrite:
1824 sensorRef.put(result.exposure,
"postISRCCD")
1825 if result.preInterpExposure
is not None:
1826 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1827 if result.ossThumb
is not None:
1828 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1829 if result.flattenedThumb
is not None:
1830 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1835 """Retrieve a calibration dataset for removing instrument signature.
1840 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1841 DataRef of the detector data to find calibration datasets
1844 Type of dataset to retrieve (e.g.
'bias',
'flat', etc).
1845 dateObs : `str`, optional
1846 Date of the observation. Used to correct butler failures
1847 when using fallback filters.
1849 If
True, disable butler proxies to enable error handling
1850 within this routine.
1855 Requested calibration frame.
1860 Raised
if no matching calibration frame can be found.
1863 exp = dataRef.get(datasetType, immediate=immediate)
1864 except Exception
as exc1:
1865 if not self.config.fallbackFilterName:
1866 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1868 if self.config.useFallbackDate
and dateObs:
1869 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1870 dateObs=dateObs, immediate=immediate)
1872 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1873 except Exception
as exc2:
1874 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1875 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1876 self.log.warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1878 if self.config.doAssembleIsrExposures:
1879 exp = self.assembleCcd.assembleCcd(exp)
1883 """Ensure that the data returned by Butler is a fully constructed exp.
1885 ISR requires exposure-level image data for historical reasons, so
if we
1886 did
not recieve that
from Butler, construct it
from what we have,
1887 modifying the input
in place.
1892 or `lsst.afw.image.ImageF`
1893 The input data structure obtained
from Butler.
1894 camera : `lsst.afw.cameraGeom.camera`, optional
1895 The camera associated
with the image. Used to find the appropriate
1896 detector
if detector
is not already set.
1897 detectorNum : `int`, optional
1898 The detector
in the camera to attach,
if the detector
is not
1904 The re-constructed exposure,
with appropriate detector parameters.
1909 Raised
if the input data cannot be used to construct an exposure.
1911 if isinstance(inputExp, afwImage.DecoratedImageU):
1912 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1913 elif isinstance(inputExp, afwImage.ImageF):
1914 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1915 elif isinstance(inputExp, afwImage.MaskedImageF):
1916 inputExp = afwImage.makeExposure(inputExp)
1917 elif isinstance(inputExp, afwImage.Exposure):
1919 elif inputExp
is None:
1923 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1926 if inputExp.getDetector()
is None:
1927 if camera
is None or detectorNum
is None:
1928 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1929 'without a detector set.')
1930 inputExp.setDetector(camera[detectorNum])
1935 """Convert exposure image from uint16 to float.
1937 If the exposure does not need to be converted, the input
is
1938 immediately returned. For exposures that are converted to use
1939 floating point pixels, the variance
is set to unity
and the
1945 The raw exposure to be converted.
1950 The input ``exposure``, converted to floating point pixels.
1955 Raised
if the exposure type cannot be converted to float.
1958 if isinstance(exposure, afwImage.ExposureF):
1960 self.log.debug(
"Exposure already of type float.")
1962 if not hasattr(exposure,
"convertF"):
1963 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1965 newexposure = exposure.convertF()
1966 newexposure.variance[:] = 1
1967 newexposure.mask[:] = 0x0
1972 """Identify bad amplifiers, saturated and suspect pixels.
1977 Input exposure to be masked.
1979 Catalog of parameters defining the amplifier on this
1982 List of defects. Used to determine if the entire
1988 If this
is true, the entire amplifier area
is covered by
1989 defects
and unusable.
1992 maskedImage = ccdExposure.getMaskedImage()
1999 if defects
is not None:
2000 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
2006 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
2008 maskView = dataView.getMask()
2009 maskView |= maskView.getPlaneBitMask(
"BAD")
2017 if self.config.doSaturation
and not badAmp:
2018 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2019 if self.config.doSuspect
and not badAmp:
2020 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2021 if math.isfinite(self.config.saturation):
2022 limits.update({self.config.saturatedMaskName: self.config.saturation})
2024 for maskName, maskThreshold
in limits.items():
2025 if not math.isnan(maskThreshold):
2026 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2027 isrFunctions.makeThresholdMask(
2028 maskedImage=dataView,
2029 threshold=maskThreshold,
2036 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2038 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2039 self.config.suspectMaskName])
2040 if numpy.all(maskView.getArray() & maskVal > 0):
2042 maskView |= maskView.getPlaneBitMask(
"BAD")
2047 """Apply overscan correction in place.
2049 This method does initial pixel rejection of the overscan
2050 region. The overscan can also be optionally segmented to
2051 allow for discontinuous overscan responses to be fit
2052 separately. The actual overscan subtraction
is performed by
2053 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2054 which
is called here after the amplifier
is preprocessed.
2059 Exposure to have overscan correction performed.
2060 amp : `lsst.afw.cameraGeom.Amplifer`
2061 The amplifier to consider
while correcting the overscan.
2065 overscanResults : `lsst.pipe.base.Struct`
2066 Result struct
with components:
2068 Value
or fit subtracted
from the amplifier image data.
2070 Value
or fit subtracted
from the overscan image data.
2072 Image of the overscan region
with the overscan
2073 correction applied. This quantity
is used to estimate
2074 the amplifier read noise empirically.
2079 Raised
if the ``amp`` does
not contain raw pixel information.
2083 lsst.ip.isr.isrFunctions.overscanCorrection
2085 if amp.getRawHorizontalOverscanBBox().isEmpty():
2086 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2089 statControl = afwMath.StatisticsControl()
2090 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2093 dataBBox = amp.getRawDataBBox()
2094 oscanBBox = amp.getRawHorizontalOverscanBBox()
2098 prescanBBox = amp.getRawPrescanBBox()
2099 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2100 dx0 += self.config.overscanNumLeadingColumnsToSkip
2101 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2103 dx0 += self.config.overscanNumTrailingColumnsToSkip
2104 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2111 if ((self.config.overscanBiasJump
2112 and self.config.overscanBiasJumpLocation)
2113 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2114 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2115 self.config.overscanBiasJumpDevices)):
2116 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2117 yLower = self.config.overscanBiasJumpLocation
2118 yUpper = dataBBox.getHeight() - yLower
2120 yUpper = self.config.overscanBiasJumpLocation
2121 yLower = dataBBox.getHeight() - yUpper
2139 oscanBBox.getHeight())))
2143 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2144 ampImage = ccdExposure.maskedImage[imageBBox]
2145 overscanImage = ccdExposure.maskedImage[overscanBBox]
2147 overscanArray = overscanImage.image.array
2148 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2149 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2150 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2152 statControl = afwMath.StatisticsControl()
2153 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2155 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2158 levelStat = afwMath.MEDIAN
2159 sigmaStat = afwMath.STDEVCLIP
2161 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2162 self.config.qa.flatness.nIter)
2163 metadata = ccdExposure.getMetadata()
2164 ampNum = amp.getName()
2166 if isinstance(overscanResults.overscanFit, float):
2167 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit
2168 metadata[f
"ISR_OSCAN_SIGMA{ampNum}"] = 0.0
2170 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2171 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat)
2172 metadata[f
"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat)
2174 return overscanResults
2177 """Set the variance plane using the gain and read noise
2179 The read noise is calculated
from the ``overscanImage``
if the
2180 ``doEmpiricalReadNoise`` option
is set
in the configuration; otherwise
2181 the value
from the amplifier data
is used.
2186 Exposure to process.
2187 amp : `lsst.afw.table.AmpInfoRecord`
or `FakeAmp`
2188 Amplifier detector data.
2190 Image of overscan, required only
for empirical read noise.
2192 PTC dataset containing the gains
and read noise.
2198 Raised
if either ``usePtcGains`` of ``usePtcReadNoise``
2199 are ``
True``, but ptcDataset
is not provided.
2201 Raised
if ```doEmpiricalReadNoise``
is ``
True`` but
2202 ``overscanImage``
is ``
None``.
2206 lsst.ip.isr.isrFunctions.updateVariance
2208 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2209 if self.config.usePtcGains:
2210 if ptcDataset
is None:
2211 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2213 gain = ptcDataset.gain[amp.getName()]
2214 self.log.info(
"Using gain from Photon Transfer Curve.")
2216 gain = amp.getGain()
2218 if math.isnan(gain):
2220 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2223 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2224 amp.getName(), gain, patchedGain)
2227 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2228 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2230 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2231 stats = afwMath.StatisticsControl()
2232 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2233 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2234 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2235 amp.getName(), readNoise)
2236 elif self.config.usePtcReadNoise:
2237 if ptcDataset
is None:
2238 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2240 readNoise = ptcDataset.noise[amp.getName()]
2241 self.log.info(
"Using read noise from Photon Transfer Curve.")
2243 readNoise = amp.getReadNoise()
2245 isrFunctions.updateVariance(
2246 maskedImage=ampExposure.getMaskedImage(),
2248 readNoise=readNoise,
2252 """Identify and mask pixels with negative variance values.
2257 Exposure to process.
2261 lsst.ip.isr.isrFunctions.updateVariance
2263 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2264 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2265 exposure.mask.array[bad] |= maskPlane
2268 """Apply dark correction in place.
2273 Exposure to process.
2275 Dark exposure of the same size as ``exposure``.
2276 invert : `Bool`, optional
2277 If
True, re-add the dark to an already corrected image.
2282 Raised
if either ``exposure``
or ``darkExposure`` do
not
2283 have their dark time defined.
2287 lsst.ip.isr.isrFunctions.darkCorrection
2289 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2290 if math.isnan(expScale):
2291 raise RuntimeError(
"Exposure darktime is NAN.")
2292 if darkExposure.getInfo().getVisitInfo()
is not None \
2293 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2294 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2298 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2301 isrFunctions.darkCorrection(
2302 maskedImage=exposure.getMaskedImage(),
2303 darkMaskedImage=darkExposure.getMaskedImage(),
2305 darkScale=darkScale,
2307 trimToFit=self.config.doTrimToMatchCalib
2311 """Check if linearization is needed for the detector cameraGeom.
2313 Checks config.doLinearize and the linearity type of the first
2319 Detector to get linearity type
from.
2323 doLinearize : `Bool`
2324 If
True, linearization should be performed.
2326 return self.config.doLinearize
and \
2327 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2330 """Apply flat correction in place.
2335 Exposure to process.
2337 Flat exposure of the same size as ``exposure``.
2338 invert : `Bool`, optional
2339 If
True, unflatten an already flattened image.
2343 lsst.ip.isr.isrFunctions.flatCorrection
2345 isrFunctions.flatCorrection(
2346 maskedImage=exposure.getMaskedImage(),
2347 flatMaskedImage=flatExposure.getMaskedImage(),
2348 scalingType=self.config.flatScalingType,
2349 userScale=self.config.flatUserScale,
2351 trimToFit=self.config.doTrimToMatchCalib
2355 """Detect and mask saturated pixels in config.saturatedMaskName.
2360 Exposure to process. Only the amplifier DataSec is processed.
2362 Amplifier detector data.
2366 lsst.ip.isr.isrFunctions.makeThresholdMask
2368 if not math.isnan(amp.getSaturation()):
2369 maskedImage = exposure.getMaskedImage()
2370 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2371 isrFunctions.makeThresholdMask(
2372 maskedImage=dataView,
2373 threshold=amp.getSaturation(),
2375 maskName=self.config.saturatedMaskName,
2379 """Interpolate over saturated pixels, in place.
2381 This method should be called after `saturationDetection`, to
2382 ensure that the saturated pixels have been identified in the
2383 SAT mask. It should also be called after `assembleCcd`, since
2384 saturated regions may cross amplifier boundaries.
2389 Exposure to process.
2393 lsst.ip.isr.isrTask.saturationDetection
2394 lsst.ip.isr.isrFunctions.interpolateFromMask
2396 isrFunctions.interpolateFromMask(
2397 maskedImage=exposure.getMaskedImage(),
2398 fwhm=self.config.fwhm,
2399 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2400 maskNameList=list(self.config.saturatedMaskName),
2404 """Detect and mask suspect pixels in config.suspectMaskName.
2409 Exposure to process. Only the amplifier DataSec is processed.
2411 Amplifier detector data.
2415 lsst.ip.isr.isrFunctions.makeThresholdMask
2419 Suspect pixels are pixels whose value
is greater than
2420 amp.getSuspectLevel(). This
is intended to indicate pixels that may be
2421 affected by unknown systematics;
for example
if non-linearity
2422 corrections above a certain level are unstable then that would be a
2423 useful value
for suspectLevel. A value of `nan` indicates that no such
2424 level exists
and no pixels are to be masked
as suspicious.
2426 suspectLevel = amp.getSuspectLevel()
2427 if math.isnan(suspectLevel):
2430 maskedImage = exposure.getMaskedImage()
2431 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2432 isrFunctions.makeThresholdMask(
2433 maskedImage=dataView,
2434 threshold=suspectLevel,
2436 maskName=self.config.suspectMaskName,
2440 """Mask defects using mask plane "BAD", in place.
2445 Exposure to process.
2448 List of defects to mask.
2452 Call this after CCD assembly, since defects may cross amplifier
2455 maskedImage = exposure.getMaskedImage()
2456 if not isinstance(defectBaseList, Defects):
2458 defectList =
Defects(defectBaseList)
2460 defectList = defectBaseList
2461 defectList.maskPixels(maskedImage, maskName=
"BAD")
2463 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2464 """Mask edge pixels with applicable mask plane.
2469 Exposure to process.
2470 numEdgePixels : `int`, optional
2471 Number of edge pixels to mask.
2472 maskPlane : `str`, optional
2473 Mask plane name to use.
2474 level : `str`, optional
2475 Level at which to mask edges.
2477 maskedImage = exposure.getMaskedImage()
2478 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2480 if numEdgePixels > 0:
2481 if level ==
'DETECTOR':
2482 boxes = [maskedImage.getBBox()]
2483 elif level ==
'AMP':
2484 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2489 subImage = maskedImage[box]
2490 box.grow(-numEdgePixels)
2492 SourceDetectionTask.setEdgeBits(
2498 """Mask and interpolate defects using mask plane "BAD", in place.
2503 Exposure to process.
2506 List of defects to mask
and interpolate.
2510 lsst.ip.isr.isrTask.maskDefect
2512 self.maskDefectmaskDefect(exposure, defectBaseList)
2513 self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2514 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2515 isrFunctions.interpolateFromMask(
2516 maskedImage=exposure.getMaskedImage(),
2517 fwhm=self.config.fwhm,
2518 growSaturatedFootprints=0,
2519 maskNameList=[
"BAD"],
2523 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2528 Exposure to process.
2532 We mask over all non-finite values (NaN, inf), including those
2533 that are masked with other bits (because those may
or may
not be
2534 interpolated over later,
and we want to remove all NaN/infs).
2535 Despite this behaviour, the
"UNMASKEDNAN" mask plane
is used to
2536 preserve the historical name.
2538 maskedImage = exposure.getMaskedImage()
2541 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2542 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2543 numNans =
maskNans(maskedImage, maskVal)
2544 self.metadata[
"NUMNANS"] = numNans
2546 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2549 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2555 Exposure to process.
2559 lsst.ip.isr.isrTask.maskNan
2562 isrFunctions.interpolateFromMask(
2563 maskedImage=exposure.getMaskedImage(),
2564 fwhm=self.config.fwhm,
2565 growSaturatedFootprints=0,
2566 maskNameList=["UNMASKEDNAN"],
2570 """Measure the image background in subgrids, for quality control.
2575 Exposure to process.
2577 Configuration object containing parameters on which background
2578 statistics and subgrids to use.
2580 if IsrQaConfig
is not None:
2581 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2582 IsrQaConfig.flatness.nIter)
2583 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2584 statsControl.setAndMask(maskVal)
2585 maskedImage = exposure.getMaskedImage()
2586 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2587 skyLevel = stats.getValue(afwMath.MEDIAN)
2588 skySigma = stats.getValue(afwMath.STDEVCLIP)
2589 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2590 metadata = exposure.getMetadata()
2591 metadata[
"SKYLEVEL"] = skyLevel
2592 metadata[
"SKYSIGMA"] = skySigma
2595 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2596 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2597 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2598 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2599 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2600 skyLevels = numpy.zeros((nX, nY))
2603 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2605 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2607 xLLC = xc - meshXHalf
2608 yLLC = yc - meshYHalf
2609 xURC = xc + meshXHalf - 1
2610 yURC = yc + meshYHalf - 1
2613 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2615 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2617 good = numpy.where(numpy.isfinite(skyLevels))
2618 skyMedian = numpy.median(skyLevels[good])
2619 flatness = (skyLevels[good] - skyMedian) / skyMedian
2620 flatness_rms = numpy.std(flatness)
2621 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2623 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2624 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2625 nX, nY, flatness_pp, flatness_rms)
2627 metadata[
"FLATNESS_PP"] = float(flatness_pp)
2628 metadata[
"FLATNESS_RMS"] = float(flatness_rms)
2629 metadata[
"FLATNESS_NGRIDS"] =
'%dx%d' % (nX, nY)
2630 metadata[
"FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2631 metadata[
"FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2634 """Set an approximate magnitude zero point for the exposure.
2639 Exposure to process.
2641 filterLabel = exposure.getFilterLabel()
2642 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2644 if physicalFilter
in self.config.fluxMag0T1:
2645 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2647 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2648 fluxMag0 = self.config.defaultFluxMag0T1
2650 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2652 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2655 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2656 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2657 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2661 """Context manager that applies and removes flats and darks,
2662 if the task
is configured to apply them.
2667 Exposure to process.
2669 Flat exposure the same size
as ``exp``.
2671 Dark exposure the same size
as ``exp``.
2676 The flat
and dark corrected exposure.
2678 if self.config.doDark
and dark
is not None:
2680 if self.config.doFlat:
2685 if self.config.doFlat:
2687 if self.config.doDark
and dark
is not None:
2691 """Utility function to examine ISR exposure at different stages.
2698 State of processing to view.
2700 frame = getDebugFrame(self._display, stepname)
2702 display = getDisplay(frame)
2703 display.scale(
'asinh',
'zscale')
2704 display.mtv(exposure)
2705 prompt =
"Press Enter to continue [c]... "
2707 ans = input(prompt).lower()
2708 if ans
in (
"",
"c",):
2713 """A Detector-like object that supports returning gain and saturation level
2715 This is used when the input exposure does
not have a detector.
2720 Exposure to generate a fake amplifier
for.
2721 config : `lsst.ip.isr.isrTaskConfig`
2722 Configuration to apply to the fake amplifier.
2726 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2728 self.
_gain_gain = config.gain
2733 return self.
_bbox_bbox
2736 return self.
_bbox_bbox
2742 return self.
_gain_gain
2755 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2759 """Task to wrap the default IsrTask to allow it to be retargeted.
2761 The standard IsrTask can be called directly from a command line
2762 program, but doing so removes the ability of the task to be
2763 retargeted. As most cameras override some set of the IsrTask
2764 methods, this would remove those data-specific methods
in the
2765 output post-ISR images. This wrapping
class fixes the issue,
2766 allowing identical post-ISR images to be generated by both the
2767 processCcd
and isrTask code.
2769 ConfigClass = RunIsrConfig
2770 _DefaultName = "runIsr"
2774 self.makeSubtask(
"isr")
2780 dataRef : `lsst.daf.persistence.ButlerDataRef`
2781 data reference of the detector data to be processed
2785 result : `pipeBase.Struct`
2786 Result struct with component:
2789 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def readIsrData(self, dataRef, rawExposure)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.