29import lsst.pipe.base
as pipeBase
30import lsst.pipe.base.connectionTypes
as cT
32from contextlib
import contextmanager
33from lsstDebug
import getDebugFrame
38from lsst.daf.persistence.butler
import NoResults
40from lsst.utils.timer
import timeMethod
42from .
import isrFunctions
44from .
import linearize
45from .defects
import Defects
47from .assembleCcdTask
import AssembleCcdTask
48from .crosstalk
import CrosstalkTask, CrosstalkCalib
49from .fringe
import FringeTask
50from .isr
import maskNans
51from .masking
import MaskingTask
52from .overscan
import OverscanCorrectionTask
53from .straylight
import StrayLightTask
54from .vignette
import VignetteTask
55from .ampOffset
import AmpOffsetTask
56from .deferredCharge
import DeferredChargeTask
57from .isrStatistics
import IsrStatisticsTask
58from lsst.daf.butler
import DimensionGraph
61__all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
65 """Lookup function to identify crosstalkSource entries.
67 This should return an empty list under most circumstances. Only
68 when inter-chip crosstalk has been identified should this be
75 registry : `lsst.daf.butler.Registry`
76 Butler registry to query.
77 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
78 Data id to transform to identify crosstalkSources. The
79 ``detector`` entry will be stripped.
80 collections : `lsst.daf.butler.CollectionSearch`
81 Collections to search through.
85 results : `list` [`lsst.daf.butler.DatasetRef`]
86 List of datasets that match the query that will be used
as
89 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"exposure"]))
90 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
97 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records))
for ref
in results]
101 dimensions={
"instrument",
"exposure",
"detector"},
102 defaultTemplates={}):
103 ccdExposure = cT.Input(
105 doc=
"Input exposure to process.",
106 storageClass=
"Exposure",
107 dimensions=[
"instrument",
"exposure",
"detector"],
109 camera = cT.PrerequisiteInput(
111 storageClass=
"Camera",
112 doc=
"Input camera to construct complete exposures.",
113 dimensions=[
"instrument"],
117 crosstalk = cT.PrerequisiteInput(
119 doc=
"Input crosstalk object",
120 storageClass=
"CrosstalkCalib",
121 dimensions=[
"instrument",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
135 bias = cT.PrerequisiteInput(
137 doc=
"Input bias calibration.",
138 storageClass=
"ExposureF",
139 dimensions=[
"instrument",
"detector"],
142 dark = cT.PrerequisiteInput(
144 doc=
"Input dark calibration.",
145 storageClass=
"ExposureF",
146 dimensions=[
"instrument",
"detector"],
149 flat = cT.PrerequisiteInput(
151 doc=
"Input flat calibration.",
152 storageClass=
"ExposureF",
153 dimensions=[
"instrument",
"physical_filter",
"detector"],
156 ptc = cT.PrerequisiteInput(
158 doc=
"Input Photon Transfer Curve dataset",
159 storageClass=
"PhotonTransferCurveDataset",
160 dimensions=[
"instrument",
"detector"],
163 fringes = cT.PrerequisiteInput(
165 doc=
"Input fringe calibration.",
166 storageClass=
"ExposureF",
167 dimensions=[
"instrument",
"physical_filter",
"detector"],
171 strayLightData = cT.PrerequisiteInput(
173 doc=
"Input stray light calibration.",
174 storageClass=
"StrayLightData",
175 dimensions=[
"instrument",
"physical_filter",
"detector"],
180 bfKernel = cT.PrerequisiteInput(
182 doc=
"Input brighter-fatter kernel.",
183 storageClass=
"NumpyArray",
184 dimensions=[
"instrument"],
188 newBFKernel = cT.PrerequisiteInput(
189 name=
'brighterFatterKernel',
190 doc=
"Newer complete kernel + gain solutions.",
191 storageClass=
"BrighterFatterKernel",
192 dimensions=[
"instrument",
"detector"],
196 defects = cT.PrerequisiteInput(
198 doc=
"Input defect tables.",
199 storageClass=
"Defects",
200 dimensions=[
"instrument",
"detector"],
203 linearizer = cT.PrerequisiteInput(
205 storageClass=
"Linearizer",
206 doc=
"Linearity correction calibration.",
207 dimensions=[
"instrument",
"detector"],
211 opticsTransmission = cT.PrerequisiteInput(
212 name=
"transmission_optics",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the optics.",
215 dimensions=[
"instrument"],
218 filterTransmission = cT.PrerequisiteInput(
219 name=
"transmission_filter",
220 storageClass=
"TransmissionCurve",
221 doc=
"Transmission curve due to the filter.",
222 dimensions=[
"instrument",
"physical_filter"],
225 sensorTransmission = cT.PrerequisiteInput(
226 name=
"transmission_sensor",
227 storageClass=
"TransmissionCurve",
228 doc=
"Transmission curve due to the sensor.",
229 dimensions=[
"instrument",
"detector"],
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name=
"transmission_atmosphere",
234 storageClass=
"TransmissionCurve",
235 doc=
"Transmission curve due to the atmosphere.",
236 dimensions=[
"instrument"],
239 illumMaskedImage = cT.PrerequisiteInput(
241 doc=
"Input illumination correction.",
242 storageClass=
"MaskedImageF",
243 dimensions=[
"instrument",
"physical_filter",
"detector"],
246 deferredChargeCalib = cT.PrerequisiteInput(
247 name=
"deferredCharge",
248 doc=
"Deferred charge/CTI correction dataset.",
249 storageClass=
"IsrCalib",
250 dimensions=[
"instrument",
"detector"],
254 outputExposure = cT.Output(
256 doc=
"Output ISR processed exposure.",
257 storageClass=
"Exposure",
258 dimensions=[
"instrument",
"exposure",
"detector"],
260 preInterpExposure = cT.Output(
261 name=
'preInterpISRCCD',
262 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
263 storageClass=
"ExposureF",
264 dimensions=[
"instrument",
"exposure",
"detector"],
266 outputOssThumbnail = cT.Output(
268 doc=
"Output Overscan-subtracted thumbnail image.",
269 storageClass=
"Thumbnail",
270 dimensions=[
"instrument",
"exposure",
"detector"],
272 outputFlattenedThumbnail = cT.Output(
273 name=
"FlattenedThumb",
274 doc=
"Output flat-corrected thumbnail image.",
275 storageClass=
"Thumbnail",
276 dimensions=[
"instrument",
"exposure",
"detector"],
278 outputStatistics = cT.Output(
279 name=
"isrStatistics",
280 doc=
"Output of additional statistics table.",
281 storageClass=
"StructuredDataDict",
282 dimensions=[
"instrument",
"exposure",
"detector"],
288 if config.doBias
is not True:
289 self.prerequisiteInputs.remove(
"bias")
290 if config.doLinearize
is not True:
291 self.prerequisiteInputs.remove(
"linearizer")
292 if config.doCrosstalk
is not True:
293 self.prerequisiteInputs.remove(
"crosstalkSources")
294 self.prerequisiteInputs.remove(
"crosstalk")
295 if config.doBrighterFatter
is not True:
296 self.prerequisiteInputs.remove(
"bfKernel")
297 self.prerequisiteInputs.remove(
"newBFKernel")
298 if config.doDefect
is not True:
299 self.prerequisiteInputs.remove(
"defects")
300 if config.doDark
is not True:
301 self.prerequisiteInputs.remove(
"dark")
302 if config.doFlat
is not True:
303 self.prerequisiteInputs.remove(
"flat")
304 if config.doFringe
is not True:
305 self.prerequisiteInputs.remove(
"fringes")
306 if config.doStrayLight
is not True:
307 self.prerequisiteInputs.remove(
"strayLightData")
308 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
309 self.prerequisiteInputs.remove(
"ptc")
310 if config.doAttachTransmissionCurve
is not True:
311 self.prerequisiteInputs.remove(
"opticsTransmission")
312 self.prerequisiteInputs.remove(
"filterTransmission")
313 self.prerequisiteInputs.remove(
"sensorTransmission")
314 self.prerequisiteInputs.remove(
"atmosphereTransmission")
316 if config.doUseOpticsTransmission
is not True:
317 self.prerequisiteInputs.remove(
"opticsTransmission")
318 if config.doUseFilterTransmission
is not True:
319 self.prerequisiteInputs.remove(
"filterTransmission")
320 if config.doUseSensorTransmission
is not True:
321 self.prerequisiteInputs.remove(
"sensorTransmission")
322 if config.doUseAtmosphereTransmission
is not True:
323 self.prerequisiteInputs.remove(
"atmosphereTransmission")
324 if config.doIlluminationCorrection
is not True:
325 self.prerequisiteInputs.remove(
"illumMaskedImage")
326 if config.doDeferredCharge
is not True:
327 self.prerequisiteInputs.remove(
"deferredChargeCalib")
329 if config.doWrite
is not True:
330 self.outputs.remove(
"outputExposure")
331 self.outputs.remove(
"preInterpExposure")
332 self.outputs.remove(
"outputFlattenedThumbnail")
333 self.outputs.remove(
"outputOssThumbnail")
334 self.outputs.remove(
"outputStatistics")
336 if config.doSaveInterpPixels
is not True:
337 self.outputs.remove(
"preInterpExposure")
338 if config.qa.doThumbnailOss
is not True:
339 self.outputs.remove(
"outputOssThumbnail")
340 if config.qa.doThumbnailFlattened
is not True:
341 self.outputs.remove(
"outputFlattenedThumbnail")
342 if config.doCalculateStatistics
is not True:
343 self.outputs.remove(
"outputStatistics")
347 pipelineConnections=IsrTaskConnections):
348 """Configuration parameters for IsrTask.
350 Items are grouped in the order
in which they are executed by the task.
352 datasetType = pexConfig.Field(
354 doc="Dataset type for input data; users will typically leave this alone, "
355 "but camera-specific ISR tasks will override it",
359 fallbackFilterName = pexConfig.Field(
361 doc=
"Fallback default filter name for calibrations.",
364 useFallbackDate = pexConfig.Field(
366 doc=
"Pass observation date when using fallback filter.",
369 expectWcs = pexConfig.Field(
372 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
374 fwhm = pexConfig.Field(
376 doc=
"FWHM of PSF in arcseconds.",
379 qa = pexConfig.ConfigField(
381 doc=
"QA related configuration options.",
385 doConvertIntToFloat = pexConfig.Field(
387 doc=
"Convert integer raw images to floating point values?",
392 doSaturation = pexConfig.Field(
394 doc=
"Mask saturated pixels? NB: this is totally independent of the"
395 " interpolation option - this is ONLY setting the bits in the mask."
396 " To have them interpolated make sure doSaturationInterpolation=True",
399 saturatedMaskName = pexConfig.Field(
401 doc=
"Name of mask plane to use in saturation detection and interpolation",
404 saturation = pexConfig.Field(
406 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
407 default=float(
"NaN"),
409 growSaturationFootprintSize = pexConfig.Field(
411 doc=
"Number of pixels by which to grow the saturation footprints",
416 doSuspect = pexConfig.Field(
418 doc=
"Mask suspect pixels?",
421 suspectMaskName = pexConfig.Field(
423 doc=
"Name of mask plane to use for suspect pixels",
426 numEdgeSuspect = pexConfig.Field(
428 doc=
"Number of edge pixels to be flagged as untrustworthy.",
431 edgeMaskLevel = pexConfig.ChoiceField(
433 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
436 'DETECTOR':
'Mask only the edges of the full detector.',
437 'AMP':
'Mask edges of each amplifier.',
442 doSetBadRegions = pexConfig.Field(
444 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
447 badStatistic = pexConfig.ChoiceField(
449 doc=
"How to estimate the average value for BAD regions.",
452 "MEANCLIP":
"Correct using the (clipped) mean of good data",
453 "MEDIAN":
"Correct using the median of the good data",
458 doOverscan = pexConfig.Field(
460 doc=
"Do overscan subtraction?",
463 overscan = pexConfig.ConfigurableField(
464 target=OverscanCorrectionTask,
465 doc=
"Overscan subtraction task for image segments.",
467 overscanFitType = pexConfig.ChoiceField(
469 doc=
"The method for fitting the overscan bias level.",
472 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
473 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
474 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
475 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
476 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
477 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
478 "MEAN":
"Correct using the mean of the overscan region",
479 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
480 "MEDIAN":
"Correct using the median of the overscan region",
481 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
483 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
484 " This option will no longer be used, and will be removed after v20.")
486 overscanOrder = pexConfig.Field(
488 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
489 "or number of spline knots if overscan fit type is a spline."),
491 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
492 " This option will no longer be used, and will be removed after v20.")
494 overscanNumSigmaClip = pexConfig.Field(
496 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
498 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
499 " This option will no longer be used, and will be removed after v20.")
501 overscanIsInt = pexConfig.Field(
503 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
504 " and overscan.FitType=MEDIAN_PER_ROW.",
506 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
507 " This option will no longer be used, and will be removed after v20.")
511 overscanNumLeadingColumnsToSkip = pexConfig.Field(
513 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
516 overscanNumTrailingColumnsToSkip = pexConfig.Field(
518 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
521 overscanMaxDev = pexConfig.Field(
523 doc=
"Maximum deviation from the median for overscan",
524 default=1000.0, check=
lambda x: x > 0
526 overscanBiasJump = pexConfig.Field(
528 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
531 overscanBiasJumpKeyword = pexConfig.Field(
533 doc=
"Header keyword containing information about devices.",
534 default=
"NO_SUCH_KEY",
536 overscanBiasJumpDevices = pexConfig.ListField(
538 doc=
"List of devices that need piecewise overscan correction.",
541 overscanBiasJumpLocation = pexConfig.Field(
543 doc=
"Location of bias jump along y-axis.",
548 doAssembleCcd = pexConfig.Field(
551 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
553 assembleCcd = pexConfig.ConfigurableField(
554 target=AssembleCcdTask,
555 doc=
"CCD assembly task",
559 doAssembleIsrExposures = pexConfig.Field(
562 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
564 doTrimToMatchCalib = pexConfig.Field(
567 doc=
"Trim raw data to match calibration bounding boxes?"
571 doBias = pexConfig.Field(
573 doc=
"Apply bias frame correction?",
576 biasDataProductName = pexConfig.Field(
578 doc=
"Name of the bias data product",
581 doBiasBeforeOverscan = pexConfig.Field(
583 doc=
"Reverse order of overscan and bias correction.",
588 doDeferredCharge = pexConfig.Field(
590 doc=
"Apply deferred charge correction?",
593 deferredChargeCorrection = pexConfig.ConfigurableField(
594 target=DeferredChargeTask,
595 doc=
"Deferred charge correction task.",
599 doVariance = pexConfig.Field(
601 doc=
"Calculate variance?",
604 gain = pexConfig.Field(
606 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
607 default=float(
"NaN"),
609 readNoise = pexConfig.Field(
611 doc=
"The read noise to use if no Detector is present in the Exposure",
614 doEmpiricalReadNoise = pexConfig.Field(
617 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
619 usePtcReadNoise = pexConfig.Field(
622 doc=
"Use readnoise values from the Photon Transfer Curve?"
624 maskNegativeVariance = pexConfig.Field(
627 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
628 "in the measurement of the overscan at an edge due to the data falling off faster "
629 "than the overscan model can account for it."
631 negativeVarianceMaskName = pexConfig.Field(
634 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
637 doLinearize = pexConfig.Field(
639 doc=
"Correct for nonlinearity of the detector's response?",
644 doCrosstalk = pexConfig.Field(
646 doc=
"Apply intra-CCD crosstalk correction?",
649 doCrosstalkBeforeAssemble = pexConfig.Field(
651 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
654 crosstalk = pexConfig.ConfigurableField(
655 target=CrosstalkTask,
656 doc=
"Intra-CCD crosstalk correction",
660 doDefect = pexConfig.Field(
662 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
665 doNanMasking = pexConfig.Field(
667 doc=
"Mask non-finite (NAN, inf) pixels?",
670 doWidenSaturationTrails = pexConfig.Field(
672 doc=
"Widen bleed trails based on their width?",
677 doBrighterFatter = pexConfig.Field(
680 doc=
"Apply the brighter-fatter correction?"
682 brighterFatterLevel = pexConfig.ChoiceField(
685 doc=
"The level at which to correct for brighter-fatter.",
687 "AMP":
"Every amplifier treated separately.",
688 "DETECTOR":
"One kernel per detector",
691 brighterFatterMaxIter = pexConfig.Field(
694 doc=
"Maximum number of iterations for the brighter-fatter correction"
696 brighterFatterThreshold = pexConfig.Field(
699 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
700 "absolute value of the difference between the current corrected image and the one "
701 "from the previous iteration summed over all the pixels."
703 brighterFatterApplyGain = pexConfig.Field(
706 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
708 brighterFatterMaskListToInterpolate = pexConfig.ListField(
710 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
712 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
714 brighterFatterMaskGrowSize = pexConfig.Field(
717 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
718 "when brighter-fatter correction is applied."
722 doDark = pexConfig.Field(
724 doc=
"Apply dark frame correction?",
727 darkDataProductName = pexConfig.Field(
729 doc=
"Name of the dark data product",
734 doStrayLight = pexConfig.Field(
736 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
739 strayLight = pexConfig.ConfigurableField(
740 target=StrayLightTask,
741 doc=
"y-band stray light correction"
745 doFlat = pexConfig.Field(
747 doc=
"Apply flat field correction?",
750 flatDataProductName = pexConfig.Field(
752 doc=
"Name of the flat data product",
755 flatScalingType = pexConfig.ChoiceField(
757 doc=
"The method for scaling the flat on the fly.",
760 "USER":
"Scale by flatUserScale",
761 "MEAN":
"Scale by the inverse of the mean",
762 "MEDIAN":
"Scale by the inverse of the median",
765 flatUserScale = pexConfig.Field(
767 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
770 doTweakFlat = pexConfig.Field(
772 doc=
"Tweak flats to match observed amplifier ratios?",
778 doApplyGains = pexConfig.Field(
780 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
783 usePtcGains = pexConfig.Field(
785 doc=
"Use the gain values from the Photon Transfer Curve?",
788 normalizeGains = pexConfig.Field(
790 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
795 doFringe = pexConfig.Field(
797 doc=
"Apply fringe correction?",
800 fringe = pexConfig.ConfigurableField(
802 doc=
"Fringe subtraction task",
804 fringeAfterFlat = pexConfig.Field(
806 doc=
"Do fringe subtraction after flat-fielding?",
811 doAmpOffset = pexConfig.Field(
812 doc=
"Calculate and apply amp offset corrections?",
816 ampOffset = pexConfig.ConfigurableField(
817 doc=
"Amp offset correction task.",
818 target=AmpOffsetTask,
822 doMeasureBackground = pexConfig.Field(
824 doc=
"Measure the background level on the reduced image?",
829 doCameraSpecificMasking = pexConfig.Field(
831 doc=
"Mask camera-specific bad regions?",
834 masking = pexConfig.ConfigurableField(
840 doInterpolate = pexConfig.Field(
842 doc=
"Interpolate masked pixels?",
845 doSaturationInterpolation = pexConfig.Field(
847 doc=
"Perform interpolation over pixels masked as saturated?"
848 " NB: This is independent of doSaturation; if that is False this plane"
849 " will likely be blank, resulting in a no-op here.",
852 doNanInterpolation = pexConfig.Field(
854 doc=
"Perform interpolation over pixels masked as NaN?"
855 " NB: This is independent of doNanMasking; if that is False this plane"
856 " will likely be blank, resulting in a no-op here.",
859 doNanInterpAfterFlat = pexConfig.Field(
861 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
862 "also have to interpolate them before flat-fielding."),
865 maskListToInterpolate = pexConfig.ListField(
867 doc=
"List of mask planes that should be interpolated.",
868 default=[
'SAT',
'BAD'],
870 doSaveInterpPixels = pexConfig.Field(
872 doc=
"Save a copy of the pre-interpolated pixel values?",
877 fluxMag0T1 = pexConfig.DictField(
880 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
881 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
884 defaultFluxMag0T1 = pexConfig.Field(
886 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
887 default=pow(10.0, 0.4*28.0)
891 doVignette = pexConfig.Field(
893 doc=(
"Compute and attach the validPolygon defining the unvignetted region to the exposure "
894 "according to vignetting parameters?"),
897 doMaskVignettePolygon = pexConfig.Field(
899 doc=(
"Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
903 vignetteValue = pexConfig.Field(
905 doc=
"Value to replace image array pixels with in the vignetted region? Ignored if None.",
909 vignette = pexConfig.ConfigurableField(
911 doc=
"Vignetting task.",
915 doAttachTransmissionCurve = pexConfig.Field(
918 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
920 doUseOpticsTransmission = pexConfig.Field(
923 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
925 doUseFilterTransmission = pexConfig.Field(
928 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
930 doUseSensorTransmission = pexConfig.Field(
933 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
935 doUseAtmosphereTransmission = pexConfig.Field(
938 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
942 doIlluminationCorrection = pexConfig.Field(
945 doc=
"Perform illumination correction?"
947 illuminationCorrectionDataProductName = pexConfig.Field(
949 doc=
"Name of the illumination correction data product.",
952 illumScale = pexConfig.Field(
954 doc=
"Scale factor for the illumination correction.",
957 illumFilters = pexConfig.ListField(
960 doc=
"Only perform illumination correction for these filters."
964 doCalculateStatistics = pexConfig.Field(
966 doc=
"Should additional ISR statistics be calculated?",
969 isrStats = pexConfig.ConfigurableField(
970 target=IsrStatisticsTask,
971 doc=
"Task to calculate additional statistics.",
976 doWrite = pexConfig.Field(
978 doc=
"Persist postISRCCD?",
985 raise ValueError(
"You may not specify both doFlat and doApplyGains")
987 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
996class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
997 """Apply common instrument signature correction algorithms to a raw frame.
999 The process for correcting imaging data
is very similar
from
1000 camera to camera. This task provides a vanilla implementation of
1001 doing these corrections, including the ability to turn certain
1002 corrections off
if they are
not needed. The inputs to the primary
1003 method, `
run()`, are a raw exposure to be corrected
and the
1004 calibration data products. The raw input
is a single chip sized
1005 mosaic of all amps including overscans
and other non-science
1006 pixels. The method `
runDataRef()` identifies
and defines the
1007 calibration data products,
and is intended
for use by a
1008 `lsst.pipe.base.cmdLineTask.CmdLineTask`
and takes
as input only a
1009 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
1010 subclassed
for different camera, although the most camera specific
1011 methods have been split into subtasks that can be redirected
1014 The __init__ method sets up the subtasks
for ISR processing, using
1020 Positional arguments passed to the Task constructor.
1021 None used at this time.
1022 kwargs : `dict`, optional
1023 Keyword arguments passed on to the Task constructor.
1024 None used at this time.
1026 ConfigClass = IsrTaskConfig
1027 _DefaultName = "isr"
1031 self.makeSubtask(
"assembleCcd")
1032 self.makeSubtask(
"crosstalk")
1033 self.makeSubtask(
"strayLight")
1034 self.makeSubtask(
"fringe")
1035 self.makeSubtask(
"masking")
1036 self.makeSubtask(
"overscan")
1037 self.makeSubtask(
"vignette")
1038 self.makeSubtask(
"ampOffset")
1039 self.makeSubtask(
"deferredChargeCorrection")
1040 self.makeSubtask(
"isrStats")
1043 inputs = butlerQC.get(inputRefs)
1046 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
1047 except Exception
as e:
1048 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
1051 inputs[
'isGen3'] =
True
1053 detector = inputs[
'ccdExposure'].getDetector()
1055 if self.config.doCrosstalk
is True:
1058 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
1059 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
1060 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
1062 coeffVector = (self.config.crosstalk.crosstalkValues
1063 if self.config.crosstalk.useConfigCoefficients
else None)
1064 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1065 inputs[
'crosstalk'] = crosstalkCalib
1066 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
1067 if 'crosstalkSources' not in inputs:
1068 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
1071 if 'linearizer' in inputs:
1072 if isinstance(inputs[
'linearizer'], dict):
1074 linearizer.fromYaml(inputs[
'linearizer'])
1075 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1076 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1080 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1082 linearizer = inputs[
'linearizer']
1083 linearizer.log = self.log
1084 inputs[
'linearizer'] = linearizer
1087 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1089 if self.config.doDefect
is True:
1090 if "defects" in inputs
and inputs[
'defects']
is not None:
1094 if not isinstance(inputs[
"defects"], Defects):
1095 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1099 if self.config.doBrighterFatter:
1100 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1101 if brighterFatterKernel
is None:
1102 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1104 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1106 detName = detector.getName()
1107 level = brighterFatterKernel.level
1110 inputs[
'bfGains'] = brighterFatterKernel.gain
1111 if self.config.brighterFatterLevel ==
'DETECTOR':
1112 if level ==
'DETECTOR':
1113 if detName
in brighterFatterKernel.detKernels:
1114 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1116 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1117 elif level ==
'AMP':
1118 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1120 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1121 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1122 elif self.config.brighterFatterLevel ==
'AMP':
1123 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1125 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1126 expId = inputs[
'ccdExposure'].info.id
1127 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1129 assembler=self.assembleCcd
1130 if self.config.doAssembleIsrExposures
else None)
1132 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1134 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1135 if 'strayLightData' not in inputs:
1136 inputs[
'strayLightData'] =
None
1138 outputs = self.
run(**inputs)
1139 butlerQC.put(outputs, outputRefs)
1142 """Retrieve necessary frames for instrument signature removal.
1144 Pre-fetching all required ISR data products limits the IO
1145 required by the ISR. Any conflict between the calibration data
1146 available and that needed
for ISR
is also detected prior to
1147 doing processing, allowing it to fail quickly.
1151 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1152 Butler reference of the detector data to be processed
1154 The raw exposure that will later be corrected
with the
1155 retrieved calibration data; should
not be modified
in this
1160 result : `lsst.pipe.base.Struct`
1161 Result struct
with components (which may be `
None`):
1163 - ``linearizer``: functor
for linearization
1165 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1168 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1170 - ``fringes``: `lsst.pipe.base.Struct`
with components:
1172 - ``seed``: random seed derived
from the ccdExposureId
for random
1173 number generator (`uint32`).
1175 A ``TransmissionCurve`` that represents the throughput of the
1176 optics, to be evaluated
in focal-plane coordinates.
1178 A ``TransmissionCurve`` that represents the throughput of the
1179 filter itself, to be evaluated
in focal-plane coordinates.
1181 A ``TransmissionCurve`` that represents the throughput of the
1182 sensor itself, to be evaluated
in post-assembly trimmed
1183 detector coordinates.
1185 A ``TransmissionCurve`` that represents the throughput of the
1186 atmosphere, assumed to be spatially constant.
1187 - ``strayLightData`` : `object`
1188 An opaque object containing calibration information
for
1189 stray-light correction. If `
None`, no correction will be
1191 - ``illumMaskedImage`` : illumination correction image
1196 NotImplementedError :
1197 Raised
if a per-amplifier brighter-fatter kernel
is requested by
1201 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1202 dateObs = dateObs.toPython().isoformat()
1203 except RuntimeError:
1204 self.log.warning(
"Unable to identify dateObs for rawExposure.")
1207 ccd = rawExposure.getDetector()
1208 filterLabel = rawExposure.getFilter()
1209 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1210 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1211 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
1212 if self.config.doBias
else None)
1215 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1217 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1218 linearizer.log = self.log
1219 if isinstance(linearizer, numpy.ndarray):
1222 crosstalkCalib =
None
1223 if self.config.doCrosstalk:
1225 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1227 coeffVector = (self.config.crosstalk.crosstalkValues
1228 if self.config.crosstalk.useConfigCoefficients
else None)
1229 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1230 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1231 if self.config.doCrosstalk
else None)
1233 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
1234 if self.config.doDark
else None)
1235 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
1237 if self.config.doFlat
else None)
1239 brighterFatterKernel =
None
1240 brighterFatterGains =
None
1241 if self.config.doBrighterFatter
is True:
1246 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1247 brighterFatterGains = brighterFatterKernel.gain
1248 self.log.info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1251 brighterFatterKernel = dataRef.get(
"bfKernel")
1252 self.log.info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1254 brighterFatterKernel =
None
1255 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1258 if self.config.brighterFatterLevel ==
'DETECTOR':
1259 if brighterFatterKernel.detKernels:
1260 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1262 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1265 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1267 defectList = (dataRef.get(
"defects")
1268 if self.config.doDefect
else None)
1269 expId = rawExposure.info.id
1270 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1271 if self.config.doAssembleIsrExposures
else None)
1272 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1273 else pipeBase.Struct(fringes=
None))
1275 if self.config.doAttachTransmissionCurve:
1276 opticsTransmission = (dataRef.get(
"transmission_optics")
1277 if self.config.doUseOpticsTransmission
else None)
1278 filterTransmission = (dataRef.get(
"transmission_filter")
1279 if self.config.doUseFilterTransmission
else None)
1280 sensorTransmission = (dataRef.get(
"transmission_sensor")
1281 if self.config.doUseSensorTransmission
else None)
1282 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1283 if self.config.doUseAtmosphereTransmission
else None)
1285 opticsTransmission =
None
1286 filterTransmission =
None
1287 sensorTransmission =
None
1288 atmosphereTransmission =
None
1290 if self.config.doStrayLight:
1291 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1293 strayLightData =
None
1296 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1297 if (self.config.doIlluminationCorrection
1298 and physicalFilter
in self.config.illumFilters)
1302 return pipeBase.Struct(bias=biasExposure,
1303 linearizer=linearizer,
1304 crosstalk=crosstalkCalib,
1305 crosstalkSources=crosstalkSources,
1308 bfKernel=brighterFatterKernel,
1309 bfGains=brighterFatterGains,
1311 fringes=fringeStruct,
1312 opticsTransmission=opticsTransmission,
1313 filterTransmission=filterTransmission,
1314 sensorTransmission=sensorTransmission,
1315 atmosphereTransmission=atmosphereTransmission,
1316 strayLightData=strayLightData,
1317 illumMaskedImage=illumMaskedImage
1321 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1322 crosstalk=None, crosstalkSources=None,
1323 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1324 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1325 sensorTransmission=
None, atmosphereTransmission=
None,
1326 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1327 deferredCharge=
None, isGen3=
False,
1329 """Perform instrument signature removal on an exposure.
1331 Steps included in the ISR processing,
in order performed, are:
1332 - saturation
and suspect pixel masking
1333 - overscan subtraction
1334 - CCD assembly of individual amplifiers
1336 - variance image construction
1337 - linearization of non-linear response
1339 - brighter-fatter correction
1342 - stray light subtraction
1344 - masking of known defects
and camera specific features
1345 - vignette calculation
1346 - appending transmission curve
and distortion model
1351 The raw exposure that
is to be run through ISR. The
1352 exposure
is modified by this method.
1354 The camera geometry
for this exposure. Required
if
1355 one
or more of ``ccdExposure``, ``bias``, ``dark``,
or
1356 ``flat`` does
not have an associated detector.
1358 Bias calibration frame.
1360 Functor
for linearization.
1362 Calibration
for crosstalk.
1363 crosstalkSources : `list`, optional
1364 List of possible crosstalk sources.
1366 Dark calibration frame.
1368 Flat calibration frame.
1370 Photon transfer curve dataset,
with, e.g., gains
1372 bfKernel : `numpy.ndarray`, optional
1373 Brighter-fatter kernel.
1374 bfGains : `dict` of `float`, optional
1375 Gains used to override the detector
's nominal gains for the
1376 brighter-fatter correction. A dict keyed by amplifier name for
1377 the detector
in question.
1380 fringes : `lsst.pipe.base.Struct`, optional
1381 Struct containing the fringe correction data,
with
1384 - ``seed``: random seed derived
from the ccdExposureId
for random
1385 number generator (`uint32`)
1387 A ``TransmissionCurve`` that represents the throughput of the,
1388 optics, to be evaluated
in focal-plane coordinates.
1390 A ``TransmissionCurve`` that represents the throughput of the
1391 filter itself, to be evaluated
in focal-plane coordinates.
1393 A ``TransmissionCurve`` that represents the throughput of the
1394 sensor itself, to be evaluated
in post-assembly trimmed detector
1397 A ``TransmissionCurve`` that represents the throughput of the
1398 atmosphere, assumed to be spatially constant.
1399 detectorNum : `int`, optional
1400 The integer number
for the detector to process.
1401 isGen3 : bool, optional
1402 Flag this call to
run()
as using the Gen3 butler environment.
1403 strayLightData : `object`, optional
1404 Opaque object containing calibration information
for stray-light
1405 correction. If `
None`, no correction will be performed.
1407 Illumination correction image.
1411 result : `lsst.pipe.base.Struct`
1412 Result struct
with component:
1414 The fully ISR corrected exposure.
1416 An alias
for `exposure`
1417 - ``ossThumb`` : `numpy.ndarray`
1418 Thumbnail image of the exposure after overscan subtraction.
1419 - ``flattenedThumb`` : `numpy.ndarray`
1420 Thumbnail image of the exposure after flat-field correction.
1421 - ``outputStatistics`` : ``
1422 Values of the additional statistics calculated.
1427 Raised
if a configuration option
is set to
True, but the
1428 required calibration data has
not been specified.
1432 The current processed exposure can be viewed by setting the
1433 appropriate lsstDebug entries
in the `debug.display`
1434 dictionary. The names of these entries correspond to some of
1435 the IsrTaskConfig Boolean options,
with the value denoting the
1436 frame to use. The exposure
is shown inside the matching
1437 option check
and after the processing of that step has
1438 finished. The steps
with debug points are:
1449 In addition, setting the
"postISRCCD" entry displays the
1450 exposure after all ISR processing has finished.
1459 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1464 if isinstance(ccdExposure, ButlerDataRef):
1467 ccd = ccdExposure.getDetector()
1468 filterLabel = ccdExposure.getFilter()
1469 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1472 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1473 ccd = [
FakeAmp(ccdExposure, self.config)]
1476 if self.config.doBias
and bias
is None:
1477 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1479 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1480 if self.config.doBrighterFatter
and bfKernel
is None:
1481 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1482 if self.config.doDark
and dark
is None:
1483 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1484 if self.config.doFlat
and flat
is None:
1485 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1486 if self.config.doDefect
and defects
is None:
1487 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1488 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1489 and fringes.fringes
is None):
1494 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1495 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1496 and illumMaskedImage
is None):
1497 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1498 if (self.config.doDeferredCharge
and deferredCharge
is None):
1499 raise RuntimeError(
"Must supply a deferred charge calibration if config.doDeferredCharge=True.")
1502 if self.config.doConvertIntToFloat:
1503 self.log.info(
"Converting exposure to floating point values.")
1506 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1507 self.log.info(
"Applying bias correction.")
1508 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1509 trimToFit=self.config.doTrimToMatchCalib)
1517 if ccdExposure.getBBox().contains(amp.getBBox()):
1522 if self.config.doOverscan
and not badAmp:
1525 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1526 if overscanResults
is not None and \
1527 self.config.qa
is not None and self.config.qa.saveStats
is True:
1528 if isinstance(overscanResults.overscanFit, float):
1529 qaMedian = overscanResults.overscanFit
1530 qaStdev = float(
"NaN")
1532 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1533 afwMath.MEDIAN | afwMath.STDEVCLIP)
1534 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1535 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1537 self.metadata[f
"FIT MEDIAN {amp.getName()}"] = qaMedian
1538 self.metadata[f
"FIT STDEV {amp.getName()}"] = qaStdev
1539 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1540 amp.getName(), qaMedian, qaStdev)
1543 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1544 afwMath.MEDIAN | afwMath.STDEVCLIP)
1545 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1546 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1548 self.metadata[f
"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter
1549 self.metadata[f
"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter
1550 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1551 amp.getName(), qaMedianAfter, qaStdevAfter)
1553 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1556 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1557 overscanResults =
None
1559 overscans.append(overscanResults
if overscanResults
is not None else None)
1561 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1563 if self.config.doDeferredCharge:
1564 self.log.info(
"Applying deferred charge/CTI correction.")
1565 self.deferredChargeCorrection.
run(ccdExposure, deferredCharge)
1566 self.
debugView(ccdExposure,
"doDeferredCharge")
1568 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1569 self.log.info(
"Applying crosstalk correction.")
1570 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1571 crosstalkSources=crosstalkSources, camera=camera)
1572 self.
debugView(ccdExposure,
"doCrosstalk")
1574 if self.config.doAssembleCcd:
1575 self.log.info(
"Assembling CCD from amplifiers.")
1576 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1578 if self.config.expectWcs
and not ccdExposure.getWcs():
1579 self.log.warning(
"No WCS found in input exposure.")
1580 self.
debugView(ccdExposure,
"doAssembleCcd")
1583 if self.config.qa.doThumbnailOss:
1584 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1586 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1587 self.log.info(
"Applying bias correction.")
1588 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1589 trimToFit=self.config.doTrimToMatchCalib)
1592 if self.config.doVariance:
1593 for amp, overscanResults
in zip(ccd, overscans):
1594 if ccdExposure.getBBox().contains(amp.getBBox()):
1595 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1596 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1597 if overscanResults
is not None:
1599 overscanImage=overscanResults.overscanImage,
1605 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1606 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1607 afwMath.MEDIAN | afwMath.STDEVCLIP)
1608 self.metadata[f
"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1609 qaStats.getValue(afwMath.MEDIAN)
1610 self.metadata[f
"ISR VARIANCE {amp.getName()} STDEV"] = \
1611 qaStats.getValue(afwMath.STDEVCLIP)
1612 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1613 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1614 qaStats.getValue(afwMath.STDEVCLIP))
1615 if self.config.maskNegativeVariance:
1619 self.log.info(
"Applying linearizer.")
1620 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1621 detector=ccd, log=self.log)
1623 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1624 self.log.info(
"Applying crosstalk correction.")
1625 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1626 crosstalkSources=crosstalkSources, isTrimmed=
True)
1627 self.
debugView(ccdExposure,
"doCrosstalk")
1632 if self.config.doDefect:
1633 self.log.info(
"Masking defects.")
1636 if self.config.numEdgeSuspect > 0:
1637 self.log.info(
"Masking edges as SUSPECT.")
1638 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1639 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1641 if self.config.doNanMasking:
1642 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1645 if self.config.doWidenSaturationTrails:
1646 self.log.info(
"Widening saturation trails.")
1647 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1649 if self.config.doCameraSpecificMasking:
1650 self.log.info(
"Masking regions for camera specific reasons.")
1651 self.masking.
run(ccdExposure)
1653 if self.config.doBrighterFatter:
1663 interpExp = ccdExposure.clone()
1665 isrFunctions.interpolateFromMask(
1666 maskedImage=interpExp.getMaskedImage(),
1667 fwhm=self.config.fwhm,
1668 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1669 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1671 bfExp = interpExp.clone()
1673 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1674 type(bfKernel), type(bfGains))
1675 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1676 self.config.brighterFatterMaxIter,
1677 self.config.brighterFatterThreshold,
1678 self.config.brighterFatterApplyGain,
1680 if bfResults[1] == self.config.brighterFatterMaxIter:
1681 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1684 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1686 image = ccdExposure.getMaskedImage().getImage()
1687 bfCorr = bfExp.getMaskedImage().getImage()
1688 bfCorr -= interpExp.getMaskedImage().getImage()
1697 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1698 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1701 if self.config.brighterFatterMaskGrowSize > 0:
1702 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1703 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1704 isrFunctions.growMasks(ccdExposure.getMask(),
1705 radius=self.config.brighterFatterMaskGrowSize,
1706 maskNameList=maskPlane,
1707 maskValue=maskPlane)
1709 self.
debugView(ccdExposure,
"doBrighterFatter")
1711 if self.config.doDark:
1712 self.log.info(
"Applying dark correction.")
1716 if self.config.doFringe
and not self.config.fringeAfterFlat:
1717 self.log.info(
"Applying fringe correction before flat.")
1718 self.fringe.
run(ccdExposure, **fringes.getDict())
1721 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1722 self.log.info(
"Checking strayLight correction.")
1723 self.strayLight.
run(ccdExposure, strayLightData)
1724 self.
debugView(ccdExposure,
"doStrayLight")
1726 if self.config.doFlat:
1727 self.log.info(
"Applying flat correction.")
1731 if self.config.doApplyGains:
1732 self.log.info(
"Applying gain correction instead of flat.")
1733 if self.config.usePtcGains:
1734 self.log.info(
"Using gains from the Photon Transfer Curve.")
1735 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1738 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1740 if self.config.doFringe
and self.config.fringeAfterFlat:
1741 self.log.info(
"Applying fringe correction after flat.")
1742 self.fringe.
run(ccdExposure, **fringes.getDict())
1744 if self.config.doVignette:
1745 if self.config.doMaskVignettePolygon:
1746 self.log.info(
"Constructing, attaching, and masking vignette polygon.")
1748 self.log.info(
"Constructing and attaching vignette polygon.")
1750 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1751 vignetteValue=self.config.vignetteValue, log=self.log)
1753 if self.config.doAttachTransmissionCurve:
1754 self.log.info(
"Adding transmission curves.")
1755 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1756 filterTransmission=filterTransmission,
1757 sensorTransmission=sensorTransmission,
1758 atmosphereTransmission=atmosphereTransmission)
1760 flattenedThumb =
None
1761 if self.config.qa.doThumbnailFlattened:
1762 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1764 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1765 self.log.info(
"Performing illumination correction.")
1766 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1767 illumMaskedImage, illumScale=self.config.illumScale,
1768 trimToFit=self.config.doTrimToMatchCalib)
1771 if self.config.doSaveInterpPixels:
1772 preInterpExp = ccdExposure.clone()
1787 if self.config.doSetBadRegions:
1788 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1789 if badPixelCount > 0:
1790 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1792 if self.config.doInterpolate:
1793 self.log.info(
"Interpolating masked pixels.")
1794 isrFunctions.interpolateFromMask(
1795 maskedImage=ccdExposure.getMaskedImage(),
1796 fwhm=self.config.fwhm,
1797 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1798 maskNameList=list(self.config.maskListToInterpolate)
1804 if self.config.doAmpOffset:
1805 self.log.info(
"Correcting amp offsets.")
1806 self.ampOffset.
run(ccdExposure)
1808 if self.config.doMeasureBackground:
1809 self.log.info(
"Measuring background level.")
1812 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1814 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1815 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1816 afwMath.MEDIAN | afwMath.STDEVCLIP)
1817 self.metadata[f
"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1818 self.metadata[f
"ISR BACKGROUND {amp.getName()} STDEV"] = \
1819 qaStats.getValue(afwMath.STDEVCLIP)
1820 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1821 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1822 qaStats.getValue(afwMath.STDEVCLIP))
1825 outputStatistics =
None
1826 if self.config.doCalculateStatistics:
1827 outputStatistics = self.isrStats.
run(ccdExposure, overscanResults=overscans,
1830 self.
debugView(ccdExposure,
"postISRCCD")
1832 return pipeBase.Struct(
1833 exposure=ccdExposure,
1835 flattenedThumb=flattenedThumb,
1837 preInterpExposure=preInterpExp,
1838 outputExposure=ccdExposure,
1839 outputOssThumbnail=ossThumb,
1840 outputFlattenedThumbnail=flattenedThumb,
1841 outputStatistics=outputStatistics,
1846 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1848 This method contains the `CmdLineTask` interface to the ISR
1849 processing. All IO is handled here, freeing the `
run()` method
1850 to manage only pixel-level calculations. The steps performed
1852 - Read
in necessary detrending/isr/calibration data.
1853 - Process raw exposure
in `
run()`.
1854 - Persist the ISR-corrected exposure
as "postISRCCD" if
1855 config.doWrite=
True.
1859 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1860 DataRef of the detector data to be processed
1864 result : `lsst.pipe.base.Struct`
1865 Result struct
with component:
1867 The fully ISR corrected exposure.
1872 Raised
if a configuration option
is set to
True, but the
1873 required calibration data does
not exist.
1876 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1878 ccdExposure = sensorRef.get(self.config.datasetType)
1880 camera = sensorRef.get(
"camera")
1881 isrData = self.
readIsrData(sensorRef, ccdExposure)
1883 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1885 if self.config.doWrite:
1886 sensorRef.put(result.exposure,
"postISRCCD")
1887 if result.preInterpExposure
is not None:
1888 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1889 if result.ossThumb
is not None:
1890 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1891 if result.flattenedThumb
is not None:
1892 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1897 """Retrieve a calibration dataset for removing instrument signature.
1902 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1903 DataRef of the detector data to find calibration datasets
1906 Type of dataset to retrieve (e.g.
'bias',
'flat', etc).
1907 dateObs : `str`, optional
1908 Date of the observation. Used to correct butler failures
1909 when using fallback filters.
1911 If
True, disable butler proxies to enable error handling
1912 within this routine.
1917 Requested calibration frame.
1922 Raised
if no matching calibration frame can be found.
1925 exp = dataRef.get(datasetType, immediate=immediate)
1926 except Exception
as exc1:
1927 if not self.config.fallbackFilterName:
1928 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1930 if self.config.useFallbackDate
and dateObs:
1931 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1932 dateObs=dateObs, immediate=immediate)
1934 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1935 except Exception
as exc2:
1936 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1937 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1938 self.log.warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1940 if self.config.doAssembleIsrExposures:
1941 exp = self.assembleCcd.assembleCcd(exp)
1945 """Ensure that the data returned by Butler is a fully constructed exp.
1947 ISR requires exposure-level image data for historical reasons, so
if we
1948 did
not recieve that
from Butler, construct it
from what we have,
1949 modifying the input
in place.
1954 or `lsst.afw.image.ImageF`
1955 The input data structure obtained
from Butler.
1956 camera : `lsst.afw.cameraGeom.camera`, optional
1957 The camera associated
with the image. Used to find the appropriate
1958 detector
if detector
is not already set.
1959 detectorNum : `int`, optional
1960 The detector
in the camera to attach,
if the detector
is not
1966 The re-constructed exposure,
with appropriate detector parameters.
1971 Raised
if the input data cannot be used to construct an exposure.
1973 if isinstance(inputExp, afwImage.DecoratedImageU):
1974 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1975 elif isinstance(inputExp, afwImage.ImageF):
1976 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1977 elif isinstance(inputExp, afwImage.MaskedImageF):
1978 inputExp = afwImage.makeExposure(inputExp)
1979 elif isinstance(inputExp, afwImage.Exposure):
1981 elif inputExp
is None:
1985 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1988 if inputExp.getDetector()
is None:
1989 if camera
is None or detectorNum
is None:
1990 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1991 'without a detector set.')
1992 inputExp.setDetector(camera[detectorNum])
1997 """Convert exposure image from uint16 to float.
1999 If the exposure does not need to be converted, the input
is
2000 immediately returned. For exposures that are converted to use
2001 floating point pixels, the variance
is set to unity
and the
2007 The raw exposure to be converted.
2012 The input ``exposure``, converted to floating point pixels.
2017 Raised
if the exposure type cannot be converted to float.
2020 if isinstance(exposure, afwImage.ExposureF):
2022 self.log.debug(
"Exposure already of type float.")
2024 if not hasattr(exposure,
"convertF"):
2025 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
2027 newexposure = exposure.convertF()
2028 newexposure.variance[:] = 1
2029 newexposure.mask[:] = 0x0
2034 """Identify bad amplifiers, saturated and suspect pixels.
2039 Input exposure to be masked.
2041 Catalog of parameters defining the amplifier on this
2044 List of defects. Used to determine if the entire
2050 If this
is true, the entire amplifier area
is covered by
2051 defects
and unusable.
2054 maskedImage = ccdExposure.getMaskedImage()
2061 if defects
is not None:
2062 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
2068 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
2070 maskView = dataView.getMask()
2071 maskView |= maskView.getPlaneBitMask(
"BAD")
2079 if self.config.doSaturation
and not badAmp:
2080 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2081 if self.config.doSuspect
and not badAmp:
2082 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2083 if math.isfinite(self.config.saturation):
2084 limits.update({self.config.saturatedMaskName: self.config.saturation})
2086 for maskName, maskThreshold
in limits.items():
2087 if not math.isnan(maskThreshold):
2088 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2089 isrFunctions.makeThresholdMask(
2090 maskedImage=dataView,
2091 threshold=maskThreshold,
2098 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2100 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2101 self.config.suspectMaskName])
2102 if numpy.all(maskView.getArray() & maskVal > 0):
2104 maskView |= maskView.getPlaneBitMask(
"BAD")
2109 """Apply overscan correction in place.
2111 This method does initial pixel rejection of the overscan
2112 region. The overscan can also be optionally segmented to
2113 allow for discontinuous overscan responses to be fit
2114 separately. The actual overscan subtraction
is performed by
2115 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2116 which
is called here after the amplifier
is preprocessed.
2121 Exposure to have overscan correction performed.
2122 amp : `lsst.afw.cameraGeom.Amplifer`
2123 The amplifier to consider
while correcting the overscan.
2127 overscanResults : `lsst.pipe.base.Struct`
2128 Result struct
with components:
2130 Value
or fit subtracted
from the amplifier image data.
2132 Value
or fit subtracted
from the overscan image data.
2134 Image of the overscan region
with the overscan
2135 correction applied. This quantity
is used to estimate
2136 the amplifier read noise empirically.
2141 Raised
if the ``amp`` does
not contain raw pixel information.
2145 lsst.ip.isr.isrFunctions.overscanCorrection
2147 if amp.getRawHorizontalOverscanBBox().isEmpty():
2148 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2151 statControl = afwMath.StatisticsControl()
2152 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2155 dataBBox = amp.getRawDataBBox()
2156 oscanBBox = amp.getRawHorizontalOverscanBBox()
2160 prescanBBox = amp.getRawPrescanBBox()
2161 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2162 dx0 += self.config.overscanNumLeadingColumnsToSkip
2163 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2165 dx0 += self.config.overscanNumTrailingColumnsToSkip
2166 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2173 if ((self.config.overscanBiasJump
2174 and self.config.overscanBiasJumpLocation)
2175 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2176 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2177 self.config.overscanBiasJumpDevices)):
2178 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2179 yLower = self.config.overscanBiasJumpLocation
2180 yUpper = dataBBox.getHeight() - yLower
2182 yUpper = self.config.overscanBiasJumpLocation
2183 yLower = dataBBox.getHeight() - yUpper
2201 oscanBBox.getHeight())))
2205 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2206 ampImage = ccdExposure.maskedImage[imageBBox]
2207 overscanImage = ccdExposure.maskedImage[overscanBBox]
2209 overscanArray = overscanImage.image.array
2210 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2211 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2212 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2214 statControl = afwMath.StatisticsControl()
2215 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2217 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2220 if dx0 != 0
or dx1 != 0:
2221 fullOverscan = ccdExposure.maskedImage[oscanBBox]
2222 overscanVector = overscanResults.overscanFit.array[:, 0]
2223 overscanModel = afwImage.ImageF(fullOverscan.getDimensions())
2224 overscanModel.array[:, :] = 0.0
2225 overscanModel.array[:, 0:dx0] = overscanVector[:, numpy.newaxis]
2226 overscanModel.array[:, dx1:] = overscanVector[:, numpy.newaxis]
2227 fullOverscanImage = fullOverscan.getImage()
2228 fullOverscanImage -= overscanModel
2229 overscanResults = pipeBase.Struct(imageFit=overscanResults.imageFit,
2230 overscanFit=overscanModel,
2231 overscanImage=fullOverscan,
2232 edgeMask=overscanResults.edgeMask)
2235 levelStat = afwMath.MEDIAN
2236 sigmaStat = afwMath.STDEVCLIP
2238 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2239 self.config.qa.flatness.nIter)
2240 metadata = ccdExposure.getMetadata()
2241 ampNum = amp.getName()
2243 if isinstance(overscanResults.overscanFit, float):
2244 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit
2245 metadata[f
"ISR_OSCAN_SIGMA{ampNum}"] = 0.0
2247 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2248 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat)
2249 metadata[f
"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat)
2251 return overscanResults
2254 """Set the variance plane using the gain and read noise
2256 The read noise is calculated
from the ``overscanImage``
if the
2257 ``doEmpiricalReadNoise`` option
is set
in the configuration; otherwise
2258 the value
from the amplifier data
is used.
2263 Exposure to process.
2264 amp : `lsst.afw.table.AmpInfoRecord`
or `FakeAmp`
2265 Amplifier detector data.
2267 Image of overscan, required only
for empirical read noise.
2269 PTC dataset containing the gains
and read noise.
2275 Raised
if either ``usePtcGains`` of ``usePtcReadNoise``
2276 are ``
True``, but ptcDataset
is not provided.
2278 Raised
if ```doEmpiricalReadNoise``
is ``
True`` but
2279 ``overscanImage``
is ``
None``.
2283 lsst.ip.isr.isrFunctions.updateVariance
2285 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2286 if self.config.usePtcGains:
2287 if ptcDataset
is None:
2288 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2290 gain = ptcDataset.gain[amp.getName()]
2291 self.log.info(
"Using gain from Photon Transfer Curve.")
2293 gain = amp.getGain()
2295 if math.isnan(gain):
2297 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2300 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2301 amp.getName(), gain, patchedGain)
2304 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2305 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2307 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2308 stats = afwMath.StatisticsControl()
2309 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2310 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2311 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2312 amp.getName(), readNoise)
2313 elif self.config.usePtcReadNoise:
2314 if ptcDataset
is None:
2315 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2317 readNoise = ptcDataset.noise[amp.getName()]
2318 self.log.info(
"Using read noise from Photon Transfer Curve.")
2320 readNoise = amp.getReadNoise()
2322 isrFunctions.updateVariance(
2323 maskedImage=ampExposure.getMaskedImage(),
2325 readNoise=readNoise,
2329 """Identify and mask pixels with negative variance values.
2334 Exposure to process.
2338 lsst.ip.isr.isrFunctions.updateVariance
2340 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2341 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2342 exposure.mask.array[bad] |= maskPlane
2345 """Apply dark correction in place.
2350 Exposure to process.
2352 Dark exposure of the same size as ``exposure``.
2353 invert : `Bool`, optional
2354 If
True, re-add the dark to an already corrected image.
2359 Raised
if either ``exposure``
or ``darkExposure`` do
not
2360 have their dark time defined.
2364 lsst.ip.isr.isrFunctions.darkCorrection
2366 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2367 if math.isnan(expScale):
2368 raise RuntimeError(
"Exposure darktime is NAN.")
2369 if darkExposure.getInfo().getVisitInfo()
is not None \
2370 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2371 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2375 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2378 isrFunctions.darkCorrection(
2379 maskedImage=exposure.getMaskedImage(),
2380 darkMaskedImage=darkExposure.getMaskedImage(),
2382 darkScale=darkScale,
2384 trimToFit=self.config.doTrimToMatchCalib
2388 """Check if linearization is needed for the detector cameraGeom.
2390 Checks config.doLinearize and the linearity type of the first
2396 Detector to get linearity type
from.
2400 doLinearize : `Bool`
2401 If
True, linearization should be performed.
2403 return self.config.doLinearize
and \
2404 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2407 """Apply flat correction in place.
2412 Exposure to process.
2414 Flat exposure of the same size as ``exposure``.
2415 invert : `Bool`, optional
2416 If
True, unflatten an already flattened image.
2420 lsst.ip.isr.isrFunctions.flatCorrection
2422 isrFunctions.flatCorrection(
2423 maskedImage=exposure.getMaskedImage(),
2424 flatMaskedImage=flatExposure.getMaskedImage(),
2425 scalingType=self.config.flatScalingType,
2426 userScale=self.config.flatUserScale,
2428 trimToFit=self.config.doTrimToMatchCalib
2432 """Detect and mask saturated pixels in config.saturatedMaskName.
2437 Exposure to process. Only the amplifier DataSec is processed.
2439 Amplifier detector data.
2443 lsst.ip.isr.isrFunctions.makeThresholdMask
2445 if not math.isnan(amp.getSaturation()):
2446 maskedImage = exposure.getMaskedImage()
2447 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2448 isrFunctions.makeThresholdMask(
2449 maskedImage=dataView,
2450 threshold=amp.getSaturation(),
2452 maskName=self.config.saturatedMaskName,
2456 """Interpolate over saturated pixels, in place.
2458 This method should be called after `saturationDetection`, to
2459 ensure that the saturated pixels have been identified in the
2460 SAT mask. It should also be called after `assembleCcd`, since
2461 saturated regions may cross amplifier boundaries.
2466 Exposure to process.
2470 lsst.ip.isr.isrTask.saturationDetection
2471 lsst.ip.isr.isrFunctions.interpolateFromMask
2473 isrFunctions.interpolateFromMask(
2474 maskedImage=exposure.getMaskedImage(),
2475 fwhm=self.config.fwhm,
2476 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2477 maskNameList=list(self.config.saturatedMaskName),
2481 """Detect and mask suspect pixels in config.suspectMaskName.
2486 Exposure to process. Only the amplifier DataSec is processed.
2488 Amplifier detector data.
2492 lsst.ip.isr.isrFunctions.makeThresholdMask
2496 Suspect pixels are pixels whose value
is greater than
2497 amp.getSuspectLevel(). This
is intended to indicate pixels that may be
2498 affected by unknown systematics;
for example
if non-linearity
2499 corrections above a certain level are unstable then that would be a
2500 useful value
for suspectLevel. A value of `nan` indicates that no such
2501 level exists
and no pixels are to be masked
as suspicious.
2503 suspectLevel = amp.getSuspectLevel()
2504 if math.isnan(suspectLevel):
2507 maskedImage = exposure.getMaskedImage()
2508 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2509 isrFunctions.makeThresholdMask(
2510 maskedImage=dataView,
2511 threshold=suspectLevel,
2513 maskName=self.config.suspectMaskName,
2517 """Mask defects using mask plane "BAD", in place.
2522 Exposure to process.
2525 List of defects to mask.
2529 Call this after CCD assembly, since defects may cross amplifier
2532 maskedImage = exposure.getMaskedImage()
2533 if not isinstance(defectBaseList, Defects):
2535 defectList =
Defects(defectBaseList)
2537 defectList = defectBaseList
2538 defectList.maskPixels(maskedImage, maskName=
"BAD")
2540 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2541 """Mask edge pixels with applicable mask plane.
2546 Exposure to process.
2547 numEdgePixels : `int`, optional
2548 Number of edge pixels to mask.
2549 maskPlane : `str`, optional
2550 Mask plane name to use.
2551 level : `str`, optional
2552 Level at which to mask edges.
2554 maskedImage = exposure.getMaskedImage()
2555 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2557 if numEdgePixels > 0:
2558 if level ==
'DETECTOR':
2559 boxes = [maskedImage.getBBox()]
2560 elif level ==
'AMP':
2561 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2566 subImage = maskedImage[box]
2567 box.grow(-numEdgePixels)
2569 SourceDetectionTask.setEdgeBits(
2575 """Mask and interpolate defects using mask plane "BAD", in place.
2580 Exposure to process.
2583 List of defects to mask
and interpolate.
2587 lsst.ip.isr.isrTask.maskDefect
2590 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2591 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2592 isrFunctions.interpolateFromMask(
2593 maskedImage=exposure.getMaskedImage(),
2594 fwhm=self.config.fwhm,
2595 growSaturatedFootprints=0,
2596 maskNameList=[
"BAD"],
2600 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2605 Exposure to process.
2609 We mask over all non-finite values (NaN, inf), including those
2610 that are masked with other bits (because those may
or may
not be
2611 interpolated over later,
and we want to remove all NaN/infs).
2612 Despite this behaviour, the
"UNMASKEDNAN" mask plane
is used to
2613 preserve the historical name.
2615 maskedImage = exposure.getMaskedImage()
2618 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2619 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2620 numNans =
maskNans(maskedImage, maskVal)
2621 self.metadata[
"NUMNANS"] = numNans
2623 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2626 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2632 Exposure to process.
2636 lsst.ip.isr.isrTask.maskNan
2639 isrFunctions.interpolateFromMask(
2640 maskedImage=exposure.getMaskedImage(),
2641 fwhm=self.config.fwhm,
2642 growSaturatedFootprints=0,
2643 maskNameList=["UNMASKEDNAN"],
2647 """Measure the image background in subgrids, for quality control.
2652 Exposure to process.
2654 Configuration object containing parameters on which background
2655 statistics and subgrids to use.
2657 if IsrQaConfig
is not None:
2658 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2659 IsrQaConfig.flatness.nIter)
2660 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2661 statsControl.setAndMask(maskVal)
2662 maskedImage = exposure.getMaskedImage()
2663 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2664 skyLevel = stats.getValue(afwMath.MEDIAN)
2665 skySigma = stats.getValue(afwMath.STDEVCLIP)
2666 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2667 metadata = exposure.getMetadata()
2668 metadata[
"SKYLEVEL"] = skyLevel
2669 metadata[
"SKYSIGMA"] = skySigma
2672 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2673 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2674 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2675 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2676 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2677 skyLevels = numpy.zeros((nX, nY))
2680 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2682 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2684 xLLC = xc - meshXHalf
2685 yLLC = yc - meshYHalf
2686 xURC = xc + meshXHalf - 1
2687 yURC = yc + meshYHalf - 1
2690 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2692 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2694 good = numpy.where(numpy.isfinite(skyLevels))
2695 skyMedian = numpy.median(skyLevels[good])
2696 flatness = (skyLevels[good] - skyMedian) / skyMedian
2697 flatness_rms = numpy.std(flatness)
2698 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2700 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2701 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2702 nX, nY, flatness_pp, flatness_rms)
2704 metadata[
"FLATNESS_PP"] = float(flatness_pp)
2705 metadata[
"FLATNESS_RMS"] = float(flatness_rms)
2706 metadata[
"FLATNESS_NGRIDS"] =
'%dx%d' % (nX, nY)
2707 metadata[
"FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2708 metadata[
"FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2711 """Set an approximate magnitude zero point for the exposure.
2716 Exposure to process.
2718 filterLabel = exposure.getFilter()
2719 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2721 if physicalFilter
in self.config.fluxMag0T1:
2722 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2724 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2725 fluxMag0 = self.config.defaultFluxMag0T1
2727 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2729 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2732 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2733 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2734 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2738 """Context manager that applies and removes flats and darks,
2739 if the task
is configured to apply them.
2744 Exposure to process.
2746 Flat exposure the same size
as ``exp``.
2748 Dark exposure the same size
as ``exp``.
2753 The flat
and dark corrected exposure.
2755 if self.config.doDark
and dark
is not None:
2757 if self.config.doFlat:
2762 if self.config.doFlat:
2764 if self.config.doDark
and dark
is not None:
2768 """Utility function to examine ISR exposure at different stages.
2775 State of processing to view.
2777 frame = getDebugFrame(self._display, stepname)
2779 display = getDisplay(frame)
2780 display.scale(
'asinh',
'zscale')
2781 display.mtv(exposure)
2782 prompt =
"Press Enter to continue [c]... "
2784 ans = input(prompt).lower()
2785 if ans
in (
"",
"c",):
2790 """A Detector-like object that supports returning gain and saturation level
2792 This is used when the input exposure does
not have a detector.
2797 Exposure to generate a fake amplifier
for.
2798 config : `lsst.ip.isr.isrTaskConfig`
2799 Configuration to apply to the fake amplifier.
2803 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2805 self.
_gain = config.gain
2832 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2836 """Task to wrap the default IsrTask to allow it to be retargeted.
2838 The standard IsrTask can be called directly from a command line
2839 program, but doing so removes the ability of the task to be
2840 retargeted. As most cameras override some set of the IsrTask
2841 methods, this would remove those data-specific methods
in the
2842 output post-ISR images. This wrapping
class fixes the issue,
2843 allowing identical post-ISR images to be generated by both the
2844 processCcd
and isrTask code.
2846 ConfigClass = RunIsrConfig
2847 _DefaultName = "runIsr"
2851 self.makeSubtask(
"isr")
2857 dataRef : `lsst.daf.persistence.ButlerDataRef`
2858 data reference of the detector data to be processed
2862 result : `pipeBase.Struct`
2863 Result struct with component:
2866 Post-ISR processed exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def readIsrData(self, dataRef, rawExposure)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, deferredCharge=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.