29import lsst.pipe.base
as pipeBase
30import lsst.pipe.base.connectionTypes
as cT
32from contextlib
import contextmanager
33from lsstDebug
import getDebugFrame
38from lsst.utils.timer
import timeMethod
40from .
import isrFunctions
42from .
import linearize
43from .defects
import Defects
45from .assembleCcdTask
import AssembleCcdTask
46from .crosstalk
import CrosstalkTask, CrosstalkCalib
47from .fringe
import FringeTask
48from .isr
import maskNans
49from .masking
import MaskingTask
50from .overscan
import OverscanCorrectionTask
51from .straylight
import StrayLightTask
52from .vignette
import VignetteTask
53from .ampOffset
import AmpOffsetTask
54from .deferredCharge
import DeferredChargeTask
55from .isrStatistics
import IsrStatisticsTask
56from lsst.daf.butler
import DimensionGraph
59__all__ = [
"IsrTask",
"IsrTaskConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
73 registry : `lsst.daf.butler.Registry`
74 Butler registry to query.
75 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
76 Data id to transform to identify crosstalkSources. The
77 ``detector`` entry will be stripped.
78 collections : `lsst.daf.butler.CollectionSearch`
79 Collections to search through.
83 results : `list` [`lsst.daf.butler.DatasetRef`]
84 List of datasets that match the query that will be used
as
87 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"exposure"]))
88 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
95 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records))
for ref
in results]
99 dimensions={
"instrument",
"exposure",
"detector"},
100 defaultTemplates={}):
101 ccdExposure = cT.Input(
103 doc=
"Input exposure to process.",
104 storageClass=
"Exposure",
105 dimensions=[
"instrument",
"exposure",
"detector"],
107 camera = cT.PrerequisiteInput(
109 storageClass=
"Camera",
110 doc=
"Input camera to construct complete exposures.",
111 dimensions=[
"instrument"],
115 crosstalk = cT.PrerequisiteInput(
117 doc=
"Input crosstalk object",
118 storageClass=
"CrosstalkCalib",
119 dimensions=[
"instrument",
"detector"],
123 crosstalkSources = cT.PrerequisiteInput(
124 name=
"isrOverscanCorrected",
125 doc=
"Overscan corrected input images.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
130 lookupFunction=crosstalkSourceLookup,
133 bias = cT.PrerequisiteInput(
135 doc=
"Input bias calibration.",
136 storageClass=
"ExposureF",
137 dimensions=[
"instrument",
"detector"],
140 dark = cT.PrerequisiteInput(
142 doc=
"Input dark calibration.",
143 storageClass=
"ExposureF",
144 dimensions=[
"instrument",
"detector"],
147 flat = cT.PrerequisiteInput(
149 doc=
"Input flat calibration.",
150 storageClass=
"ExposureF",
151 dimensions=[
"instrument",
"physical_filter",
"detector"],
154 ptc = cT.PrerequisiteInput(
156 doc=
"Input Photon Transfer Curve dataset",
157 storageClass=
"PhotonTransferCurveDataset",
158 dimensions=[
"instrument",
"detector"],
161 fringes = cT.PrerequisiteInput(
163 doc=
"Input fringe calibration.",
164 storageClass=
"ExposureF",
165 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 strayLightData = cT.PrerequisiteInput(
171 doc=
"Input stray light calibration.",
172 storageClass=
"StrayLightData",
173 dimensions=[
"instrument",
"physical_filter",
"detector"],
178 bfKernel = cT.PrerequisiteInput(
180 doc=
"Input brighter-fatter kernel.",
181 storageClass=
"NumpyArray",
182 dimensions=[
"instrument"],
186 newBFKernel = cT.PrerequisiteInput(
187 name=
'brighterFatterKernel',
188 doc=
"Newer complete kernel + gain solutions.",
189 storageClass=
"BrighterFatterKernel",
190 dimensions=[
"instrument",
"detector"],
194 defects = cT.PrerequisiteInput(
196 doc=
"Input defect tables.",
197 storageClass=
"Defects",
198 dimensions=[
"instrument",
"detector"],
201 linearizer = cT.PrerequisiteInput(
203 storageClass=
"Linearizer",
204 doc=
"Linearity correction calibration.",
205 dimensions=[
"instrument",
"detector"],
209 opticsTransmission = cT.PrerequisiteInput(
210 name=
"transmission_optics",
211 storageClass=
"TransmissionCurve",
212 doc=
"Transmission curve due to the optics.",
213 dimensions=[
"instrument"],
216 filterTransmission = cT.PrerequisiteInput(
217 name=
"transmission_filter",
218 storageClass=
"TransmissionCurve",
219 doc=
"Transmission curve due to the filter.",
220 dimensions=[
"instrument",
"physical_filter"],
223 sensorTransmission = cT.PrerequisiteInput(
224 name=
"transmission_sensor",
225 storageClass=
"TransmissionCurve",
226 doc=
"Transmission curve due to the sensor.",
227 dimensions=[
"instrument",
"detector"],
230 atmosphereTransmission = cT.PrerequisiteInput(
231 name=
"transmission_atmosphere",
232 storageClass=
"TransmissionCurve",
233 doc=
"Transmission curve due to the atmosphere.",
234 dimensions=[
"instrument"],
237 illumMaskedImage = cT.PrerequisiteInput(
239 doc=
"Input illumination correction.",
240 storageClass=
"MaskedImageF",
241 dimensions=[
"instrument",
"physical_filter",
"detector"],
244 deferredChargeCalib = cT.PrerequisiteInput(
245 name=
"deferredCharge",
246 doc=
"Deferred charge/CTI correction dataset.",
247 storageClass=
"IsrCalib",
248 dimensions=[
"instrument",
"detector"],
252 outputExposure = cT.Output(
254 doc=
"Output ISR processed exposure.",
255 storageClass=
"Exposure",
256 dimensions=[
"instrument",
"exposure",
"detector"],
258 preInterpExposure = cT.Output(
259 name=
'preInterpISRCCD',
260 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
261 storageClass=
"ExposureF",
262 dimensions=[
"instrument",
"exposure",
"detector"],
264 outputOssThumbnail = cT.Output(
266 doc=
"Output Overscan-subtracted thumbnail image.",
267 storageClass=
"Thumbnail",
268 dimensions=[
"instrument",
"exposure",
"detector"],
270 outputFlattenedThumbnail = cT.Output(
271 name=
"FlattenedThumb",
272 doc=
"Output flat-corrected thumbnail image.",
273 storageClass=
"Thumbnail",
274 dimensions=[
"instrument",
"exposure",
"detector"],
276 outputStatistics = cT.Output(
277 name=
"isrStatistics",
278 doc=
"Output of additional statistics table.",
279 storageClass=
"StructuredDataDict",
280 dimensions=[
"instrument",
"exposure",
"detector"],
286 if config.doBias
is not True:
287 self.prerequisiteInputs.remove(
"bias")
288 if config.doLinearize
is not True:
289 self.prerequisiteInputs.remove(
"linearizer")
290 if config.doCrosstalk
is not True:
291 self.prerequisiteInputs.remove(
"crosstalkSources")
292 self.prerequisiteInputs.remove(
"crosstalk")
293 if config.doBrighterFatter
is not True:
294 self.prerequisiteInputs.remove(
"bfKernel")
295 self.prerequisiteInputs.remove(
"newBFKernel")
296 if config.doDefect
is not True:
297 self.prerequisiteInputs.remove(
"defects")
298 if config.doDark
is not True:
299 self.prerequisiteInputs.remove(
"dark")
300 if config.doFlat
is not True:
301 self.prerequisiteInputs.remove(
"flat")
302 if config.doFringe
is not True:
303 self.prerequisiteInputs.remove(
"fringes")
304 if config.doStrayLight
is not True:
305 self.prerequisiteInputs.remove(
"strayLightData")
306 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
307 self.prerequisiteInputs.remove(
"ptc")
308 if config.doAttachTransmissionCurve
is not True:
309 self.prerequisiteInputs.remove(
"opticsTransmission")
310 self.prerequisiteInputs.remove(
"filterTransmission")
311 self.prerequisiteInputs.remove(
"sensorTransmission")
312 self.prerequisiteInputs.remove(
"atmosphereTransmission")
314 if config.doUseOpticsTransmission
is not True:
315 self.prerequisiteInputs.remove(
"opticsTransmission")
316 if config.doUseFilterTransmission
is not True:
317 self.prerequisiteInputs.remove(
"filterTransmission")
318 if config.doUseSensorTransmission
is not True:
319 self.prerequisiteInputs.remove(
"sensorTransmission")
320 if config.doUseAtmosphereTransmission
is not True:
321 self.prerequisiteInputs.remove(
"atmosphereTransmission")
322 if config.doIlluminationCorrection
is not True:
323 self.prerequisiteInputs.remove(
"illumMaskedImage")
324 if config.doDeferredCharge
is not True:
325 self.prerequisiteInputs.remove(
"deferredChargeCalib")
327 if config.doWrite
is not True:
328 self.outputs.remove(
"outputExposure")
329 self.outputs.remove(
"preInterpExposure")
330 self.outputs.remove(
"outputFlattenedThumbnail")
331 self.outputs.remove(
"outputOssThumbnail")
332 self.outputs.remove(
"outputStatistics")
334 if config.doSaveInterpPixels
is not True:
335 self.outputs.remove(
"preInterpExposure")
336 if config.qa.doThumbnailOss
is not True:
337 self.outputs.remove(
"outputOssThumbnail")
338 if config.qa.doThumbnailFlattened
is not True:
339 self.outputs.remove(
"outputFlattenedThumbnail")
340 if config.doCalculateStatistics
is not True:
341 self.outputs.remove(
"outputStatistics")
345 pipelineConnections=IsrTaskConnections):
346 """Configuration parameters for IsrTask.
348 Items are grouped in the order
in which they are executed by the task.
350 datasetType = pexConfig.Field(
352 doc="Dataset type for input data; users will typically leave this alone, "
353 "but camera-specific ISR tasks will override it",
357 fallbackFilterName = pexConfig.Field(
359 doc=
"Fallback default filter name for calibrations.",
362 useFallbackDate = pexConfig.Field(
364 doc=
"Pass observation date when using fallback filter.",
367 expectWcs = pexConfig.Field(
370 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
372 fwhm = pexConfig.Field(
374 doc=
"FWHM of PSF in arcseconds.",
377 qa = pexConfig.ConfigField(
379 doc=
"QA related configuration options.",
383 doConvertIntToFloat = pexConfig.Field(
385 doc=
"Convert integer raw images to floating point values?",
390 doSaturation = pexConfig.Field(
392 doc=
"Mask saturated pixels? NB: this is totally independent of the"
393 " interpolation option - this is ONLY setting the bits in the mask."
394 " To have them interpolated make sure doSaturationInterpolation=True",
397 saturatedMaskName = pexConfig.Field(
399 doc=
"Name of mask plane to use in saturation detection and interpolation",
402 saturation = pexConfig.Field(
404 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
405 default=float(
"NaN"),
407 growSaturationFootprintSize = pexConfig.Field(
409 doc=
"Number of pixels by which to grow the saturation footprints",
414 doSuspect = pexConfig.Field(
416 doc=
"Mask suspect pixels?",
419 suspectMaskName = pexConfig.Field(
421 doc=
"Name of mask plane to use for suspect pixels",
424 numEdgeSuspect = pexConfig.Field(
426 doc=
"Number of edge pixels to be flagged as untrustworthy.",
429 edgeMaskLevel = pexConfig.ChoiceField(
431 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
434 'DETECTOR':
'Mask only the edges of the full detector.',
435 'AMP':
'Mask edges of each amplifier.',
440 doSetBadRegions = pexConfig.Field(
442 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
445 badStatistic = pexConfig.ChoiceField(
447 doc=
"How to estimate the average value for BAD regions.",
450 "MEANCLIP":
"Correct using the (clipped) mean of good data",
451 "MEDIAN":
"Correct using the median of the good data",
456 doOverscan = pexConfig.Field(
458 doc=
"Do overscan subtraction?",
461 overscan = pexConfig.ConfigurableField(
462 target=OverscanCorrectionTask,
463 doc=
"Overscan subtraction task for image segments.",
467 doAssembleCcd = pexConfig.Field(
470 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
472 assembleCcd = pexConfig.ConfigurableField(
473 target=AssembleCcdTask,
474 doc=
"CCD assembly task",
478 doAssembleIsrExposures = pexConfig.Field(
481 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
483 doTrimToMatchCalib = pexConfig.Field(
486 doc=
"Trim raw data to match calibration bounding boxes?"
490 doBias = pexConfig.Field(
492 doc=
"Apply bias frame correction?",
495 biasDataProductName = pexConfig.Field(
497 doc=
"Name of the bias data product",
500 doBiasBeforeOverscan = pexConfig.Field(
502 doc=
"Reverse order of overscan and bias correction.",
507 doDeferredCharge = pexConfig.Field(
509 doc=
"Apply deferred charge correction?",
512 deferredChargeCorrection = pexConfig.ConfigurableField(
513 target=DeferredChargeTask,
514 doc=
"Deferred charge correction task.",
518 doVariance = pexConfig.Field(
520 doc=
"Calculate variance?",
523 gain = pexConfig.Field(
525 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
526 default=float(
"NaN"),
528 readNoise = pexConfig.Field(
530 doc=
"The read noise to use if no Detector is present in the Exposure",
533 doEmpiricalReadNoise = pexConfig.Field(
536 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
538 usePtcReadNoise = pexConfig.Field(
541 doc=
"Use readnoise values from the Photon Transfer Curve?"
543 maskNegativeVariance = pexConfig.Field(
546 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
547 "in the measurement of the overscan at an edge due to the data falling off faster "
548 "than the overscan model can account for it."
550 negativeVarianceMaskName = pexConfig.Field(
553 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
556 doLinearize = pexConfig.Field(
558 doc=
"Correct for nonlinearity of the detector's response?",
563 doCrosstalk = pexConfig.Field(
565 doc=
"Apply intra-CCD crosstalk correction?",
568 doCrosstalkBeforeAssemble = pexConfig.Field(
570 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
573 crosstalk = pexConfig.ConfigurableField(
574 target=CrosstalkTask,
575 doc=
"Intra-CCD crosstalk correction",
579 doDefect = pexConfig.Field(
581 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
584 doNanMasking = pexConfig.Field(
586 doc=
"Mask non-finite (NAN, inf) pixels?",
589 doWidenSaturationTrails = pexConfig.Field(
591 doc=
"Widen bleed trails based on their width?",
596 doBrighterFatter = pexConfig.Field(
599 doc=
"Apply the brighter-fatter correction?"
601 brighterFatterLevel = pexConfig.ChoiceField(
604 doc=
"The level at which to correct for brighter-fatter.",
606 "AMP":
"Every amplifier treated separately.",
607 "DETECTOR":
"One kernel per detector",
610 brighterFatterMaxIter = pexConfig.Field(
613 doc=
"Maximum number of iterations for the brighter-fatter correction"
615 brighterFatterThreshold = pexConfig.Field(
618 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
619 "absolute value of the difference between the current corrected image and the one "
620 "from the previous iteration summed over all the pixels."
622 brighterFatterApplyGain = pexConfig.Field(
625 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
627 brighterFatterMaskListToInterpolate = pexConfig.ListField(
629 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
631 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
633 brighterFatterMaskGrowSize = pexConfig.Field(
636 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
637 "when brighter-fatter correction is applied."
641 doDark = pexConfig.Field(
643 doc=
"Apply dark frame correction?",
646 darkDataProductName = pexConfig.Field(
648 doc=
"Name of the dark data product",
653 doStrayLight = pexConfig.Field(
655 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
658 strayLight = pexConfig.ConfigurableField(
659 target=StrayLightTask,
660 doc=
"y-band stray light correction"
664 doFlat = pexConfig.Field(
666 doc=
"Apply flat field correction?",
669 flatDataProductName = pexConfig.Field(
671 doc=
"Name of the flat data product",
674 flatScalingType = pexConfig.ChoiceField(
676 doc=
"The method for scaling the flat on the fly.",
679 "USER":
"Scale by flatUserScale",
680 "MEAN":
"Scale by the inverse of the mean",
681 "MEDIAN":
"Scale by the inverse of the median",
684 flatUserScale = pexConfig.Field(
686 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
689 doTweakFlat = pexConfig.Field(
691 doc=
"Tweak flats to match observed amplifier ratios?",
697 doApplyGains = pexConfig.Field(
699 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
702 usePtcGains = pexConfig.Field(
704 doc=
"Use the gain values from the Photon Transfer Curve?",
707 normalizeGains = pexConfig.Field(
709 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
714 doFringe = pexConfig.Field(
716 doc=
"Apply fringe correction?",
719 fringe = pexConfig.ConfigurableField(
721 doc=
"Fringe subtraction task",
723 fringeAfterFlat = pexConfig.Field(
725 doc=
"Do fringe subtraction after flat-fielding?",
730 doAmpOffset = pexConfig.Field(
731 doc=
"Calculate and apply amp offset corrections?",
735 ampOffset = pexConfig.ConfigurableField(
736 doc=
"Amp offset correction task.",
737 target=AmpOffsetTask,
741 doMeasureBackground = pexConfig.Field(
743 doc=
"Measure the background level on the reduced image?",
748 doCameraSpecificMasking = pexConfig.Field(
750 doc=
"Mask camera-specific bad regions?",
753 masking = pexConfig.ConfigurableField(
759 doInterpolate = pexConfig.Field(
761 doc=
"Interpolate masked pixels?",
764 doSaturationInterpolation = pexConfig.Field(
766 doc=
"Perform interpolation over pixels masked as saturated?"
767 " NB: This is independent of doSaturation; if that is False this plane"
768 " will likely be blank, resulting in a no-op here.",
771 doNanInterpolation = pexConfig.Field(
773 doc=
"Perform interpolation over pixels masked as NaN?"
774 " NB: This is independent of doNanMasking; if that is False this plane"
775 " will likely be blank, resulting in a no-op here.",
778 doNanInterpAfterFlat = pexConfig.Field(
780 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
781 "also have to interpolate them before flat-fielding."),
784 maskListToInterpolate = pexConfig.ListField(
786 doc=
"List of mask planes that should be interpolated.",
787 default=[
'SAT',
'BAD'],
789 doSaveInterpPixels = pexConfig.Field(
791 doc=
"Save a copy of the pre-interpolated pixel values?",
796 fluxMag0T1 = pexConfig.DictField(
799 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
800 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
803 defaultFluxMag0T1 = pexConfig.Field(
805 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
806 default=pow(10.0, 0.4*28.0)
810 doVignette = pexConfig.Field(
812 doc=(
"Compute and attach the validPolygon defining the unvignetted region to the exposure "
813 "according to vignetting parameters?"),
816 doMaskVignettePolygon = pexConfig.Field(
818 doc=(
"Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
822 vignetteValue = pexConfig.Field(
824 doc=
"Value to replace image array pixels with in the vignetted region? Ignored if None.",
828 vignette = pexConfig.ConfigurableField(
830 doc=
"Vignetting task.",
834 doAttachTransmissionCurve = pexConfig.Field(
837 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
839 doUseOpticsTransmission = pexConfig.Field(
842 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
844 doUseFilterTransmission = pexConfig.Field(
847 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
849 doUseSensorTransmission = pexConfig.Field(
852 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
854 doUseAtmosphereTransmission = pexConfig.Field(
857 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
861 doIlluminationCorrection = pexConfig.Field(
864 doc=
"Perform illumination correction?"
866 illuminationCorrectionDataProductName = pexConfig.Field(
868 doc=
"Name of the illumination correction data product.",
871 illumScale = pexConfig.Field(
873 doc=
"Scale factor for the illumination correction.",
876 illumFilters = pexConfig.ListField(
879 doc=
"Only perform illumination correction for these filters."
883 doCalculateStatistics = pexConfig.Field(
885 doc=
"Should additional ISR statistics be calculated?",
888 isrStats = pexConfig.ConfigurableField(
889 target=IsrStatisticsTask,
890 doc=
"Task to calculate additional statistics.",
895 doWrite = pexConfig.Field(
897 doc=
"Persist postISRCCD?",
904 raise ValueError(
"You may not specify both doFlat and doApplyGains")
906 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
916 """Apply common instrument signature correction algorithms to a raw frame.
918 The process for correcting imaging data
is very similar
from
919 camera to camera. This task provides a vanilla implementation of
920 doing these corrections, including the ability to turn certain
921 corrections off
if they are
not needed. The inputs to the primary
922 method, `
run()`, are a raw exposure to be corrected
and the
923 calibration data products. The raw input
is a single chip sized
924 mosaic of all amps including overscans
and other non-science
927 The __init__ method sets up the subtasks
for ISR processing, using
933 Positional arguments passed to the Task constructor.
934 None used at this time.
935 kwargs : `dict`, optional
936 Keyword arguments passed on to the Task constructor.
937 None used at this time.
939 ConfigClass = IsrTaskConfig
944 self.makeSubtask(
"assembleCcd")
945 self.makeSubtask(
"crosstalk")
946 self.makeSubtask(
"strayLight")
947 self.makeSubtask(
"fringe")
948 self.makeSubtask(
"masking")
949 self.makeSubtask(
"overscan")
950 self.makeSubtask(
"vignette")
951 self.makeSubtask(
"ampOffset")
952 self.makeSubtask(
"deferredChargeCorrection")
953 self.makeSubtask(
"isrStats")
956 inputs = butlerQC.get(inputRefs)
959 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
960 except Exception
as e:
961 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
964 detector = inputs[
'ccdExposure'].getDetector()
966 if self.config.doCrosstalk
is True:
969 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
970 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
971 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
973 coeffVector = (self.config.crosstalk.crosstalkValues
974 if self.config.crosstalk.useConfigCoefficients
else None)
975 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
976 inputs[
'crosstalk'] = crosstalkCalib
977 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
978 if 'crosstalkSources' not in inputs:
979 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
982 if 'linearizer' in inputs:
983 if isinstance(inputs[
'linearizer'], dict):
985 linearizer.fromYaml(inputs[
'linearizer'])
986 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
987 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
991 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
993 linearizer = inputs[
'linearizer']
994 linearizer.log = self.log
995 inputs[
'linearizer'] = linearizer
998 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1000 if self.config.doDefect
is True:
1001 if "defects" in inputs
and inputs[
'defects']
is not None:
1005 if not isinstance(inputs[
"defects"], Defects):
1006 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1010 if self.config.doBrighterFatter:
1011 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1012 if brighterFatterKernel
is None:
1013 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1015 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1017 detName = detector.getName()
1018 level = brighterFatterKernel.level
1021 inputs[
'bfGains'] = brighterFatterKernel.gain
1022 if self.config.brighterFatterLevel ==
'DETECTOR':
1023 if level ==
'DETECTOR':
1024 if detName
in brighterFatterKernel.detKernels:
1025 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1027 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1028 elif level ==
'AMP':
1029 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1031 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1032 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1033 elif self.config.brighterFatterLevel ==
'AMP':
1034 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1036 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
1037 expId = inputs[
'ccdExposure'].info.id
1038 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1040 assembler=self.assembleCcd
1041 if self.config.doAssembleIsrExposures
else None)
1043 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1045 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
1046 if 'strayLightData' not in inputs:
1047 inputs[
'strayLightData'] =
None
1049 outputs = self.
run(**inputs)
1050 butlerQC.put(outputs, outputRefs)
1053 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1054 crosstalk=None, crosstalkSources=None,
1055 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1056 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1057 sensorTransmission=
None, atmosphereTransmission=
None,
1058 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1059 deferredCharge=
None,
1061 """Perform instrument signature removal on an exposure.
1063 Steps included in the ISR processing,
in order performed, are:
1064 - saturation
and suspect pixel masking
1065 - overscan subtraction
1066 - CCD assembly of individual amplifiers
1068 - variance image construction
1069 - linearization of non-linear response
1071 - brighter-fatter correction
1074 - stray light subtraction
1076 - masking of known defects
and camera specific features
1077 - vignette calculation
1078 - appending transmission curve
and distortion model
1083 The raw exposure that
is to be run through ISR. The
1084 exposure
is modified by this method.
1086 The camera geometry
for this exposure. Required
if
1087 one
or more of ``ccdExposure``, ``bias``, ``dark``,
or
1088 ``flat`` does
not have an associated detector.
1090 Bias calibration frame.
1092 Functor
for linearization.
1094 Calibration
for crosstalk.
1095 crosstalkSources : `list`, optional
1096 List of possible crosstalk sources.
1098 Dark calibration frame.
1100 Flat calibration frame.
1102 Photon transfer curve dataset,
with, e.g., gains
1104 bfKernel : `numpy.ndarray`, optional
1105 Brighter-fatter kernel.
1106 bfGains : `dict` of `float`, optional
1107 Gains used to override the detector
's nominal gains for the
1108 brighter-fatter correction. A dict keyed by amplifier name for
1109 the detector
in question.
1112 fringes : `lsst.pipe.base.Struct`, optional
1113 Struct containing the fringe correction data,
with
1116 - ``seed``: random seed derived
from the ccdExposureId
for random
1117 number generator (`uint32`)
1119 A ``TransmissionCurve`` that represents the throughput of the,
1120 optics, to be evaluated
in focal-plane coordinates.
1122 A ``TransmissionCurve`` that represents the throughput of the
1123 filter itself, to be evaluated
in focal-plane coordinates.
1125 A ``TransmissionCurve`` that represents the throughput of the
1126 sensor itself, to be evaluated
in post-assembly trimmed detector
1129 A ``TransmissionCurve`` that represents the throughput of the
1130 atmosphere, assumed to be spatially constant.
1131 detectorNum : `int`, optional
1132 The integer number
for the detector to process.
1133 strayLightData : `object`, optional
1134 Opaque object containing calibration information
for stray-light
1135 correction. If `
None`, no correction will be performed.
1137 Illumination correction image.
1141 result : `lsst.pipe.base.Struct`
1142 Result struct
with component:
1144 The fully ISR corrected exposure.
1146 An alias
for `exposure`
1147 - ``ossThumb`` : `numpy.ndarray`
1148 Thumbnail image of the exposure after overscan subtraction.
1149 - ``flattenedThumb`` : `numpy.ndarray`
1150 Thumbnail image of the exposure after flat-field correction.
1151 - ``outputStatistics`` : ``
1152 Values of the additional statistics calculated.
1157 Raised
if a configuration option
is set to
True, but the
1158 required calibration data has
not been specified.
1162 The current processed exposure can be viewed by setting the
1163 appropriate lsstDebug entries
in the `debug.display`
1164 dictionary. The names of these entries correspond to some of
1165 the IsrTaskConfig Boolean options,
with the value denoting the
1166 frame to use. The exposure
is shown inside the matching
1167 option check
and after the processing of that step has
1168 finished. The steps
with debug points are:
1179 In addition, setting the
"postISRCCD" entry displays the
1180 exposure after all ISR processing has finished.
1184 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1189 ccd = ccdExposure.getDetector()
1190 filterLabel = ccdExposure.getFilter()
1191 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1194 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1195 ccd = [
FakeAmp(ccdExposure, self.config)]
1198 if self.config.doBias
and bias
is None:
1199 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1201 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1202 if self.config.doBrighterFatter
and bfKernel
is None:
1203 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1204 if self.config.doDark
and dark
is None:
1205 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1206 if self.config.doFlat
and flat
is None:
1207 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1208 if self.config.doDefect
and defects
is None:
1209 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1210 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1211 and fringes.fringes
is None):
1216 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1217 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1218 and illumMaskedImage
is None):
1219 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1220 if (self.config.doDeferredCharge
and deferredCharge
is None):
1221 raise RuntimeError(
"Must supply a deferred charge calibration if config.doDeferredCharge=True.")
1224 if self.config.doConvertIntToFloat:
1225 self.log.info(
"Converting exposure to floating point values.")
1228 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1229 self.log.info(
"Applying bias correction.")
1230 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1231 trimToFit=self.config.doTrimToMatchCalib)
1239 if ccdExposure.getBBox().contains(amp.getBBox()):
1244 if self.config.doOverscan
and not badAmp:
1247 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1248 if overscanResults
is not None and \
1249 self.config.qa
is not None and self.config.qa.saveStats
is True:
1251 self.metadata[f
"FIT MEDIAN {amp.getName()}"] = overscanResults.overscanMean
1252 self.metadata[f
"FIT STDEV {amp.getName()}"] = overscanResults.overscanSigma
1253 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1254 amp.getName(), overscanResults.overscanMean,
1255 overscanResults.overscanSigma)
1257 self.metadata[f
"RESIDUAL MEDIAN {amp.getName()}"] = overscanResults.residualMean
1258 self.metadata[f
"RESIDUAL STDEV {amp.getName()}"] = overscanResults.residualSigma
1259 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1260 amp.getName(), overscanResults.residualMean,
1261 overscanResults.residualSigma)
1263 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1266 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1267 overscanResults =
None
1269 overscans.append(overscanResults
if overscanResults
is not None else None)
1271 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1273 if self.config.doDeferredCharge:
1274 self.log.info(
"Applying deferred charge/CTI correction.")
1275 self.deferredChargeCorrection.
run(ccdExposure, deferredCharge)
1276 self.
debugView(ccdExposure,
"doDeferredCharge")
1278 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1279 self.log.info(
"Applying crosstalk correction.")
1280 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1281 crosstalkSources=crosstalkSources, camera=camera)
1282 self.
debugView(ccdExposure,
"doCrosstalk")
1284 if self.config.doAssembleCcd:
1285 self.log.info(
"Assembling CCD from amplifiers.")
1286 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1288 if self.config.expectWcs
and not ccdExposure.getWcs():
1289 self.log.warning(
"No WCS found in input exposure.")
1290 self.
debugView(ccdExposure,
"doAssembleCcd")
1293 if self.config.qa.doThumbnailOss:
1294 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1296 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1297 self.log.info(
"Applying bias correction.")
1298 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1299 trimToFit=self.config.doTrimToMatchCalib)
1302 if self.config.doVariance:
1303 for amp, overscanResults
in zip(ccd, overscans):
1304 if ccdExposure.getBBox().contains(amp.getBBox()):
1305 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1306 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1307 if overscanResults
is not None:
1309 overscanImage=overscanResults.overscanImage,
1315 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1316 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1317 afwMath.MEDIAN | afwMath.STDEVCLIP)
1318 self.metadata[f
"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1319 qaStats.getValue(afwMath.MEDIAN)
1320 self.metadata[f
"ISR VARIANCE {amp.getName()} STDEV"] = \
1321 qaStats.getValue(afwMath.STDEVCLIP)
1322 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1323 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1324 qaStats.getValue(afwMath.STDEVCLIP))
1325 if self.config.maskNegativeVariance:
1329 self.log.info(
"Applying linearizer.")
1330 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1331 detector=ccd, log=self.log)
1333 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1334 self.log.info(
"Applying crosstalk correction.")
1335 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1336 crosstalkSources=crosstalkSources, isTrimmed=
True)
1337 self.
debugView(ccdExposure,
"doCrosstalk")
1342 if self.config.doDefect:
1343 self.log.info(
"Masking defects.")
1346 if self.config.numEdgeSuspect > 0:
1347 self.log.info(
"Masking edges as SUSPECT.")
1348 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1349 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1351 if self.config.doNanMasking:
1352 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1355 if self.config.doWidenSaturationTrails:
1356 self.log.info(
"Widening saturation trails.")
1357 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1359 if self.config.doCameraSpecificMasking:
1360 self.log.info(
"Masking regions for camera specific reasons.")
1361 self.masking.
run(ccdExposure)
1363 if self.config.doBrighterFatter:
1373 interpExp = ccdExposure.clone()
1375 isrFunctions.interpolateFromMask(
1376 maskedImage=interpExp.getMaskedImage(),
1377 fwhm=self.config.fwhm,
1378 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1379 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1381 bfExp = interpExp.clone()
1383 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1384 type(bfKernel), type(bfGains))
1385 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1386 self.config.brighterFatterMaxIter,
1387 self.config.brighterFatterThreshold,
1388 self.config.brighterFatterApplyGain,
1390 if bfResults[1] == self.config.brighterFatterMaxIter:
1391 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1394 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1396 image = ccdExposure.getMaskedImage().getImage()
1397 bfCorr = bfExp.getMaskedImage().getImage()
1398 bfCorr -= interpExp.getMaskedImage().getImage()
1407 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1408 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1411 if self.config.brighterFatterMaskGrowSize > 0:
1412 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1413 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1414 isrFunctions.growMasks(ccdExposure.getMask(),
1415 radius=self.config.brighterFatterMaskGrowSize,
1416 maskNameList=maskPlane,
1417 maskValue=maskPlane)
1419 self.
debugView(ccdExposure,
"doBrighterFatter")
1421 if self.config.doDark:
1422 self.log.info(
"Applying dark correction.")
1426 if self.config.doFringe
and not self.config.fringeAfterFlat:
1427 self.log.info(
"Applying fringe correction before flat.")
1428 self.fringe.
run(ccdExposure, **fringes.getDict())
1431 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1432 self.log.info(
"Checking strayLight correction.")
1433 self.strayLight.
run(ccdExposure, strayLightData)
1434 self.
debugView(ccdExposure,
"doStrayLight")
1436 if self.config.doFlat:
1437 self.log.info(
"Applying flat correction.")
1441 if self.config.doApplyGains:
1442 self.log.info(
"Applying gain correction instead of flat.")
1443 if self.config.usePtcGains:
1444 self.log.info(
"Using gains from the Photon Transfer Curve.")
1445 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1448 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1450 if self.config.doFringe
and self.config.fringeAfterFlat:
1451 self.log.info(
"Applying fringe correction after flat.")
1452 self.fringe.
run(ccdExposure, **fringes.getDict())
1454 if self.config.doVignette:
1455 if self.config.doMaskVignettePolygon:
1456 self.log.info(
"Constructing, attaching, and masking vignette polygon.")
1458 self.log.info(
"Constructing and attaching vignette polygon.")
1460 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1461 vignetteValue=self.config.vignetteValue, log=self.log)
1463 if self.config.doAttachTransmissionCurve:
1464 self.log.info(
"Adding transmission curves.")
1465 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1466 filterTransmission=filterTransmission,
1467 sensorTransmission=sensorTransmission,
1468 atmosphereTransmission=atmosphereTransmission)
1470 flattenedThumb =
None
1471 if self.config.qa.doThumbnailFlattened:
1472 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1474 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1475 self.log.info(
"Performing illumination correction.")
1476 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1477 illumMaskedImage, illumScale=self.config.illumScale,
1478 trimToFit=self.config.doTrimToMatchCalib)
1481 if self.config.doSaveInterpPixels:
1482 preInterpExp = ccdExposure.clone()
1497 if self.config.doSetBadRegions:
1498 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1499 if badPixelCount > 0:
1500 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1502 if self.config.doInterpolate:
1503 self.log.info(
"Interpolating masked pixels.")
1504 isrFunctions.interpolateFromMask(
1505 maskedImage=ccdExposure.getMaskedImage(),
1506 fwhm=self.config.fwhm,
1507 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1508 maskNameList=list(self.config.maskListToInterpolate)
1514 if self.config.doAmpOffset:
1515 self.log.info(
"Correcting amp offsets.")
1516 self.ampOffset.
run(ccdExposure)
1518 if self.config.doMeasureBackground:
1519 self.log.info(
"Measuring background level.")
1522 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1524 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1525 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1526 afwMath.MEDIAN | afwMath.STDEVCLIP)
1527 self.metadata[f
"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1528 self.metadata[f
"ISR BACKGROUND {amp.getName()} STDEV"] = \
1529 qaStats.getValue(afwMath.STDEVCLIP)
1530 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1531 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1532 qaStats.getValue(afwMath.STDEVCLIP))
1535 outputStatistics =
None
1536 if self.config.doCalculateStatistics:
1537 outputStatistics = self.isrStats.
run(ccdExposure, overscanResults=overscans,
1540 self.
debugView(ccdExposure,
"postISRCCD")
1542 return pipeBase.Struct(
1543 exposure=ccdExposure,
1545 flattenedThumb=flattenedThumb,
1547 preInterpExposure=preInterpExp,
1548 outputExposure=ccdExposure,
1549 outputOssThumbnail=ossThumb,
1550 outputFlattenedThumbnail=flattenedThumb,
1551 outputStatistics=outputStatistics,
1555 """Ensure that the data returned by Butler is a fully constructed exp.
1557 ISR requires exposure-level image data for historical reasons, so
if we
1558 did
not recieve that
from Butler, construct it
from what we have,
1559 modifying the input
in place.
1564 or `lsst.afw.image.ImageF`
1565 The input data structure obtained
from Butler.
1566 camera : `lsst.afw.cameraGeom.camera`, optional
1567 The camera associated
with the image. Used to find the appropriate
1568 detector
if detector
is not already set.
1569 detectorNum : `int`, optional
1570 The detector
in the camera to attach,
if the detector
is not
1576 The re-constructed exposure,
with appropriate detector parameters.
1581 Raised
if the input data cannot be used to construct an exposure.
1583 if isinstance(inputExp, afwImage.DecoratedImageU):
1584 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1585 elif isinstance(inputExp, afwImage.ImageF):
1586 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1587 elif isinstance(inputExp, afwImage.MaskedImageF):
1588 inputExp = afwImage.makeExposure(inputExp)
1589 elif isinstance(inputExp, afwImage.Exposure):
1591 elif inputExp
is None:
1595 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1598 if inputExp.getDetector()
is None:
1599 if camera
is None or detectorNum
is None:
1600 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1601 'without a detector set.')
1602 inputExp.setDetector(camera[detectorNum])
1607 """Convert exposure image from uint16 to float.
1609 If the exposure does not need to be converted, the input
is
1610 immediately returned. For exposures that are converted to use
1611 floating point pixels, the variance
is set to unity
and the
1617 The raw exposure to be converted.
1622 The input ``exposure``, converted to floating point pixels.
1627 Raised
if the exposure type cannot be converted to float.
1630 if isinstance(exposure, afwImage.ExposureF):
1632 self.log.debug(
"Exposure already of type float.")
1634 if not hasattr(exposure,
"convertF"):
1635 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1637 newexposure = exposure.convertF()
1638 newexposure.variance[:] = 1
1639 newexposure.mask[:] = 0x0
1644 """Identify bad amplifiers, saturated and suspect pixels.
1649 Input exposure to be masked.
1651 Catalog of parameters defining the amplifier on this
1654 List of defects. Used to determine if the entire
1660 If this
is true, the entire amplifier area
is covered by
1661 defects
and unusable.
1664 maskedImage = ccdExposure.getMaskedImage()
1671 if defects
is not None:
1672 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1678 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1680 maskView = dataView.getMask()
1681 maskView |= maskView.getPlaneBitMask(
"BAD")
1689 if self.config.doSaturation
and not badAmp:
1690 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1691 if self.config.doSuspect
and not badAmp:
1692 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1693 if math.isfinite(self.config.saturation):
1694 limits.update({self.config.saturatedMaskName: self.config.saturation})
1696 for maskName, maskThreshold
in limits.items():
1697 if not math.isnan(maskThreshold):
1698 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1699 isrFunctions.makeThresholdMask(
1700 maskedImage=dataView,
1701 threshold=maskThreshold,
1708 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1710 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1711 self.config.suspectMaskName])
1712 if numpy.all(maskView.getArray() & maskVal > 0):
1714 maskView |= maskView.getPlaneBitMask(
"BAD")
1719 """Apply overscan correction in place.
1721 This method does initial pixel rejection of the overscan
1722 region. The overscan can also be optionally segmented to
1723 allow for discontinuous overscan responses to be fit
1724 separately. The actual overscan subtraction
is performed by
1725 the `lsst.ip.isr.overscan.OverscanTask`, which
is called here
1726 after the amplifier
is preprocessed.
1731 Exposure to have overscan correction performed.
1732 amp : `lsst.afw.cameraGeom.Amplifer`
1733 The amplifier to consider
while correcting the overscan.
1737 overscanResults : `lsst.pipe.base.Struct`
1738 Result struct
with components:
1740 Value
or fit subtracted
from the amplifier image data.
1742 Value
or fit subtracted
from the overscan image data.
1744 Image of the overscan region
with the overscan
1745 correction applied. This quantity
is used to estimate
1746 the amplifier read noise empirically.
1748 Mask of the suspect pixels.
1749 - ``overscanMean`` : `float`
1750 Median overscan fit value.
1751 - ``overscanSigma`` : `float`
1752 Clipped standard deviation of the overscan after
1758 Raised
if the ``amp`` does
not contain raw pixel information.
1762 lsst.ip.isr.overscan.OverscanTask
1765 if amp.getRawHorizontalOverscanBBox().isEmpty():
1766 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1770 overscanResults = self.overscan.
run(ccdExposure, amp)
1772 metadata = ccdExposure.getMetadata()
1773 ampNum = amp.getName()
1774 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanMean
1775 metadata[f
"ISR_OSCAN_SIGMA{ampNum}"] = overscanResults.overscanSigma
1777 return overscanResults
1780 """Set the variance plane using the gain and read noise
1782 The read noise is calculated
from the ``overscanImage``
if the
1783 ``doEmpiricalReadNoise`` option
is set
in the configuration; otherwise
1784 the value
from the amplifier data
is used.
1789 Exposure to process.
1791 Amplifier detector data.
1793 Image of overscan, required only
for empirical read noise.
1795 PTC dataset containing the gains
and read noise.
1800 Raised
if either ``usePtcGains`` of ``usePtcReadNoise``
1801 are ``
True``, but ptcDataset
is not provided.
1803 Raised
if ```doEmpiricalReadNoise``
is ``
True`` but
1804 ``overscanImage``
is ``
None``.
1808 lsst.ip.isr.isrFunctions.updateVariance
1810 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1811 if self.config.usePtcGains:
1812 if ptcDataset
is None:
1813 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
1815 gain = ptcDataset.gain[amp.getName()]
1816 self.log.info(
"Using gain from Photon Transfer Curve.")
1818 gain = amp.getGain()
1820 if math.isnan(gain):
1822 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1825 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
1826 amp.getName(), gain, patchedGain)
1829 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1830 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
1832 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1833 stats = afwMath.StatisticsControl()
1834 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1835 readNoise = afwMath.makeStatistics(overscanImage.getImage(),
1836 afwMath.STDEVCLIP, stats).getValue()
1837 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
1838 amp.getName(), readNoise)
1839 elif self.config.usePtcReadNoise:
1840 if ptcDataset
is None:
1841 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
1843 readNoise = ptcDataset.noise[amp.getName()]
1844 self.log.info(
"Using read noise from Photon Transfer Curve.")
1846 readNoise = amp.getReadNoise()
1848 isrFunctions.updateVariance(
1849 maskedImage=ampExposure.getMaskedImage(),
1851 readNoise=readNoise,
1855 """Identify and mask pixels with negative variance values.
1860 Exposure to process.
1864 lsst.ip.isr.isrFunctions.updateVariance
1866 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
1867 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
1868 exposure.mask.array[bad] |= maskPlane
1871 """Apply dark correction in place.
1876 Exposure to process.
1878 Dark exposure of the same size as ``exposure``.
1879 invert : `Bool`, optional
1880 If
True, re-add the dark to an already corrected image.
1885 Raised
if either ``exposure``
or ``darkExposure`` do
not
1886 have their dark time defined.
1890 lsst.ip.isr.isrFunctions.darkCorrection
1892 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1893 if math.isnan(expScale):
1894 raise RuntimeError(
"Exposure darktime is NAN.")
1895 if darkExposure.getInfo().getVisitInfo()
is not None \
1896 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
1897 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1901 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1904 isrFunctions.darkCorrection(
1905 maskedImage=exposure.getMaskedImage(),
1906 darkMaskedImage=darkExposure.getMaskedImage(),
1908 darkScale=darkScale,
1910 trimToFit=self.config.doTrimToMatchCalib
1914 """Check if linearization is needed for the detector cameraGeom.
1916 Checks config.doLinearize and the linearity type of the first
1922 Detector to get linearity type
from.
1926 doLinearize : `Bool`
1927 If
True, linearization should be performed.
1929 return self.config.doLinearize
and \
1930 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
1933 """Apply flat correction in place.
1938 Exposure to process.
1940 Flat exposure of the same size as ``exposure``.
1941 invert : `Bool`, optional
1942 If
True, unflatten an already flattened image.
1946 lsst.ip.isr.isrFunctions.flatCorrection
1948 isrFunctions.flatCorrection(
1949 maskedImage=exposure.getMaskedImage(),
1950 flatMaskedImage=flatExposure.getMaskedImage(),
1951 scalingType=self.config.flatScalingType,
1952 userScale=self.config.flatUserScale,
1954 trimToFit=self.config.doTrimToMatchCalib
1958 """Detect and mask saturated pixels in config.saturatedMaskName.
1963 Exposure to process. Only the amplifier DataSec is processed.
1965 Amplifier detector data.
1969 lsst.ip.isr.isrFunctions.makeThresholdMask
1971 if not math.isnan(amp.getSaturation()):
1972 maskedImage = exposure.getMaskedImage()
1973 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1974 isrFunctions.makeThresholdMask(
1975 maskedImage=dataView,
1976 threshold=amp.getSaturation(),
1978 maskName=self.config.saturatedMaskName,
1982 """Interpolate over saturated pixels, in place.
1984 This method should be called after `saturationDetection`, to
1985 ensure that the saturated pixels have been identified in the
1986 SAT mask. It should also be called after `assembleCcd`, since
1987 saturated regions may cross amplifier boundaries.
1992 Exposure to process.
1996 lsst.ip.isr.isrTask.saturationDetection
1997 lsst.ip.isr.isrFunctions.interpolateFromMask
1999 isrFunctions.interpolateFromMask(
2000 maskedImage=exposure.getMaskedImage(),
2001 fwhm=self.config.fwhm,
2002 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2003 maskNameList=list(self.config.saturatedMaskName),
2007 """Detect and mask suspect pixels in config.suspectMaskName.
2012 Exposure to process. Only the amplifier DataSec is processed.
2014 Amplifier detector data.
2018 lsst.ip.isr.isrFunctions.makeThresholdMask
2022 Suspect pixels are pixels whose value
is greater than
2023 amp.getSuspectLevel(). This
is intended to indicate pixels that may be
2024 affected by unknown systematics;
for example
if non-linearity
2025 corrections above a certain level are unstable then that would be a
2026 useful value
for suspectLevel. A value of `nan` indicates that no such
2027 level exists
and no pixels are to be masked
as suspicious.
2029 suspectLevel = amp.getSuspectLevel()
2030 if math.isnan(suspectLevel):
2033 maskedImage = exposure.getMaskedImage()
2034 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2035 isrFunctions.makeThresholdMask(
2036 maskedImage=dataView,
2037 threshold=suspectLevel,
2039 maskName=self.config.suspectMaskName,
2043 """Mask defects using mask plane "BAD", in place.
2048 Exposure to process.
2051 List of defects to mask.
2055 Call this after CCD assembly, since defects may cross amplifier
2058 maskedImage = exposure.getMaskedImage()
2059 if not isinstance(defectBaseList, Defects):
2061 defectList =
Defects(defectBaseList)
2063 defectList = defectBaseList
2064 defectList.maskPixels(maskedImage, maskName=
"BAD")
2066 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2067 """Mask edge pixels with applicable mask plane.
2072 Exposure to process.
2073 numEdgePixels : `int`, optional
2074 Number of edge pixels to mask.
2075 maskPlane : `str`, optional
2076 Mask plane name to use.
2077 level : `str`, optional
2078 Level at which to mask edges.
2080 maskedImage = exposure.getMaskedImage()
2081 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2083 if numEdgePixels > 0:
2084 if level ==
'DETECTOR':
2085 boxes = [maskedImage.getBBox()]
2086 elif level ==
'AMP':
2087 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2092 subImage = maskedImage[box]
2093 box.grow(-numEdgePixels)
2095 SourceDetectionTask.setEdgeBits(
2101 """Mask and interpolate defects using mask plane "BAD", in place.
2106 Exposure to process.
2109 List of defects to mask
and interpolate.
2113 lsst.ip.isr.isrTask.maskDefect
2116 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2117 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2118 isrFunctions.interpolateFromMask(
2119 maskedImage=exposure.getMaskedImage(),
2120 fwhm=self.config.fwhm,
2121 growSaturatedFootprints=0,
2122 maskNameList=[
"BAD"],
2126 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2131 Exposure to process.
2135 We mask over all non-finite values (NaN, inf), including those
2136 that are masked with other bits (because those may
or may
not be
2137 interpolated over later,
and we want to remove all NaN/infs).
2138 Despite this behaviour, the
"UNMASKEDNAN" mask plane
is used to
2139 preserve the historical name.
2141 maskedImage = exposure.getMaskedImage()
2144 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2145 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2146 numNans = maskNans(maskedImage, maskVal)
2147 self.metadata[
"NUMNANS"] = numNans
2149 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2152 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2158 Exposure to process.
2162 lsst.ip.isr.isrTask.maskNan
2165 isrFunctions.interpolateFromMask(
2166 maskedImage=exposure.getMaskedImage(),
2167 fwhm=self.config.fwhm,
2168 growSaturatedFootprints=0,
2169 maskNameList=["UNMASKEDNAN"],
2173 """Measure the image background in subgrids, for quality control.
2178 Exposure to process.
2180 Configuration object containing parameters on which background
2181 statistics and subgrids to use.
2183 if IsrQaConfig
is not None:
2184 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2185 IsrQaConfig.flatness.nIter)
2186 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2187 statsControl.setAndMask(maskVal)
2188 maskedImage = exposure.getMaskedImage()
2189 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2190 skyLevel = stats.getValue(afwMath.MEDIAN)
2191 skySigma = stats.getValue(afwMath.STDEVCLIP)
2192 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2193 metadata = exposure.getMetadata()
2194 metadata[
"SKYLEVEL"] = skyLevel
2195 metadata[
"SKYSIGMA"] = skySigma
2198 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2199 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2200 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2201 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2202 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2203 skyLevels = numpy.zeros((nX, nY))
2206 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2208 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2210 xLLC = xc - meshXHalf
2211 yLLC = yc - meshYHalf
2212 xURC = xc + meshXHalf - 1
2213 yURC = yc + meshYHalf - 1
2216 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2218 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2220 good = numpy.where(numpy.isfinite(skyLevels))
2221 skyMedian = numpy.median(skyLevels[good])
2222 flatness = (skyLevels[good] - skyMedian) / skyMedian
2223 flatness_rms = numpy.std(flatness)
2224 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2226 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2227 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2228 nX, nY, flatness_pp, flatness_rms)
2230 metadata[
"FLATNESS_PP"] = float(flatness_pp)
2231 metadata[
"FLATNESS_RMS"] = float(flatness_rms)
2232 metadata[
"FLATNESS_NGRIDS"] =
'%dx%d' % (nX, nY)
2233 metadata[
"FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2234 metadata[
"FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2237 """Set an approximate magnitude zero point for the exposure.
2242 Exposure to process.
2244 filterLabel = exposure.getFilter()
2245 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2247 if physicalFilter
in self.config.fluxMag0T1:
2248 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2250 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2251 fluxMag0 = self.config.defaultFluxMag0T1
2253 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2255 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2258 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2259 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2260 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2264 """Context manager that applies and removes flats and darks,
2265 if the task
is configured to apply them.
2270 Exposure to process.
2272 Flat exposure the same size
as ``exp``.
2274 Dark exposure the same size
as ``exp``.
2279 The flat
and dark corrected exposure.
2281 if self.config.doDark
and dark
is not None:
2283 if self.config.doFlat:
2288 if self.config.doFlat:
2290 if self.config.doDark
and dark
is not None:
2294 """Utility function to examine ISR exposure at different stages.
2301 State of processing to view.
2303 frame = getDebugFrame(self._display, stepname)
2305 display = getDisplay(frame)
2306 display.scale(
'asinh',
'zscale')
2307 display.mtv(exposure)
2308 prompt =
"Press Enter to continue [c]... "
2310 ans = input(prompt).lower()
2311 if ans
in (
"",
"c",):
2316 """A Detector-like object that supports returning gain and saturation level
2318 This is used when the input exposure does
not have a detector.
2323 Exposure to generate a fake amplifier
for.
2324 config : `lsst.ip.isr.isrTaskConfig`
2325 Configuration to apply to the fake amplifier.
2329 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2331 self.
_gain = config.gain
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, deferredCharge=None)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)