22__all__ = [
"IsrTask",
"IsrTaskConfig"]
31import lsst.pipe.base
as pipeBase
32import lsst.pipe.base.connectionTypes
as cT
34from contextlib
import contextmanager
35from lsstDebug
import getDebugFrame
40from lsst.utils.timer
import timeMethod
42from .
import isrFunctions
44from .
import linearize
45from .defects
import Defects
47from .assembleCcdTask
import AssembleCcdTask
48from .crosstalk
import CrosstalkTask, CrosstalkCalib
49from .fringe
import FringeTask
50from .isr
import maskNans
51from .masking
import MaskingTask
52from .overscan
import OverscanCorrectionTask
53from .straylight
import StrayLightTask
54from .vignette
import VignetteTask
55from .ampOffset
import AmpOffsetTask
56from .deferredCharge
import DeferredChargeTask
57from .isrStatistics
import IsrStatisticsTask
58from lsst.daf.butler
import DimensionGraph
62 """Lookup function to identify crosstalkSource entries.
64 This should return an empty list under most circumstances. Only
65 when inter-chip crosstalk has been identified should this be
72 registry : `lsst.daf.butler.Registry`
73 Butler registry to query.
74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
75 Data id to transform to identify crosstalkSources. The
76 ``detector`` entry will be stripped.
77 collections : `lsst.daf.butler.CollectionSearch`
78 Collections to search through.
82 results : `list` [`lsst.daf.butler.DatasetRef`]
83 List of datasets that match the query that will be used
as
86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"exposure"]))
87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records))
for ref
in results]
98 dimensions={
"instrument",
"exposure",
"detector"},
100 ccdExposure = cT.Input(
102 doc=
"Input exposure to process.",
103 storageClass=
"Exposure",
104 dimensions=[
"instrument",
"exposure",
"detector"],
106 camera = cT.PrerequisiteInput(
108 storageClass=
"Camera",
109 doc=
"Input camera to construct complete exposures.",
110 dimensions=[
"instrument"],
114 crosstalk = cT.PrerequisiteInput(
116 doc=
"Input crosstalk object",
117 storageClass=
"CrosstalkCalib",
118 dimensions=[
"instrument",
"detector"],
122 crosstalkSources = cT.PrerequisiteInput(
123 name=
"isrOverscanCorrected",
124 doc=
"Overscan corrected input images.",
125 storageClass=
"Exposure",
126 dimensions=[
"instrument",
"exposure",
"detector"],
129 lookupFunction=crosstalkSourceLookup,
132 bias = cT.PrerequisiteInput(
134 doc=
"Input bias calibration.",
135 storageClass=
"ExposureF",
136 dimensions=[
"instrument",
"detector"],
139 dark = cT.PrerequisiteInput(
141 doc=
"Input dark calibration.",
142 storageClass=
"ExposureF",
143 dimensions=[
"instrument",
"detector"],
146 flat = cT.PrerequisiteInput(
148 doc=
"Input flat calibration.",
149 storageClass=
"ExposureF",
150 dimensions=[
"instrument",
"physical_filter",
"detector"],
153 ptc = cT.PrerequisiteInput(
155 doc=
"Input Photon Transfer Curve dataset",
156 storageClass=
"PhotonTransferCurveDataset",
157 dimensions=[
"instrument",
"detector"],
160 fringes = cT.PrerequisiteInput(
162 doc=
"Input fringe calibration.",
163 storageClass=
"ExposureF",
164 dimensions=[
"instrument",
"physical_filter",
"detector"],
168 strayLightData = cT.PrerequisiteInput(
170 doc=
"Input stray light calibration.",
171 storageClass=
"StrayLightData",
172 dimensions=[
"instrument",
"physical_filter",
"detector"],
177 bfKernel = cT.PrerequisiteInput(
179 doc=
"Input brighter-fatter kernel.",
180 storageClass=
"NumpyArray",
181 dimensions=[
"instrument"],
185 newBFKernel = cT.PrerequisiteInput(
186 name=
'brighterFatterKernel',
187 doc=
"Newer complete kernel + gain solutions.",
188 storageClass=
"BrighterFatterKernel",
189 dimensions=[
"instrument",
"detector"],
193 defects = cT.PrerequisiteInput(
195 doc=
"Input defect tables.",
196 storageClass=
"Defects",
197 dimensions=[
"instrument",
"detector"],
200 linearizer = cT.PrerequisiteInput(
202 storageClass=
"Linearizer",
203 doc=
"Linearity correction calibration.",
204 dimensions=[
"instrument",
"detector"],
208 opticsTransmission = cT.PrerequisiteInput(
209 name=
"transmission_optics",
210 storageClass=
"TransmissionCurve",
211 doc=
"Transmission curve due to the optics.",
212 dimensions=[
"instrument"],
215 filterTransmission = cT.PrerequisiteInput(
216 name=
"transmission_filter",
217 storageClass=
"TransmissionCurve",
218 doc=
"Transmission curve due to the filter.",
219 dimensions=[
"instrument",
"physical_filter"],
222 sensorTransmission = cT.PrerequisiteInput(
223 name=
"transmission_sensor",
224 storageClass=
"TransmissionCurve",
225 doc=
"Transmission curve due to the sensor.",
226 dimensions=[
"instrument",
"detector"],
229 atmosphereTransmission = cT.PrerequisiteInput(
230 name=
"transmission_atmosphere",
231 storageClass=
"TransmissionCurve",
232 doc=
"Transmission curve due to the atmosphere.",
233 dimensions=[
"instrument"],
236 illumMaskedImage = cT.PrerequisiteInput(
238 doc=
"Input illumination correction.",
239 storageClass=
"MaskedImageF",
240 dimensions=[
"instrument",
"physical_filter",
"detector"],
243 deferredChargeCalib = cT.PrerequisiteInput(
245 doc=
"Deferred charge/CTI correction dataset.",
246 storageClass=
"IsrCalib",
247 dimensions=[
"instrument",
"detector"],
251 outputExposure = cT.Output(
253 doc=
"Output ISR processed exposure.",
254 storageClass=
"Exposure",
255 dimensions=[
"instrument",
"exposure",
"detector"],
257 preInterpExposure = cT.Output(
258 name=
'preInterpISRCCD',
259 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
260 storageClass=
"ExposureF",
261 dimensions=[
"instrument",
"exposure",
"detector"],
263 outputOssThumbnail = cT.Output(
265 doc=
"Output Overscan-subtracted thumbnail image.",
266 storageClass=
"Thumbnail",
267 dimensions=[
"instrument",
"exposure",
"detector"],
269 outputFlattenedThumbnail = cT.Output(
270 name=
"FlattenedThumb",
271 doc=
"Output flat-corrected thumbnail image.",
272 storageClass=
"Thumbnail",
273 dimensions=[
"instrument",
"exposure",
"detector"],
275 outputStatistics = cT.Output(
276 name=
"isrStatistics",
277 doc=
"Output of additional statistics table.",
278 storageClass=
"StructuredDataDict",
279 dimensions=[
"instrument",
"exposure",
"detector"],
285 if config.doBias
is not True:
286 self.prerequisiteInputs.remove(
"bias")
287 if config.doLinearize
is not True:
288 self.prerequisiteInputs.remove(
"linearizer")
289 if config.doCrosstalk
is not True:
290 self.prerequisiteInputs.remove(
"crosstalkSources")
291 self.prerequisiteInputs.remove(
"crosstalk")
292 if config.doBrighterFatter
is not True:
293 self.prerequisiteInputs.remove(
"bfKernel")
294 self.prerequisiteInputs.remove(
"newBFKernel")
295 if config.doDefect
is not True:
296 self.prerequisiteInputs.remove(
"defects")
297 if config.doDark
is not True:
298 self.prerequisiteInputs.remove(
"dark")
299 if config.doFlat
is not True:
300 self.prerequisiteInputs.remove(
"flat")
301 if config.doFringe
is not True:
302 self.prerequisiteInputs.remove(
"fringes")
303 if config.doStrayLight
is not True:
304 self.prerequisiteInputs.remove(
"strayLightData")
305 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
306 self.prerequisiteInputs.remove(
"ptc")
307 if config.doAttachTransmissionCurve
is not True:
308 self.prerequisiteInputs.remove(
"opticsTransmission")
309 self.prerequisiteInputs.remove(
"filterTransmission")
310 self.prerequisiteInputs.remove(
"sensorTransmission")
311 self.prerequisiteInputs.remove(
"atmosphereTransmission")
313 if config.doUseOpticsTransmission
is not True:
314 self.prerequisiteInputs.remove(
"opticsTransmission")
315 if config.doUseFilterTransmission
is not True:
316 self.prerequisiteInputs.remove(
"filterTransmission")
317 if config.doUseSensorTransmission
is not True:
318 self.prerequisiteInputs.remove(
"sensorTransmission")
319 if config.doUseAtmosphereTransmission
is not True:
320 self.prerequisiteInputs.remove(
"atmosphereTransmission")
321 if config.doIlluminationCorrection
is not True:
322 self.prerequisiteInputs.remove(
"illumMaskedImage")
323 if config.doDeferredCharge
is not True:
324 self.prerequisiteInputs.remove(
"deferredChargeCalib")
326 if config.doWrite
is not True:
327 self.outputs.remove(
"outputExposure")
328 self.outputs.remove(
"preInterpExposure")
329 self.outputs.remove(
"outputFlattenedThumbnail")
330 self.outputs.remove(
"outputOssThumbnail")
331 self.outputs.remove(
"outputStatistics")
333 if config.doSaveInterpPixels
is not True:
334 self.outputs.remove(
"preInterpExposure")
335 if config.qa.doThumbnailOss
is not True:
336 self.outputs.remove(
"outputOssThumbnail")
337 if config.qa.doThumbnailFlattened
is not True:
338 self.outputs.remove(
"outputFlattenedThumbnail")
339 if config.doCalculateStatistics
is not True:
340 self.outputs.remove(
"outputStatistics")
344 pipelineConnections=IsrTaskConnections):
345 """Configuration parameters for IsrTask.
347 Items are grouped in the order
in which they are executed by the task.
349 datasetType = pexConfig.Field(
351 doc="Dataset type for input data; users will typically leave this alone, "
352 "but camera-specific ISR tasks will override it",
356 fallbackFilterName = pexConfig.Field(
358 doc=
"Fallback default filter name for calibrations.",
361 useFallbackDate = pexConfig.Field(
363 doc=
"Pass observation date when using fallback filter.",
366 expectWcs = pexConfig.Field(
369 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
371 fwhm = pexConfig.Field(
373 doc=
"FWHM of PSF in arcseconds.",
376 qa = pexConfig.ConfigField(
378 doc=
"QA related configuration options.",
380 doHeaderProvenance = pexConfig.Field(
383 doc=
"Write calibration identifiers into output exposure header?",
387 doRaiseOnCalibMismatch = pexConfig.Field(
390 doc=
"Should IsrTask halt if exposure and calibration header values do not match?",
392 cameraKeywordsToCompare = pexConfig.ListField(
394 doc=
"List of header keywords to compare between exposure and calibrations.",
399 doConvertIntToFloat = pexConfig.Field(
401 doc=
"Convert integer raw images to floating point values?",
406 doSaturation = pexConfig.Field(
408 doc=
"Mask saturated pixels? NB: this is totally independent of the"
409 " interpolation option - this is ONLY setting the bits in the mask."
410 " To have them interpolated make sure doSaturationInterpolation=True",
413 saturatedMaskName = pexConfig.Field(
415 doc=
"Name of mask plane to use in saturation detection and interpolation",
418 saturation = pexConfig.Field(
420 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
421 default=float(
"NaN"),
423 growSaturationFootprintSize = pexConfig.Field(
425 doc=
"Number of pixels by which to grow the saturation footprints",
430 doSuspect = pexConfig.Field(
432 doc=
"Mask suspect pixels?",
435 suspectMaskName = pexConfig.Field(
437 doc=
"Name of mask plane to use for suspect pixels",
440 numEdgeSuspect = pexConfig.Field(
442 doc=
"Number of edge pixels to be flagged as untrustworthy.",
445 edgeMaskLevel = pexConfig.ChoiceField(
447 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
450 'DETECTOR':
'Mask only the edges of the full detector.',
451 'AMP':
'Mask edges of each amplifier.',
456 doSetBadRegions = pexConfig.Field(
458 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
461 badStatistic = pexConfig.ChoiceField(
463 doc=
"How to estimate the average value for BAD regions.",
466 "MEANCLIP":
"Correct using the (clipped) mean of good data",
467 "MEDIAN":
"Correct using the median of the good data",
472 doOverscan = pexConfig.Field(
474 doc=
"Do overscan subtraction?",
477 overscan = pexConfig.ConfigurableField(
478 target=OverscanCorrectionTask,
479 doc=
"Overscan subtraction task for image segments.",
483 doAssembleCcd = pexConfig.Field(
486 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
488 assembleCcd = pexConfig.ConfigurableField(
489 target=AssembleCcdTask,
490 doc=
"CCD assembly task",
494 doAssembleIsrExposures = pexConfig.Field(
497 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
499 doTrimToMatchCalib = pexConfig.Field(
502 doc=
"Trim raw data to match calibration bounding boxes?"
506 doBias = pexConfig.Field(
508 doc=
"Apply bias frame correction?",
511 biasDataProductName = pexConfig.Field(
513 doc=
"Name of the bias data product",
516 doBiasBeforeOverscan = pexConfig.Field(
518 doc=
"Reverse order of overscan and bias correction.",
523 doDeferredCharge = pexConfig.Field(
525 doc=
"Apply deferred charge correction?",
528 deferredChargeCorrection = pexConfig.ConfigurableField(
529 target=DeferredChargeTask,
530 doc=
"Deferred charge correction task.",
534 doVariance = pexConfig.Field(
536 doc=
"Calculate variance?",
539 gain = pexConfig.Field(
541 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
542 default=float(
"NaN"),
544 readNoise = pexConfig.Field(
546 doc=
"The read noise to use if no Detector is present in the Exposure",
549 doEmpiricalReadNoise = pexConfig.Field(
552 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
554 usePtcReadNoise = pexConfig.Field(
557 doc=
"Use readnoise values from the Photon Transfer Curve?"
559 maskNegativeVariance = pexConfig.Field(
562 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
563 "in the measurement of the overscan at an edge due to the data falling off faster "
564 "than the overscan model can account for it."
566 negativeVarianceMaskName = pexConfig.Field(
569 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
572 doLinearize = pexConfig.Field(
574 doc=
"Correct for nonlinearity of the detector's response?",
579 doCrosstalk = pexConfig.Field(
581 doc=
"Apply intra-CCD crosstalk correction?",
584 doCrosstalkBeforeAssemble = pexConfig.Field(
586 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
589 crosstalk = pexConfig.ConfigurableField(
590 target=CrosstalkTask,
591 doc=
"Intra-CCD crosstalk correction",
595 doDefect = pexConfig.Field(
597 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
600 doNanMasking = pexConfig.Field(
602 doc=
"Mask non-finite (NAN, inf) pixels?",
605 doWidenSaturationTrails = pexConfig.Field(
607 doc=
"Widen bleed trails based on their width?",
612 doBrighterFatter = pexConfig.Field(
615 doc=
"Apply the brighter-fatter correction?"
617 brighterFatterLevel = pexConfig.ChoiceField(
620 doc=
"The level at which to correct for brighter-fatter.",
622 "AMP":
"Every amplifier treated separately.",
623 "DETECTOR":
"One kernel per detector",
626 brighterFatterMaxIter = pexConfig.Field(
629 doc=
"Maximum number of iterations for the brighter-fatter correction"
631 brighterFatterThreshold = pexConfig.Field(
634 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
635 "absolute value of the difference between the current corrected image and the one "
636 "from the previous iteration summed over all the pixels."
638 brighterFatterApplyGain = pexConfig.Field(
641 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
643 brighterFatterMaskListToInterpolate = pexConfig.ListField(
645 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
647 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
649 brighterFatterMaskGrowSize = pexConfig.Field(
652 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
653 "when brighter-fatter correction is applied."
657 doDark = pexConfig.Field(
659 doc=
"Apply dark frame correction?",
662 darkDataProductName = pexConfig.Field(
664 doc=
"Name of the dark data product",
669 doStrayLight = pexConfig.Field(
671 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
674 strayLight = pexConfig.ConfigurableField(
675 target=StrayLightTask,
676 doc=
"y-band stray light correction"
680 doFlat = pexConfig.Field(
682 doc=
"Apply flat field correction?",
685 flatDataProductName = pexConfig.Field(
687 doc=
"Name of the flat data product",
690 flatScalingType = pexConfig.ChoiceField(
692 doc=
"The method for scaling the flat on the fly.",
695 "USER":
"Scale by flatUserScale",
696 "MEAN":
"Scale by the inverse of the mean",
697 "MEDIAN":
"Scale by the inverse of the median",
700 flatUserScale = pexConfig.Field(
702 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
705 doTweakFlat = pexConfig.Field(
707 doc=
"Tweak flats to match observed amplifier ratios?",
713 doApplyGains = pexConfig.Field(
715 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
718 usePtcGains = pexConfig.Field(
720 doc=
"Use the gain values from the Photon Transfer Curve?",
723 normalizeGains = pexConfig.Field(
725 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
730 doFringe = pexConfig.Field(
732 doc=
"Apply fringe correction?",
735 fringe = pexConfig.ConfigurableField(
737 doc=
"Fringe subtraction task",
739 fringeAfterFlat = pexConfig.Field(
741 doc=
"Do fringe subtraction after flat-fielding?",
746 doAmpOffset = pexConfig.Field(
747 doc=
"Calculate and apply amp offset corrections?",
751 ampOffset = pexConfig.ConfigurableField(
752 doc=
"Amp offset correction task.",
753 target=AmpOffsetTask,
757 doMeasureBackground = pexConfig.Field(
759 doc=
"Measure the background level on the reduced image?",
764 doCameraSpecificMasking = pexConfig.Field(
766 doc=
"Mask camera-specific bad regions?",
769 masking = pexConfig.ConfigurableField(
775 doInterpolate = pexConfig.Field(
777 doc=
"Interpolate masked pixels?",
780 doSaturationInterpolation = pexConfig.Field(
782 doc=
"Perform interpolation over pixels masked as saturated?"
783 " NB: This is independent of doSaturation; if that is False this plane"
784 " will likely be blank, resulting in a no-op here.",
787 doNanInterpolation = pexConfig.Field(
789 doc=
"Perform interpolation over pixels masked as NaN?"
790 " NB: This is independent of doNanMasking; if that is False this plane"
791 " will likely be blank, resulting in a no-op here.",
794 doNanInterpAfterFlat = pexConfig.Field(
796 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
797 "also have to interpolate them before flat-fielding."),
800 maskListToInterpolate = pexConfig.ListField(
802 doc=
"List of mask planes that should be interpolated.",
803 default=[
'SAT',
'BAD'],
805 doSaveInterpPixels = pexConfig.Field(
807 doc=
"Save a copy of the pre-interpolated pixel values?",
812 fluxMag0T1 = pexConfig.DictField(
815 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
816 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
819 defaultFluxMag0T1 = pexConfig.Field(
821 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
822 default=pow(10.0, 0.4*28.0)
826 doVignette = pexConfig.Field(
828 doc=(
"Compute and attach the validPolygon defining the unvignetted region to the exposure "
829 "according to vignetting parameters?"),
832 doMaskVignettePolygon = pexConfig.Field(
834 doc=(
"Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
838 vignetteValue = pexConfig.Field(
840 doc=
"Value to replace image array pixels with in the vignetted region? Ignored if None.",
844 vignette = pexConfig.ConfigurableField(
846 doc=
"Vignetting task.",
850 doAttachTransmissionCurve = pexConfig.Field(
853 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
855 doUseOpticsTransmission = pexConfig.Field(
858 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
860 doUseFilterTransmission = pexConfig.Field(
863 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
865 doUseSensorTransmission = pexConfig.Field(
868 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
870 doUseAtmosphereTransmission = pexConfig.Field(
873 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
877 doIlluminationCorrection = pexConfig.Field(
880 doc=
"Perform illumination correction?"
882 illuminationCorrectionDataProductName = pexConfig.Field(
884 doc=
"Name of the illumination correction data product.",
887 illumScale = pexConfig.Field(
889 doc=
"Scale factor for the illumination correction.",
892 illumFilters = pexConfig.ListField(
895 doc=
"Only perform illumination correction for these filters."
899 doStandardStatistics = pexConfig.Field(
901 doc=
"Should standard image quality statistics be calculated?",
905 doCalculateStatistics = pexConfig.Field(
907 doc=
"Should additional ISR statistics be calculated?",
910 isrStats = pexConfig.ConfigurableField(
911 target=IsrStatisticsTask,
912 doc=
"Task to calculate additional statistics.",
917 doWrite = pexConfig.Field(
919 doc=
"Persist postISRCCD?",
926 raise ValueError(
"You may not specify both doFlat and doApplyGains")
928 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
938 """Apply common instrument signature correction algorithms to a raw frame.
940 The process for correcting imaging data
is very similar
from
941 camera to camera. This task provides a vanilla implementation of
942 doing these corrections, including the ability to turn certain
943 corrections off
if they are
not needed. The inputs to the primary
944 method, `
run()`, are a raw exposure to be corrected
and the
945 calibration data products. The raw input
is a single chip sized
946 mosaic of all amps including overscans
and other non-science
949 The __init__ method sets up the subtasks
for ISR processing, using
955 Positional arguments passed to the Task constructor.
956 None used at this time.
957 kwargs : `dict`, optional
958 Keyword arguments passed on to the Task constructor.
959 None used at this time.
961 ConfigClass = IsrTaskConfig
966 self.makeSubtask(
"assembleCcd")
967 self.makeSubtask(
"crosstalk")
968 self.makeSubtask(
"strayLight")
969 self.makeSubtask(
"fringe")
970 self.makeSubtask(
"masking")
971 self.makeSubtask(
"overscan")
972 self.makeSubtask(
"vignette")
973 self.makeSubtask(
"ampOffset")
974 self.makeSubtask(
"deferredChargeCorrection")
975 self.makeSubtask(
"isrStats")
978 inputs = butlerQC.get(inputRefs)
981 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
982 except Exception
as e:
983 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
986 detector = inputs[
'ccdExposure'].getDetector()
988 if self.config.doCrosstalk
is True:
991 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
992 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
993 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
995 coeffVector = (self.config.crosstalk.crosstalkValues
996 if self.config.crosstalk.useConfigCoefficients
else None)
997 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
998 inputs[
'crosstalk'] = crosstalkCalib
999 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
1000 if 'crosstalkSources' not in inputs:
1001 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
1004 if 'linearizer' in inputs:
1005 if isinstance(inputs[
'linearizer'], dict):
1007 linearizer.fromYaml(inputs[
'linearizer'])
1008 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1009 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1013 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1015 linearizer = inputs[
'linearizer']
1016 linearizer.log = self.log
1017 inputs[
'linearizer'] = linearizer
1020 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1022 if self.config.doDefect
is True:
1023 if "defects" in inputs
and inputs[
'defects']
is not None:
1027 if not isinstance(inputs[
"defects"], Defects):
1028 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1032 if self.config.doBrighterFatter:
1033 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1034 if brighterFatterKernel
is None:
1035 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1037 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1039 detName = detector.getName()
1040 level = brighterFatterKernel.level
1043 inputs[
'bfGains'] = brighterFatterKernel.gain
1044 if self.config.brighterFatterLevel ==
'DETECTOR':
1045 if level ==
'DETECTOR':
1046 if detName
in brighterFatterKernel.detKernels:
1047 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1049 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1050 elif level ==
'AMP':
1051 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1053 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1054 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1055 elif self.config.brighterFatterLevel ==
'AMP':
1056 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1058 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
1059 expId = inputs[
'ccdExposure'].info.id
1060 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1062 assembler=self.assembleCcd
1063 if self.config.doAssembleIsrExposures
else None)
1065 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1067 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
1068 if 'strayLightData' not in inputs:
1069 inputs[
'strayLightData'] =
None
1071 if self.config.doHeaderProvenance:
1073 exposureMetadata = inputs[
'ccdExposure'].getMetadata()
1074 for inputName
in sorted(inputs.keys()):
1075 reference = getattr(inputRefs, inputName,
None)
1076 if reference
is not None and hasattr(reference,
"run"):
1077 runKey = f
"LSST CALIB RUN {inputName.upper()}"
1078 runValue = reference.run
1079 idKey = f
"LSST CALIB UUID {inputName.upper()}"
1080 idValue = str(reference.id)
1082 exposureMetadata[runKey] = runValue
1083 exposureMetadata[idKey] = idValue
1085 outputs = self.
run(**inputs)
1086 butlerQC.put(outputs, outputRefs)
1089 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1090 crosstalk=None, crosstalkSources=None,
1091 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1092 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1093 sensorTransmission=
None, atmosphereTransmission=
None,
1094 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1095 deferredChargeCalib=
None,
1097 """Perform instrument signature removal on an exposure.
1099 Steps included in the ISR processing,
in order performed, are:
1101 - saturation
and suspect pixel masking
1102 - overscan subtraction
1103 - CCD assembly of individual amplifiers
1105 - variance image construction
1106 - linearization of non-linear response
1108 - brighter-fatter correction
1111 - stray light subtraction
1113 - masking of known defects
and camera specific features
1114 - vignette calculation
1115 - appending transmission curve
and distortion model
1120 The raw exposure that
is to be run through ISR. The
1121 exposure
is modified by this method.
1123 The camera geometry
for this exposure. Required
if
1124 one
or more of ``ccdExposure``, ``bias``, ``dark``,
or
1125 ``flat`` does
not have an associated detector.
1127 Bias calibration frame.
1129 Functor
for linearization.
1131 Calibration
for crosstalk.
1132 crosstalkSources : `list`, optional
1133 List of possible crosstalk sources.
1135 Dark calibration frame.
1137 Flat calibration frame.
1139 Photon transfer curve dataset,
with, e.g., gains
1141 bfKernel : `numpy.ndarray`, optional
1142 Brighter-fatter kernel.
1143 bfGains : `dict` of `float`, optional
1144 Gains used to override the detector
's nominal gains for the
1145 brighter-fatter correction. A dict keyed by amplifier name for
1146 the detector
in question.
1149 fringes : `lsst.pipe.base.Struct`, optional
1150 Struct containing the fringe correction data,
with
1156 random seed derived
from the ``ccdExposureId``
for random
1157 number generator (`numpy.uint32`)
1159 A ``TransmissionCurve`` that represents the throughput of the,
1160 optics, to be evaluated
in focal-plane coordinates.
1162 A ``TransmissionCurve`` that represents the throughput of the
1163 filter itself, to be evaluated
in focal-plane coordinates.
1165 A ``TransmissionCurve`` that represents the throughput of the
1166 sensor itself, to be evaluated
in post-assembly trimmed detector
1169 A ``TransmissionCurve`` that represents the throughput of the
1170 atmosphere, assumed to be spatially constant.
1171 detectorNum : `int`, optional
1172 The integer number
for the detector to process.
1173 strayLightData : `object`, optional
1174 Opaque object containing calibration information
for stray-light
1175 correction. If `
None`, no correction will be performed.
1177 Illumination correction image.
1181 result : `lsst.pipe.base.Struct`
1182 Result struct
with component:
1185 The fully ISR corrected exposure.
1190 Thumbnail image of the exposure after overscan subtraction.
1193 Thumbnail image of the exposure after flat-field correction.
1195 ``outputStatistics``
1196 Values of the additional statistics calculated.
1201 Raised
if a configuration option
is set to `
True`, but the
1202 required calibration data has
not been specified.
1206 The current processed exposure can be viewed by setting the
1207 appropriate `lsstDebug` entries
in the ``debug.display``
1208 dictionary. The names of these entries correspond to some of
1209 the `IsrTaskConfig` Boolean options,
with the value denoting the
1210 frame to use. The exposure
is shown inside the matching
1211 option check
and after the processing of that step has
1212 finished. The steps
with debug points are:
1223 In addition, setting the ``postISRCCD`` entry displays the
1224 exposure after all ISR processing has finished.
1227 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1232 ccd = ccdExposure.getDetector()
1233 filterLabel = ccdExposure.getFilter()
1234 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1237 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1238 ccd = [
FakeAmp(ccdExposure, self.config)]
1241 if self.config.doBias
and bias
is None:
1242 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1244 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1245 if self.config.doBrighterFatter
and bfKernel
is None:
1246 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1247 if self.config.doDark
and dark
is None:
1248 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1249 if self.config.doFlat
and flat
is None:
1250 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1251 if self.config.doDefect
and defects
is None:
1252 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1253 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1254 and fringes.fringes
is None):
1259 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1260 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1261 and illumMaskedImage
is None):
1262 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1263 if (self.config.doDeferredCharge
and deferredChargeCalib
is None):
1264 raise RuntimeError(
"Must supply a deferred charge calibration if config.doDeferredCharge=True.")
1266 if self.config.doHeaderProvenance:
1269 exposureMetadata = ccdExposure.getMetadata()
1270 if self.config.doBias:
1273 if self.config.doBrighterFatter:
1276 if self.config.doCrosstalk:
1277 exposureMetadata[
"LSST CALIB DATE CROSSTALK"] = self.
extractCalibDate(crosstalk)
1279 if self.config.doDark:
1282 if self.config.doDefect:
1283 exposureMetadata[
"LSST CALIB DATE DEFECTS"] = self.
extractCalibDate(defects)
1285 if self.config.doDeferredCharge:
1286 exposureMetadata[
"LSST CALIB DATE CTI"] = self.
extractCalibDate(deferredChargeCalib)
1288 if self.config.doFlat:
1291 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters):
1292 exposureMetadata[
"LSST CALIB DATE FRINGE"] = self.
extractCalibDate(fringes.fringes)
1294 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters):
1295 exposureMetadata[
"LSST CALIB DATE ILLUMINATION"] = self.
extractCalibDate(illumMaskedImage)
1298 exposureMetadata[
"LSST CALIB DATE LINEARIZER"] = self.
extractCalibDate(linearizer)
1300 if self.config.usePtcGains
or self.config.usePtcReadNoise:
1303 if self.config.doStrayLight:
1304 exposureMetadata[
"LSST CALIB DATE STRAYLIGHT"] = self.
extractCalibDate(strayLightData)
1306 if self.config.doAttachTransmissionCurve:
1307 exposureMetadata[
"LSST CALIB DATE OPTICS_TR"] = self.
extractCalibDate(opticsTransmission)
1308 exposureMetadata[
"LSST CALIB DATE FILTER_TR"] = self.
extractCalibDate(filterTransmission)
1309 exposureMetadata[
"LSST CALIB DATE SENSOR_TR"] = self.
extractCalibDate(sensorTransmission)
1310 exposureMetadata[
"LSST CALIB DATE ATMOSP_TR"] = self.
extractCalibDate(atmosphereTransmission)
1313 if self.config.doConvertIntToFloat:
1314 self.log.info(
"Converting exposure to floating point values.")
1317 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1318 self.log.info(
"Applying bias correction.")
1319 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1320 trimToFit=self.config.doTrimToMatchCalib)
1326 if self.config.doOverscan
and self.config.overscan.doParallelOverscan:
1328 self.overscan.maskParallelOverscan(ccdExposure, ccd)
1333 if ccdExposure.getBBox().contains(amp.getBBox()):
1338 if self.config.doOverscan
and not badAmp:
1341 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1342 if overscanResults
is not None and \
1343 self.config.qa
is not None and self.config.qa.saveStats
is True:
1344 if isinstance(overscanResults.overscanMean, float):
1346 mean = overscanResults.overscanMean
1347 sigma = overscanResults.overscanSigma
1348 residMean = overscanResults.residualMean
1349 residSigma = overscanResults.residualSigma
1353 mean = overscanResults.overscanMean[0]
1354 sigma = overscanResults.overscanSigma[0]
1355 residMean = overscanResults.residualMean[0]
1356 residSigma = overscanResults.residualSigma[0]
1358 self.metadata[f
"FIT MEDIAN {amp.getName()}"] = mean
1359 self.metadata[f
"FIT STDEV {amp.getName()}"] = sigma
1360 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1361 amp.getName(), mean, sigma)
1363 self.metadata[f
"RESIDUAL MEDIAN {amp.getName()}"] = residMean
1364 self.metadata[f
"RESIDUAL STDEV {amp.getName()}"] = residSigma
1365 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1366 amp.getName(), residMean, residSigma)
1368 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1371 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1372 overscanResults =
None
1374 overscans.append(overscanResults
if overscanResults
is not None else None)
1376 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1378 if self.config.doDeferredCharge:
1379 self.log.info(
"Applying deferred charge/CTI correction.")
1380 self.deferredChargeCorrection.
run(ccdExposure, deferredChargeCalib)
1381 self.
debugView(ccdExposure,
"doDeferredCharge")
1383 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1384 self.log.info(
"Applying crosstalk correction.")
1385 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1386 crosstalkSources=crosstalkSources, camera=camera)
1387 self.
debugView(ccdExposure,
"doCrosstalk")
1389 if self.config.doAssembleCcd:
1390 self.log.info(
"Assembling CCD from amplifiers.")
1391 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1393 if self.config.expectWcs
and not ccdExposure.getWcs():
1394 self.log.warning(
"No WCS found in input exposure.")
1395 self.
debugView(ccdExposure,
"doAssembleCcd")
1398 if self.config.qa.doThumbnailOss:
1399 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1401 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1402 self.log.info(
"Applying bias correction.")
1403 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1404 trimToFit=self.config.doTrimToMatchCalib)
1407 if self.config.doVariance:
1408 for amp, overscanResults
in zip(ccd, overscans):
1409 if ccdExposure.getBBox().contains(amp.getBBox()):
1410 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1411 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1412 if overscanResults
is not None:
1414 overscanImage=overscanResults.overscanImage,
1420 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1421 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1422 afwMath.MEDIAN | afwMath.STDEVCLIP)
1423 self.metadata[f
"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1424 qaStats.getValue(afwMath.MEDIAN)
1425 self.metadata[f
"ISR VARIANCE {amp.getName()} STDEV"] = \
1426 qaStats.getValue(afwMath.STDEVCLIP)
1427 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1428 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1429 qaStats.getValue(afwMath.STDEVCLIP))
1430 if self.config.maskNegativeVariance:
1434 self.log.info(
"Applying linearizer.")
1435 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1436 detector=ccd, log=self.log)
1438 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1439 self.log.info(
"Applying crosstalk correction.")
1440 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1441 crosstalkSources=crosstalkSources, isTrimmed=
True)
1442 self.
debugView(ccdExposure,
"doCrosstalk")
1447 if self.config.doDefect:
1448 self.log.info(
"Masking defects.")
1451 if self.config.numEdgeSuspect > 0:
1452 self.log.info(
"Masking edges as SUSPECT.")
1453 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1454 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1456 if self.config.doNanMasking:
1457 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1460 if self.config.doWidenSaturationTrails:
1461 self.log.info(
"Widening saturation trails.")
1462 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1464 if self.config.doCameraSpecificMasking:
1465 self.log.info(
"Masking regions for camera specific reasons.")
1466 self.masking.
run(ccdExposure)
1468 if self.config.doBrighterFatter:
1478 interpExp = ccdExposure.clone()
1480 isrFunctions.interpolateFromMask(
1481 maskedImage=interpExp.getMaskedImage(),
1482 fwhm=self.config.fwhm,
1483 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1484 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1486 bfExp = interpExp.clone()
1488 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1489 type(bfKernel), type(bfGains))
1490 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1491 self.config.brighterFatterMaxIter,
1492 self.config.brighterFatterThreshold,
1493 self.config.brighterFatterApplyGain,
1495 if bfResults[1] == self.config.brighterFatterMaxIter:
1496 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1499 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1501 image = ccdExposure.getMaskedImage().getImage()
1502 bfCorr = bfExp.getMaskedImage().getImage()
1503 bfCorr -= interpExp.getMaskedImage().getImage()
1512 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1513 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1516 if self.config.brighterFatterMaskGrowSize > 0:
1517 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1518 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1519 isrFunctions.growMasks(ccdExposure.getMask(),
1520 radius=self.config.brighterFatterMaskGrowSize,
1521 maskNameList=maskPlane,
1522 maskValue=maskPlane)
1524 self.
debugView(ccdExposure,
"doBrighterFatter")
1526 if self.config.doDark:
1527 self.log.info(
"Applying dark correction.")
1531 if self.config.doFringe
and not self.config.fringeAfterFlat:
1532 self.log.info(
"Applying fringe correction before flat.")
1533 self.fringe.
run(ccdExposure, **fringes.getDict())
1536 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1537 self.log.info(
"Checking strayLight correction.")
1538 self.strayLight.
run(ccdExposure, strayLightData)
1539 self.
debugView(ccdExposure,
"doStrayLight")
1541 if self.config.doFlat:
1542 self.log.info(
"Applying flat correction.")
1546 if self.config.doApplyGains:
1547 self.log.info(
"Applying gain correction instead of flat.")
1548 if self.config.usePtcGains:
1549 self.log.info(
"Using gains from the Photon Transfer Curve.")
1550 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1553 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1555 if self.config.doFringe
and self.config.fringeAfterFlat:
1556 self.log.info(
"Applying fringe correction after flat.")
1557 self.fringe.
run(ccdExposure, **fringes.getDict())
1559 if self.config.doVignette:
1560 if self.config.doMaskVignettePolygon:
1561 self.log.info(
"Constructing, attaching, and masking vignette polygon.")
1563 self.log.info(
"Constructing and attaching vignette polygon.")
1565 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1566 vignetteValue=self.config.vignetteValue, log=self.log)
1568 if self.config.doAttachTransmissionCurve:
1569 self.log.info(
"Adding transmission curves.")
1570 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1571 filterTransmission=filterTransmission,
1572 sensorTransmission=sensorTransmission,
1573 atmosphereTransmission=atmosphereTransmission)
1575 flattenedThumb =
None
1576 if self.config.qa.doThumbnailFlattened:
1577 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1579 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1580 self.log.info(
"Performing illumination correction.")
1581 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1582 illumMaskedImage, illumScale=self.config.illumScale,
1583 trimToFit=self.config.doTrimToMatchCalib)
1586 if self.config.doSaveInterpPixels:
1587 preInterpExp = ccdExposure.clone()
1602 if self.config.doSetBadRegions:
1603 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1604 if badPixelCount > 0:
1605 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1607 if self.config.doInterpolate:
1608 self.log.info(
"Interpolating masked pixels.")
1609 isrFunctions.interpolateFromMask(
1610 maskedImage=ccdExposure.getMaskedImage(),
1611 fwhm=self.config.fwhm,
1612 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1613 maskNameList=list(self.config.maskListToInterpolate)
1619 if self.config.doAmpOffset:
1620 self.log.info(
"Correcting amp offsets.")
1621 self.ampOffset.
run(ccdExposure)
1623 if self.config.doMeasureBackground:
1624 self.log.info(
"Measuring background level.")
1627 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1629 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1630 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1631 afwMath.MEDIAN | afwMath.STDEVCLIP)
1632 self.metadata[f
"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1633 self.metadata[f
"ISR BACKGROUND {amp.getName()} STDEV"] = \
1634 qaStats.getValue(afwMath.STDEVCLIP)
1635 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1636 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1637 qaStats.getValue(afwMath.STDEVCLIP))
1640 if self.config.doStandardStatistics:
1641 metadata = ccdExposure.getMetadata()
1643 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1644 ampName = amp.getName()
1645 metadata[f
"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels(
1646 ampExposure.getMaskedImage(),
1647 [self.config.saturatedMaskName]
1649 metadata[f
"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels(
1650 ampExposure.getMaskedImage(),
1653 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1654 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP)
1656 metadata[f
"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN)
1657 metadata[f
"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN)
1658 metadata[f
"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP)
1660 k1 = f
"LSST ISR FINAL MEDIAN {ampName}"
1661 k2 = f
"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}"
1662 if self.config.doOverscan
and k1
in metadata
and k2
in metadata:
1663 metadata[f
"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2]
1665 metadata[f
"LSST ISR LEVEL {ampName}"] = numpy.nan
1668 outputStatistics =
None
1669 if self.config.doCalculateStatistics:
1670 outputStatistics = self.isrStats.
run(ccdExposure, overscanResults=overscans,
1673 self.
debugView(ccdExposure,
"postISRCCD")
1675 return pipeBase.Struct(
1676 exposure=ccdExposure,
1678 flattenedThumb=flattenedThumb,
1680 preInterpExposure=preInterpExp,
1681 outputExposure=ccdExposure,
1682 outputOssThumbnail=ossThumb,
1683 outputFlattenedThumbnail=flattenedThumb,
1684 outputStatistics=outputStatistics,
1688 """Ensure that the data returned by Butler is a fully constructed exp.
1690 ISR requires exposure-level image data for historical reasons, so
if we
1691 did
not recieve that
from Butler, construct it
from what we have,
1692 modifying the input
in place.
1697 The input data structure obtained
from Butler.
1699 `lsst.afw.image.DecoratedImageU`,
1700 or `lsst.afw.image.ImageF`
1701 camera : `lsst.afw.cameraGeom.camera`, optional
1702 The camera associated
with the image. Used to find the appropriate
1703 detector
if detector
is not already set.
1704 detectorNum : `int`, optional
1705 The detector
in the camera to attach,
if the detector
is not
1711 The re-constructed exposure,
with appropriate detector parameters.
1716 Raised
if the input data cannot be used to construct an exposure.
1718 if isinstance(inputExp, afwImage.DecoratedImageU):
1719 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1720 elif isinstance(inputExp, afwImage.ImageF):
1721 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1722 elif isinstance(inputExp, afwImage.MaskedImageF):
1723 inputExp = afwImage.makeExposure(inputExp)
1724 elif isinstance(inputExp, afwImage.Exposure):
1726 elif inputExp
is None:
1730 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1733 if inputExp.getDetector()
is None:
1734 if camera
is None or detectorNum
is None:
1735 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1736 'without a detector set.')
1737 inputExp.setDetector(camera[detectorNum])
1743 """Extract common calibration metadata values that will be written to
1749 Calibration to pull date information
from.
1754 Calibration creation date string to add to header.
1756 if hasattr(calib,
"getMetadata"):
1757 if 'CALIB_CREATION_DATE' in calib.getMetadata():
1758 return " ".join((calib.getMetadata().get(
"CALIB_CREATION_DATE",
"Unknown"),
1759 calib.getMetadata().get(
"CALIB_CREATION_TIME",
"Unknown")))
1761 return " ".join((calib.getMetadata().get(
"CALIB_CREATE_DATE",
"Unknown"),
1762 calib.getMetadata().get(
"CALIB_CREATE_TIME",
"Unknown")))
1764 return "Unknown Unknown"
1767 """Compare header keywords to confirm camera states match.
1772 Header for the exposure being processed.
1774 Calibration to be applied.
1776 Calib type
for log message.
1779 calibMetadata = calib.getMetadata()
1780 except AttributeError:
1782 for keyword
in self.config.cameraKeywordsToCompare:
1783 if keyword
in exposureMetadata
and keyword
in calibMetadata:
1784 if exposureMetadata[keyword] != calibMetadata[keyword]:
1785 if self.config.doRaiseOnCalibMismatch:
1786 raise RuntimeError(
"Sequencer mismatch for %s [%s]: exposure: %s calib: %s",
1788 exposureMetadata[keyword], calibMetadata[keyword])
1790 self.log.warning(
"Sequencer mismatch for %s [%s]: exposure: %s calib: %s",
1792 exposureMetadata[keyword], calibMetadata[keyword])
1794 self.log.debug(
"Sequencer keyword %s not found.", keyword)
1797 """Convert exposure image from uint16 to float.
1799 If the exposure does not need to be converted, the input
is
1800 immediately returned. For exposures that are converted to use
1801 floating point pixels, the variance
is set to unity
and the
1807 The raw exposure to be converted.
1812 The input ``exposure``, converted to floating point pixels.
1817 Raised
if the exposure type cannot be converted to float.
1820 if isinstance(exposure, afwImage.ExposureF):
1822 self.log.debug(
"Exposure already of type float.")
1824 if not hasattr(exposure,
"convertF"):
1825 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1827 newexposure = exposure.convertF()
1828 newexposure.variance[:] = 1
1829 newexposure.mask[:] = 0x0
1834 """Identify bad amplifiers, saturated and suspect pixels.
1839 Input exposure to be masked.
1841 Catalog of parameters defining the amplifier on this
1844 List of defects. Used to determine if the entire
1850 If this
is true, the entire amplifier area
is covered by
1851 defects
and unusable.
1854 maskedImage = ccdExposure.getMaskedImage()
1861 if defects
is not None:
1862 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1868 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1870 maskView = dataView.getMask()
1871 maskView |= maskView.getPlaneBitMask(
"BAD")
1879 if self.config.doSaturation
and not badAmp:
1880 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1881 if self.config.doSuspect
and not badAmp:
1882 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1883 if math.isfinite(self.config.saturation):
1884 limits.update({self.config.saturatedMaskName: self.config.saturation})
1886 for maskName, maskThreshold
in limits.items():
1887 if not math.isnan(maskThreshold):
1888 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1889 isrFunctions.makeThresholdMask(
1890 maskedImage=dataView,
1891 threshold=maskThreshold,
1898 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1900 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1901 self.config.suspectMaskName])
1902 if numpy.all(maskView.getArray() & maskVal > 0):
1904 maskView |= maskView.getPlaneBitMask(
"BAD")
1909 """Apply overscan correction in place.
1911 This method does initial pixel rejection of the overscan
1912 region. The overscan can also be optionally segmented to
1913 allow for discontinuous overscan responses to be fit
1914 separately. The actual overscan subtraction
is performed by
1915 the `lsst.ip.isr.overscan.OverscanTask`, which
is called here
1916 after the amplifier
is preprocessed.
1921 Exposure to have overscan correction performed.
1922 amp : `lsst.afw.cameraGeom.Amplifer`
1923 The amplifier to consider
while correcting the overscan.
1927 overscanResults : `lsst.pipe.base.Struct`
1928 Result struct
with components:
1931 Value
or fit subtracted
from the amplifier image data.
1934 Value
or fit subtracted
from the overscan image data.
1937 Image of the overscan region
with the overscan
1938 correction applied. This quantity
is used to estimate
1939 the amplifier read noise empirically.
1944 Median overscan fit value. (`float`)
1946 Clipped standard deviation of the overscan after
1947 correction. (`float`)
1952 Raised
if the ``amp`` does
not contain raw pixel information.
1956 lsst.ip.isr.overscan.OverscanTask
1958 if amp.getRawHorizontalOverscanBBox().isEmpty():
1959 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1963 overscanResults = self.overscan.
run(ccdExposure, amp)
1965 metadata = ccdExposure.getMetadata()
1966 ampName = amp.getName()
1968 keyBase =
"LSST ISR OVERSCAN"
1970 if isinstance(overscanResults.overscanMean, float):
1972 metadata[f
"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean
1973 metadata[f
"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian
1974 metadata[f
"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma
1976 metadata[f
"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean
1977 metadata[f
"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian
1978 metadata[f
"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma
1979 elif isinstance(overscanResults.overscanMean, tuple):
1981 metadata[f
"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0]
1982 metadata[f
"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0]
1983 metadata[f
"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0]
1985 metadata[f
"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1]
1986 metadata[f
"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1]
1987 metadata[f
"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1]
1989 metadata[f
"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0]
1990 metadata[f
"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0]
1991 metadata[f
"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0]
1993 metadata[f
"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1]
1994 metadata[f
"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1]
1995 metadata[f
"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1]
1997 self.log.warning(
"Unexpected type for overscan values; none added to header.")
1999 return overscanResults
2002 """Set the variance plane using the gain and read noise
2004 The read noise is calculated
from the ``overscanImage``
if the
2005 ``doEmpiricalReadNoise`` option
is set
in the configuration; otherwise
2006 the value
from the amplifier data
is used.
2011 Exposure to process.
2013 Amplifier detector data.
2015 Image of overscan, required only
for empirical read noise.
2017 PTC dataset containing the gains
and read noise.
2022 Raised
if either ``usePtcGains`` of ``usePtcReadNoise``
2023 are ``
True``, but ptcDataset
is not provided.
2025 Raised
if ```doEmpiricalReadNoise``
is ``
True`` but
2026 ``overscanImage``
is ``
None``.
2030 lsst.ip.isr.isrFunctions.updateVariance
2032 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2033 if self.config.usePtcGains:
2034 if ptcDataset
is None:
2035 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2037 gain = ptcDataset.gain[amp.getName()]
2038 self.log.info(
"Using gain from Photon Transfer Curve.")
2040 gain = amp.getGain()
2042 if math.isnan(gain):
2044 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2047 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2048 amp.getName(), gain, patchedGain)
2051 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2052 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(),
2053 [self.config.saturatedMaskName,
2054 self.config.suspectMaskName,
2056 allPixels = ampExposure.getWidth() * ampExposure.getHeight()
2057 if allPixels == badPixels:
2059 self.log.info(
"Skipping empirical read noise for amp %s. No good pixels.",
2062 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2064 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2065 stats = afwMath.StatisticsControl()
2066 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2067 readNoise = afwMath.makeStatistics(overscanImage.getImage(),
2068 afwMath.STDEVCLIP, stats).getValue()
2069 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2070 amp.getName(), readNoise)
2071 elif self.config.usePtcReadNoise:
2072 if ptcDataset
is None:
2073 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2075 readNoise = ptcDataset.noise[amp.getName()]
2076 self.log.info(
"Using read noise from Photon Transfer Curve.")
2078 readNoise = amp.getReadNoise()
2080 metadata = ampExposure.getMetadata()
2081 metadata[f
'LSST GAIN {amp.getName()}'] = gain
2082 metadata[f
'LSST READNOISE {amp.getName()}'] = readNoise
2084 isrFunctions.updateVariance(
2085 maskedImage=ampExposure.getMaskedImage(),
2087 readNoise=readNoise,
2091 """Identify and mask pixels with negative variance values.
2096 Exposure to process.
2100 lsst.ip.isr.isrFunctions.updateVariance
2102 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2103 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2104 exposure.mask.array[bad] |= maskPlane
2107 """Apply dark correction in place.
2112 Exposure to process.
2114 Dark exposure of the same size as ``exposure``.
2115 invert : `Bool`, optional
2116 If
True, re-add the dark to an already corrected image.
2121 Raised
if either ``exposure``
or ``darkExposure`` do
not
2122 have their dark time defined.
2126 lsst.ip.isr.isrFunctions.darkCorrection
2128 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2129 if math.isnan(expScale):
2130 raise RuntimeError(
"Exposure darktime is NAN.")
2131 if darkExposure.getInfo().getVisitInfo()
is not None \
2132 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2133 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2137 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2140 isrFunctions.darkCorrection(
2141 maskedImage=exposure.getMaskedImage(),
2142 darkMaskedImage=darkExposure.getMaskedImage(),
2144 darkScale=darkScale,
2146 trimToFit=self.config.doTrimToMatchCalib
2150 """Check if linearization is needed for the detector cameraGeom.
2152 Checks config.doLinearize and the linearity type of the first
2158 Detector to get linearity type
from.
2162 doLinearize : `Bool`
2163 If
True, linearization should be performed.
2165 return self.config.doLinearize
and \
2166 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2169 """Apply flat correction in place.
2174 Exposure to process.
2176 Flat exposure of the same size as ``exposure``.
2177 invert : `Bool`, optional
2178 If
True, unflatten an already flattened image.
2182 lsst.ip.isr.isrFunctions.flatCorrection
2184 isrFunctions.flatCorrection(
2185 maskedImage=exposure.getMaskedImage(),
2186 flatMaskedImage=flatExposure.getMaskedImage(),
2187 scalingType=self.config.flatScalingType,
2188 userScale=self.config.flatUserScale,
2190 trimToFit=self.config.doTrimToMatchCalib
2194 """Detect and mask saturated pixels in config.saturatedMaskName.
2199 Exposure to process. Only the amplifier DataSec is processed.
2201 Amplifier detector data.
2205 lsst.ip.isr.isrFunctions.makeThresholdMask
2207 if not math.isnan(amp.getSaturation()):
2208 maskedImage = exposure.getMaskedImage()
2209 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2210 isrFunctions.makeThresholdMask(
2211 maskedImage=dataView,
2212 threshold=amp.getSaturation(),
2214 maskName=self.config.saturatedMaskName,
2218 """Interpolate over saturated pixels, in place.
2220 This method should be called after `saturationDetection`, to
2221 ensure that the saturated pixels have been identified in the
2222 SAT mask. It should also be called after `assembleCcd`, since
2223 saturated regions may cross amplifier boundaries.
2228 Exposure to process.
2232 lsst.ip.isr.isrTask.saturationDetection
2233 lsst.ip.isr.isrFunctions.interpolateFromMask
2235 isrFunctions.interpolateFromMask(
2236 maskedImage=exposure.getMaskedImage(),
2237 fwhm=self.config.fwhm,
2238 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2239 maskNameList=list(self.config.saturatedMaskName),
2243 """Detect and mask suspect pixels in config.suspectMaskName.
2248 Exposure to process. Only the amplifier DataSec is processed.
2250 Amplifier detector data.
2254 lsst.ip.isr.isrFunctions.makeThresholdMask
2258 Suspect pixels are pixels whose value
is greater than
2259 amp.getSuspectLevel(). This
is intended to indicate pixels that may be
2260 affected by unknown systematics;
for example
if non-linearity
2261 corrections above a certain level are unstable then that would be a
2262 useful value
for suspectLevel. A value of `nan` indicates that no such
2263 level exists
and no pixels are to be masked
as suspicious.
2265 suspectLevel = amp.getSuspectLevel()
2266 if math.isnan(suspectLevel):
2269 maskedImage = exposure.getMaskedImage()
2270 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2271 isrFunctions.makeThresholdMask(
2272 maskedImage=dataView,
2273 threshold=suspectLevel,
2275 maskName=self.config.suspectMaskName,
2279 """Mask defects using mask plane "BAD", in place.
2284 Exposure to process.
2285 defectBaseList : defect-type
2291 Call this after CCD assembly, since defects may cross amplifier
2294 maskedImage = exposure.getMaskedImage()
2295 if not isinstance(defectBaseList, Defects):
2297 defectList =
Defects(defectBaseList)
2299 defectList = defectBaseList
2300 defectList.maskPixels(maskedImage, maskName=
"BAD")
2302 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2303 """Mask edge pixels with applicable mask plane.
2308 Exposure to process.
2309 numEdgePixels : `int`, optional
2310 Number of edge pixels to mask.
2311 maskPlane : `str`, optional
2312 Mask plane name to use.
2313 level : `str`, optional
2314 Level at which to mask edges.
2316 maskedImage = exposure.getMaskedImage()
2317 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2319 if numEdgePixels > 0:
2320 if level ==
'DETECTOR':
2321 boxes = [maskedImage.getBBox()]
2322 elif level ==
'AMP':
2323 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2328 subImage = maskedImage[box]
2329 box.grow(-numEdgePixels)
2331 SourceDetectionTask.setEdgeBits(
2337 """Mask and interpolate defects using mask plane "BAD", in place.
2342 Exposure to process.
2343 defectBaseList : defects-like
2344 List of defects to mask and interpolate. Can be
2349 lsst.ip.isr.isrTask.maskDefect
2352 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2353 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2354 isrFunctions.interpolateFromMask(
2355 maskedImage=exposure.getMaskedImage(),
2356 fwhm=self.config.fwhm,
2357 growSaturatedFootprints=0,
2358 maskNameList=[
"BAD"],
2362 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2367 Exposure to process.
2371 We mask over all non-finite values (NaN, inf), including those
2372 that are masked with other bits (because those may
or may
not be
2373 interpolated over later,
and we want to remove all NaN/infs).
2374 Despite this behaviour, the
"UNMASKEDNAN" mask plane
is used to
2375 preserve the historical name.
2377 maskedImage = exposure.getMaskedImage()
2380 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2381 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2382 numNans = maskNans(maskedImage, maskVal)
2383 self.metadata[
"NUMNANS"] = numNans
2385 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2388 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2394 Exposure to process.
2398 lsst.ip.isr.isrTask.maskNan
2401 isrFunctions.interpolateFromMask(
2402 maskedImage=exposure.getMaskedImage(),
2403 fwhm=self.config.fwhm,
2404 growSaturatedFootprints=0,
2405 maskNameList=["UNMASKEDNAN"],
2409 """Measure the image background in subgrids, for quality control.
2414 Exposure to process.
2416 Configuration object containing parameters on which background
2417 statistics and subgrids to use.
2419 if IsrQaConfig
is not None:
2420 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2421 IsrQaConfig.flatness.nIter)
2422 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2423 statsControl.setAndMask(maskVal)
2424 maskedImage = exposure.getMaskedImage()
2425 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2426 skyLevel = stats.getValue(afwMath.MEDIAN)
2427 skySigma = stats.getValue(afwMath.STDEVCLIP)
2428 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2429 metadata = exposure.getMetadata()
2430 metadata[
"SKYLEVEL"] = skyLevel
2431 metadata[
"SKYSIGMA"] = skySigma
2434 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2435 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2436 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2437 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2438 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2439 skyLevels = numpy.zeros((nX, nY))
2442 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2444 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2446 xLLC = xc - meshXHalf
2447 yLLC = yc - meshYHalf
2448 xURC = xc + meshXHalf - 1
2449 yURC = yc + meshYHalf - 1
2452 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2454 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2456 good = numpy.where(numpy.isfinite(skyLevels))
2457 skyMedian = numpy.median(skyLevels[good])
2458 flatness = (skyLevels[good] - skyMedian) / skyMedian
2459 flatness_rms = numpy.std(flatness)
2460 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2462 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2463 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2464 nX, nY, flatness_pp, flatness_rms)
2466 metadata[
"FLATNESS_PP"] = float(flatness_pp)
2467 metadata[
"FLATNESS_RMS"] = float(flatness_rms)
2468 metadata[
"FLATNESS_NGRIDS"] =
'%dx%d' % (nX, nY)
2469 metadata[
"FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2470 metadata[
"FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2473 """Set an approximate magnitude zero point for the exposure.
2478 Exposure to process.
2480 filterLabel = exposure.getFilter()
2481 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2483 if physicalFilter
in self.config.fluxMag0T1:
2484 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2486 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2487 fluxMag0 = self.config.defaultFluxMag0T1
2489 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2491 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2494 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2495 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2496 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2500 """Context manager that applies and removes flats and darks,
2501 if the task
is configured to apply them.
2506 Exposure to process.
2508 Flat exposure the same size
as ``exp``.
2510 Dark exposure the same size
as ``exp``.
2515 The flat
and dark corrected exposure.
2517 if self.config.doDark
and dark
is not None:
2519 if self.config.doFlat:
2524 if self.config.doFlat:
2526 if self.config.doDark
and dark
is not None:
2530 """Utility function to examine ISR exposure at different stages.
2537 State of processing to view.
2539 frame = getDebugFrame(self._display, stepname)
2541 display = getDisplay(frame)
2542 display.scale(
'asinh',
'zscale')
2543 display.mtv(exposure)
2544 prompt =
"Press Enter to continue [c]... "
2546 ans = input(prompt).lower()
2547 if ans
in (
"",
"c",):
2552 """A Detector-like object that supports returning gain and saturation level
2554 This is used when the input exposure does
not have a detector.
2559 Exposure to generate a fake amplifier
for.
2560 config : `lsst.ip.isr.isrTaskConfig`
2561 Configuration to apply to the fake amplifier.
2565 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
pexConfig doTrimToMatchCalib
pexConfig doSaturationInterpolation
pexConfig maskListToInterpolate
pexConfig saturatedMaskName
pexConfig doBiasBeforeOverscan
pexConfig doNanInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def extractCalibDate(calib)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def maskNegativeVariance(self, exposure)
def compareCameraKeywords(self, exposureMetadata, calib, calibName)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, deferredChargeCalib=None)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)