22__all__ = [
"IsrTask",
"IsrTaskConfig"]
31import lsst.pipe.base
as pipeBase
32import lsst.pipe.base.connectionTypes
as cT
34from contextlib
import contextmanager
35from lsstDebug
import getDebugFrame
40from lsst.utils.timer
import timeMethod
42from .
import isrFunctions
44from .
import linearize
45from .defects
import Defects
47from .assembleCcdTask
import AssembleCcdTask
48from .crosstalk
import CrosstalkTask, CrosstalkCalib
49from .fringe
import FringeTask
50from .isr
import maskNans
51from .masking
import MaskingTask
52from .overscan
import OverscanCorrectionTask
53from .straylight
import StrayLightTask
54from .vignette
import VignetteTask
55from .ampOffset
import AmpOffsetTask
56from .deferredCharge
import DeferredChargeTask
57from .isrStatistics
import IsrStatisticsTask
58from lsst.daf.butler
import DimensionGraph
62 """Lookup function to identify crosstalkSource entries.
64 This should return an empty list under most circumstances. Only
65 when inter-chip crosstalk has been identified should this be
72 registry : `lsst.daf.butler.Registry`
73 Butler registry to query.
74 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
75 Data id to transform to identify crosstalkSources. The
76 ``detector`` entry will be stripped.
77 collections : `lsst.daf.butler.CollectionSearch`
78 Collections to search through.
82 results : `list` [`lsst.daf.butler.DatasetRef`]
83 List of datasets that match the query that will be used
as
86 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"exposure"]))
87 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
94 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records))
for ref
in results]
98 dimensions={
"instrument",
"exposure",
"detector"},
100 ccdExposure = cT.Input(
102 doc=
"Input exposure to process.",
103 storageClass=
"Exposure",
104 dimensions=[
"instrument",
"exposure",
"detector"],
106 camera = cT.PrerequisiteInput(
108 storageClass=
"Camera",
109 doc=
"Input camera to construct complete exposures.",
110 dimensions=[
"instrument"],
114 crosstalk = cT.PrerequisiteInput(
116 doc=
"Input crosstalk object",
117 storageClass=
"CrosstalkCalib",
118 dimensions=[
"instrument",
"detector"],
122 crosstalkSources = cT.PrerequisiteInput(
123 name=
"isrOverscanCorrected",
124 doc=
"Overscan corrected input images.",
125 storageClass=
"Exposure",
126 dimensions=[
"instrument",
"exposure",
"detector"],
129 lookupFunction=crosstalkSourceLookup,
132 bias = cT.PrerequisiteInput(
134 doc=
"Input bias calibration.",
135 storageClass=
"ExposureF",
136 dimensions=[
"instrument",
"detector"],
139 dark = cT.PrerequisiteInput(
141 doc=
"Input dark calibration.",
142 storageClass=
"ExposureF",
143 dimensions=[
"instrument",
"detector"],
146 flat = cT.PrerequisiteInput(
148 doc=
"Input flat calibration.",
149 storageClass=
"ExposureF",
150 dimensions=[
"instrument",
"physical_filter",
"detector"],
153 ptc = cT.PrerequisiteInput(
155 doc=
"Input Photon Transfer Curve dataset",
156 storageClass=
"PhotonTransferCurveDataset",
157 dimensions=[
"instrument",
"detector"],
160 fringes = cT.PrerequisiteInput(
162 doc=
"Input fringe calibration.",
163 storageClass=
"ExposureF",
164 dimensions=[
"instrument",
"physical_filter",
"detector"],
168 strayLightData = cT.PrerequisiteInput(
170 doc=
"Input stray light calibration.",
171 storageClass=
"StrayLightData",
172 dimensions=[
"instrument",
"physical_filter",
"detector"],
177 bfKernel = cT.PrerequisiteInput(
179 doc=
"Input brighter-fatter kernel.",
180 storageClass=
"NumpyArray",
181 dimensions=[
"instrument"],
185 newBFKernel = cT.PrerequisiteInput(
186 name=
'brighterFatterKernel',
187 doc=
"Newer complete kernel + gain solutions.",
188 storageClass=
"BrighterFatterKernel",
189 dimensions=[
"instrument",
"detector"],
193 defects = cT.PrerequisiteInput(
195 doc=
"Input defect tables.",
196 storageClass=
"Defects",
197 dimensions=[
"instrument",
"detector"],
200 linearizer = cT.PrerequisiteInput(
202 storageClass=
"Linearizer",
203 doc=
"Linearity correction calibration.",
204 dimensions=[
"instrument",
"detector"],
208 opticsTransmission = cT.PrerequisiteInput(
209 name=
"transmission_optics",
210 storageClass=
"TransmissionCurve",
211 doc=
"Transmission curve due to the optics.",
212 dimensions=[
"instrument"],
215 filterTransmission = cT.PrerequisiteInput(
216 name=
"transmission_filter",
217 storageClass=
"TransmissionCurve",
218 doc=
"Transmission curve due to the filter.",
219 dimensions=[
"instrument",
"physical_filter"],
222 sensorTransmission = cT.PrerequisiteInput(
223 name=
"transmission_sensor",
224 storageClass=
"TransmissionCurve",
225 doc=
"Transmission curve due to the sensor.",
226 dimensions=[
"instrument",
"detector"],
229 atmosphereTransmission = cT.PrerequisiteInput(
230 name=
"transmission_atmosphere",
231 storageClass=
"TransmissionCurve",
232 doc=
"Transmission curve due to the atmosphere.",
233 dimensions=[
"instrument"],
236 illumMaskedImage = cT.PrerequisiteInput(
238 doc=
"Input illumination correction.",
239 storageClass=
"MaskedImageF",
240 dimensions=[
"instrument",
"physical_filter",
"detector"],
243 deferredChargeCalib = cT.PrerequisiteInput(
245 doc=
"Deferred charge/CTI correction dataset.",
246 storageClass=
"IsrCalib",
247 dimensions=[
"instrument",
"detector"],
251 outputExposure = cT.Output(
253 doc=
"Output ISR processed exposure.",
254 storageClass=
"Exposure",
255 dimensions=[
"instrument",
"exposure",
"detector"],
257 preInterpExposure = cT.Output(
258 name=
'preInterpISRCCD',
259 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
260 storageClass=
"ExposureF",
261 dimensions=[
"instrument",
"exposure",
"detector"],
263 outputOssThumbnail = cT.Output(
265 doc=
"Output Overscan-subtracted thumbnail image.",
266 storageClass=
"Thumbnail",
267 dimensions=[
"instrument",
"exposure",
"detector"],
269 outputFlattenedThumbnail = cT.Output(
270 name=
"FlattenedThumb",
271 doc=
"Output flat-corrected thumbnail image.",
272 storageClass=
"Thumbnail",
273 dimensions=[
"instrument",
"exposure",
"detector"],
275 outputStatistics = cT.Output(
276 name=
"isrStatistics",
277 doc=
"Output of additional statistics table.",
278 storageClass=
"StructuredDataDict",
279 dimensions=[
"instrument",
"exposure",
"detector"],
285 if config.doBias
is not True:
286 self.prerequisiteInputs.remove(
"bias")
287 if config.doLinearize
is not True:
288 self.prerequisiteInputs.remove(
"linearizer")
289 if config.doCrosstalk
is not True:
290 self.prerequisiteInputs.remove(
"crosstalkSources")
291 self.prerequisiteInputs.remove(
"crosstalk")
292 if config.doBrighterFatter
is not True:
293 self.prerequisiteInputs.remove(
"bfKernel")
294 self.prerequisiteInputs.remove(
"newBFKernel")
295 if config.doDefect
is not True:
296 self.prerequisiteInputs.remove(
"defects")
297 if config.doDark
is not True:
298 self.prerequisiteInputs.remove(
"dark")
299 if config.doFlat
is not True:
300 self.prerequisiteInputs.remove(
"flat")
301 if config.doFringe
is not True:
302 self.prerequisiteInputs.remove(
"fringes")
303 if config.doStrayLight
is not True:
304 self.prerequisiteInputs.remove(
"strayLightData")
305 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
306 self.prerequisiteInputs.remove(
"ptc")
307 if config.doAttachTransmissionCurve
is not True:
308 self.prerequisiteInputs.remove(
"opticsTransmission")
309 self.prerequisiteInputs.remove(
"filterTransmission")
310 self.prerequisiteInputs.remove(
"sensorTransmission")
311 self.prerequisiteInputs.remove(
"atmosphereTransmission")
313 if config.doUseOpticsTransmission
is not True:
314 self.prerequisiteInputs.remove(
"opticsTransmission")
315 if config.doUseFilterTransmission
is not True:
316 self.prerequisiteInputs.remove(
"filterTransmission")
317 if config.doUseSensorTransmission
is not True:
318 self.prerequisiteInputs.remove(
"sensorTransmission")
319 if config.doUseAtmosphereTransmission
is not True:
320 self.prerequisiteInputs.remove(
"atmosphereTransmission")
321 if config.doIlluminationCorrection
is not True:
322 self.prerequisiteInputs.remove(
"illumMaskedImage")
323 if config.doDeferredCharge
is not True:
324 self.prerequisiteInputs.remove(
"deferredChargeCalib")
326 if config.doWrite
is not True:
327 self.outputs.remove(
"outputExposure")
328 self.outputs.remove(
"preInterpExposure")
329 self.outputs.remove(
"outputFlattenedThumbnail")
330 self.outputs.remove(
"outputOssThumbnail")
331 self.outputs.remove(
"outputStatistics")
333 if config.doSaveInterpPixels
is not True:
334 self.outputs.remove(
"preInterpExposure")
335 if config.qa.doThumbnailOss
is not True:
336 self.outputs.remove(
"outputOssThumbnail")
337 if config.qa.doThumbnailFlattened
is not True:
338 self.outputs.remove(
"outputFlattenedThumbnail")
339 if config.doCalculateStatistics
is not True:
340 self.outputs.remove(
"outputStatistics")
344 pipelineConnections=IsrTaskConnections):
345 """Configuration parameters for IsrTask.
347 Items are grouped in the order
in which they are executed by the task.
349 datasetType = pexConfig.Field(
351 doc="Dataset type for input data; users will typically leave this alone, "
352 "but camera-specific ISR tasks will override it",
356 fallbackFilterName = pexConfig.Field(
358 doc=
"Fallback default filter name for calibrations.",
361 useFallbackDate = pexConfig.Field(
363 doc=
"Pass observation date when using fallback filter.",
366 expectWcs = pexConfig.Field(
369 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
371 fwhm = pexConfig.Field(
373 doc=
"FWHM of PSF in arcseconds.",
376 qa = pexConfig.ConfigField(
378 doc=
"QA related configuration options.",
380 doHeaderProvenance = pexConfig.Field(
383 doc=
"Write calibration identifiers into output exposure header?",
387 doRaiseOnCalibMismatch = pexConfig.Field(
390 doc=
"Should IsrTask halt if exposure and calibration header values do not match?",
392 cameraKeywordsToCompare = pexConfig.ListField(
394 doc=
"List of header keywords to compare between exposure and calibrations.",
399 doConvertIntToFloat = pexConfig.Field(
401 doc=
"Convert integer raw images to floating point values?",
406 doSaturation = pexConfig.Field(
408 doc=
"Mask saturated pixels? NB: this is totally independent of the"
409 " interpolation option - this is ONLY setting the bits in the mask."
410 " To have them interpolated make sure doSaturationInterpolation=True",
413 saturatedMaskName = pexConfig.Field(
415 doc=
"Name of mask plane to use in saturation detection and interpolation",
418 saturation = pexConfig.Field(
420 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
421 default=float(
"NaN"),
423 growSaturationFootprintSize = pexConfig.Field(
425 doc=
"Number of pixels by which to grow the saturation footprints",
430 doSuspect = pexConfig.Field(
432 doc=
"Mask suspect pixels?",
435 suspectMaskName = pexConfig.Field(
437 doc=
"Name of mask plane to use for suspect pixels",
440 numEdgeSuspect = pexConfig.Field(
442 doc=
"Number of edge pixels to be flagged as untrustworthy.",
445 edgeMaskLevel = pexConfig.ChoiceField(
447 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
450 'DETECTOR':
'Mask only the edges of the full detector.',
451 'AMP':
'Mask edges of each amplifier.',
456 doSetBadRegions = pexConfig.Field(
458 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
461 badStatistic = pexConfig.ChoiceField(
463 doc=
"How to estimate the average value for BAD regions.",
466 "MEANCLIP":
"Correct using the (clipped) mean of good data",
467 "MEDIAN":
"Correct using the median of the good data",
472 doOverscan = pexConfig.Field(
474 doc=
"Do overscan subtraction?",
477 overscan = pexConfig.ConfigurableField(
478 target=OverscanCorrectionTask,
479 doc=
"Overscan subtraction task for image segments.",
483 doAssembleCcd = pexConfig.Field(
486 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
488 assembleCcd = pexConfig.ConfigurableField(
489 target=AssembleCcdTask,
490 doc=
"CCD assembly task",
494 doAssembleIsrExposures = pexConfig.Field(
497 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
499 doTrimToMatchCalib = pexConfig.Field(
502 doc=
"Trim raw data to match calibration bounding boxes?"
506 doBias = pexConfig.Field(
508 doc=
"Apply bias frame correction?",
511 biasDataProductName = pexConfig.Field(
513 doc=
"Name of the bias data product",
516 doBiasBeforeOverscan = pexConfig.Field(
518 doc=
"Reverse order of overscan and bias correction.",
523 doDeferredCharge = pexConfig.Field(
525 doc=
"Apply deferred charge correction?",
528 deferredChargeCorrection = pexConfig.ConfigurableField(
529 target=DeferredChargeTask,
530 doc=
"Deferred charge correction task.",
534 doVariance = pexConfig.Field(
536 doc=
"Calculate variance?",
539 gain = pexConfig.Field(
541 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
542 default=float(
"NaN"),
544 readNoise = pexConfig.Field(
546 doc=
"The read noise to use if no Detector is present in the Exposure",
549 doEmpiricalReadNoise = pexConfig.Field(
552 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
554 usePtcReadNoise = pexConfig.Field(
557 doc=
"Use readnoise values from the Photon Transfer Curve?"
559 maskNegativeVariance = pexConfig.Field(
562 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
563 "in the measurement of the overscan at an edge due to the data falling off faster "
564 "than the overscan model can account for it."
566 negativeVarianceMaskName = pexConfig.Field(
569 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
572 doLinearize = pexConfig.Field(
574 doc=
"Correct for nonlinearity of the detector's response?",
579 doCrosstalk = pexConfig.Field(
581 doc=
"Apply intra-CCD crosstalk correction?",
584 doCrosstalkBeforeAssemble = pexConfig.Field(
586 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
589 crosstalk = pexConfig.ConfigurableField(
590 target=CrosstalkTask,
591 doc=
"Intra-CCD crosstalk correction",
595 doDefect = pexConfig.Field(
597 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
600 doNanMasking = pexConfig.Field(
602 doc=
"Mask non-finite (NAN, inf) pixels?",
605 doWidenSaturationTrails = pexConfig.Field(
607 doc=
"Widen bleed trails based on their width?",
612 doBrighterFatter = pexConfig.Field(
615 doc=
"Apply the brighter-fatter correction?"
617 doFluxConservingBrighterFatterCorrection = pexConfig.Field(
620 doc=
"Apply the flux-conserving BFE correction by Miller et al.?"
622 brighterFatterLevel = pexConfig.ChoiceField(
625 doc=
"The level at which to correct for brighter-fatter.",
627 "AMP":
"Every amplifier treated separately.",
628 "DETECTOR":
"One kernel per detector",
631 brighterFatterMaxIter = pexConfig.Field(
634 doc=
"Maximum number of iterations for the brighter-fatter correction"
636 brighterFatterThreshold = pexConfig.Field(
639 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
640 "absolute value of the difference between the current corrected image and the one "
641 "from the previous iteration summed over all the pixels."
643 brighterFatterApplyGain = pexConfig.Field(
646 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
648 brighterFatterMaskListToInterpolate = pexConfig.ListField(
650 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
652 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
654 brighterFatterMaskGrowSize = pexConfig.Field(
657 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
658 "when brighter-fatter correction is applied."
662 doDark = pexConfig.Field(
664 doc=
"Apply dark frame correction?",
667 darkDataProductName = pexConfig.Field(
669 doc=
"Name of the dark data product",
674 doStrayLight = pexConfig.Field(
676 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
679 strayLight = pexConfig.ConfigurableField(
680 target=StrayLightTask,
681 doc=
"y-band stray light correction"
685 doFlat = pexConfig.Field(
687 doc=
"Apply flat field correction?",
690 flatDataProductName = pexConfig.Field(
692 doc=
"Name of the flat data product",
695 flatScalingType = pexConfig.ChoiceField(
697 doc=
"The method for scaling the flat on the fly.",
700 "USER":
"Scale by flatUserScale",
701 "MEAN":
"Scale by the inverse of the mean",
702 "MEDIAN":
"Scale by the inverse of the median",
705 flatUserScale = pexConfig.Field(
707 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
710 doTweakFlat = pexConfig.Field(
712 doc=
"Tweak flats to match observed amplifier ratios?",
718 doApplyGains = pexConfig.Field(
720 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
723 usePtcGains = pexConfig.Field(
725 doc=
"Use the gain values from the Photon Transfer Curve?",
728 normalizeGains = pexConfig.Field(
730 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
735 doFringe = pexConfig.Field(
737 doc=
"Apply fringe correction?",
740 fringe = pexConfig.ConfigurableField(
742 doc=
"Fringe subtraction task",
744 fringeAfterFlat = pexConfig.Field(
746 doc=
"Do fringe subtraction after flat-fielding?",
751 doAmpOffset = pexConfig.Field(
752 doc=
"Calculate and apply amp offset corrections?",
756 ampOffset = pexConfig.ConfigurableField(
757 doc=
"Amp offset correction task.",
758 target=AmpOffsetTask,
762 doMeasureBackground = pexConfig.Field(
764 doc=
"Measure the background level on the reduced image?",
769 doCameraSpecificMasking = pexConfig.Field(
771 doc=
"Mask camera-specific bad regions?",
774 masking = pexConfig.ConfigurableField(
780 doInterpolate = pexConfig.Field(
782 doc=
"Interpolate masked pixels?",
785 doSaturationInterpolation = pexConfig.Field(
787 doc=
"Perform interpolation over pixels masked as saturated?"
788 " NB: This is independent of doSaturation; if that is False this plane"
789 " will likely be blank, resulting in a no-op here.",
792 doNanInterpolation = pexConfig.Field(
794 doc=
"Perform interpolation over pixels masked as NaN?"
795 " NB: This is independent of doNanMasking; if that is False this plane"
796 " will likely be blank, resulting in a no-op here.",
799 doNanInterpAfterFlat = pexConfig.Field(
801 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
802 "also have to interpolate them before flat-fielding."),
805 maskListToInterpolate = pexConfig.ListField(
807 doc=
"List of mask planes that should be interpolated.",
808 default=[
'SAT',
'BAD'],
810 doSaveInterpPixels = pexConfig.Field(
812 doc=
"Save a copy of the pre-interpolated pixel values?",
817 fluxMag0T1 = pexConfig.DictField(
820 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
821 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
824 defaultFluxMag0T1 = pexConfig.Field(
826 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
827 default=pow(10.0, 0.4*28.0)
831 doVignette = pexConfig.Field(
833 doc=(
"Compute and attach the validPolygon defining the unvignetted region to the exposure "
834 "according to vignetting parameters?"),
837 doMaskVignettePolygon = pexConfig.Field(
839 doc=(
"Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
843 vignetteValue = pexConfig.Field(
845 doc=
"Value to replace image array pixels with in the vignetted region? Ignored if None.",
849 vignette = pexConfig.ConfigurableField(
851 doc=
"Vignetting task.",
855 doAttachTransmissionCurve = pexConfig.Field(
858 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
860 doUseOpticsTransmission = pexConfig.Field(
863 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
865 doUseFilterTransmission = pexConfig.Field(
868 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
870 doUseSensorTransmission = pexConfig.Field(
873 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
875 doUseAtmosphereTransmission = pexConfig.Field(
878 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
882 doIlluminationCorrection = pexConfig.Field(
885 doc=
"Perform illumination correction?"
887 illuminationCorrectionDataProductName = pexConfig.Field(
889 doc=
"Name of the illumination correction data product.",
892 illumScale = pexConfig.Field(
894 doc=
"Scale factor for the illumination correction.",
897 illumFilters = pexConfig.ListField(
900 doc=
"Only perform illumination correction for these filters."
904 doStandardStatistics = pexConfig.Field(
906 doc=
"Should standard image quality statistics be calculated?",
910 doCalculateStatistics = pexConfig.Field(
912 doc=
"Should additional ISR statistics be calculated?",
915 isrStats = pexConfig.ConfigurableField(
916 target=IsrStatisticsTask,
917 doc=
"Task to calculate additional statistics.",
922 doWrite = pexConfig.Field(
924 doc=
"Persist postISRCCD?",
931 raise ValueError(
"You may not specify both doFlat and doApplyGains")
933 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
943 """Apply common instrument signature correction algorithms to a raw frame.
945 The process for correcting imaging data
is very similar
from
946 camera to camera. This task provides a vanilla implementation of
947 doing these corrections, including the ability to turn certain
948 corrections off
if they are
not needed. The inputs to the primary
949 method, `
run()`, are a raw exposure to be corrected
and the
950 calibration data products. The raw input
is a single chip sized
951 mosaic of all amps including overscans
and other non-science
954 The __init__ method sets up the subtasks
for ISR processing, using
960 Positional arguments passed to the Task constructor.
961 None used at this time.
962 kwargs : `dict`, optional
963 Keyword arguments passed on to the Task constructor.
964 None used at this time.
966 ConfigClass = IsrTaskConfig
971 self.makeSubtask(
"assembleCcd")
972 self.makeSubtask(
"crosstalk")
973 self.makeSubtask(
"strayLight")
974 self.makeSubtask(
"fringe")
975 self.makeSubtask(
"masking")
976 self.makeSubtask(
"overscan")
977 self.makeSubtask(
"vignette")
978 self.makeSubtask(
"ampOffset")
979 self.makeSubtask(
"deferredChargeCorrection")
980 self.makeSubtask(
"isrStats")
983 inputs = butlerQC.get(inputRefs)
986 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
987 except Exception
as e:
988 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
991 detector = inputs[
'ccdExposure'].getDetector()
993 if self.config.doCrosstalk
is True:
996 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
997 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
998 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
1000 coeffVector = (self.config.crosstalk.crosstalkValues
1001 if self.config.crosstalk.useConfigCoefficients
else None)
1002 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1003 inputs[
'crosstalk'] = crosstalkCalib
1004 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
1005 if 'crosstalkSources' not in inputs:
1006 self.log.warning(
"No crosstalkSources found for chip with interChip terms!")
1009 if 'linearizer' in inputs:
1010 if isinstance(inputs[
'linearizer'], dict):
1012 linearizer.fromYaml(inputs[
'linearizer'])
1013 self.log.warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1014 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1018 self.log.warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1020 linearizer = inputs[
'linearizer']
1021 linearizer.log = self.log
1022 inputs[
'linearizer'] = linearizer
1025 self.log.warning(
"Constructing linearizer from cameraGeom information.")
1027 if self.config.doDefect
is True:
1028 if "defects" in inputs
and inputs[
'defects']
is not None:
1032 if not isinstance(inputs[
"defects"], Defects):
1033 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1037 if self.config.doBrighterFatter:
1038 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1039 if brighterFatterKernel
is None:
1040 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1042 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1044 detName = detector.getName()
1045 level = brighterFatterKernel.level
1048 inputs[
'bfGains'] = brighterFatterKernel.gain
1049 if self.config.brighterFatterLevel ==
'DETECTOR':
1050 if level ==
'DETECTOR':
1051 if detName
in brighterFatterKernel.detKernels:
1052 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1054 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1055 elif level ==
'AMP':
1056 self.log.warning(
"Making DETECTOR level kernel from AMP based brighter "
1058 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1059 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1060 elif self.config.brighterFatterLevel ==
'AMP':
1061 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1063 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
1064 expId = inputs[
'ccdExposure'].info.id
1065 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1067 assembler=self.assembleCcd
1068 if self.config.doAssembleIsrExposures
else None)
1070 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1072 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
1073 if 'strayLightData' not in inputs:
1074 inputs[
'strayLightData'] =
None
1076 if self.config.doHeaderProvenance:
1078 exposureMetadata = inputs[
'ccdExposure'].getMetadata()
1079 for inputName
in sorted(inputs.keys()):
1080 reference = getattr(inputRefs, inputName,
None)
1081 if reference
is not None and hasattr(reference,
"run"):
1082 runKey = f
"LSST CALIB RUN {inputName.upper()}"
1083 runValue = reference.run
1084 idKey = f
"LSST CALIB UUID {inputName.upper()}"
1085 idValue =
str(reference.id)
1087 exposureMetadata[runKey] = runValue
1088 exposureMetadata[idKey] = idValue
1090 outputs = self.
run(**inputs)
1091 butlerQC.put(outputs, outputRefs)
1094 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1095 crosstalk=None, crosstalkSources=None,
1096 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1097 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1098 sensorTransmission=
None, atmosphereTransmission=
None,
1099 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1100 deferredChargeCalib=
None,
1102 """Perform instrument signature removal on an exposure.
1104 Steps included in the ISR processing,
in order performed, are:
1106 - saturation
and suspect pixel masking
1107 - overscan subtraction
1108 - CCD assembly of individual amplifiers
1110 - variance image construction
1111 - linearization of non-linear response
1113 - brighter-fatter correction
1116 - stray light subtraction
1118 - masking of known defects
and camera specific features
1119 - vignette calculation
1120 - appending transmission curve
and distortion model
1125 The raw exposure that
is to be run through ISR. The
1126 exposure
is modified by this method.
1128 The camera geometry
for this exposure. Required
if
1129 one
or more of ``ccdExposure``, ``bias``, ``dark``,
or
1130 ``flat`` does
not have an associated detector.
1132 Bias calibration frame.
1134 Functor
for linearization.
1136 Calibration
for crosstalk.
1137 crosstalkSources : `list`, optional
1138 List of possible crosstalk sources.
1140 Dark calibration frame.
1142 Flat calibration frame.
1144 Photon transfer curve dataset,
with, e.g., gains
1146 bfKernel : `numpy.ndarray`, optional
1147 Brighter-fatter kernel.
1148 bfGains : `dict` of `float`, optional
1149 Gains used to override the detector
's nominal gains for the
1150 brighter-fatter correction. A dict keyed by amplifier name for
1151 the detector
in question.
1154 fringes : `lsst.pipe.base.Struct`, optional
1155 Struct containing the fringe correction data,
with
1161 random seed derived
from the ``ccdExposureId``
for random
1162 number generator (`numpy.uint32`)
1164 A ``TransmissionCurve`` that represents the throughput of the,
1165 optics, to be evaluated
in focal-plane coordinates.
1167 A ``TransmissionCurve`` that represents the throughput of the
1168 filter itself, to be evaluated
in focal-plane coordinates.
1170 A ``TransmissionCurve`` that represents the throughput of the
1171 sensor itself, to be evaluated
in post-assembly trimmed detector
1174 A ``TransmissionCurve`` that represents the throughput of the
1175 atmosphere, assumed to be spatially constant.
1176 detectorNum : `int`, optional
1177 The integer number
for the detector to process.
1178 strayLightData : `object`, optional
1179 Opaque object containing calibration information
for stray-light
1180 correction. If `
None`, no correction will be performed.
1182 Illumination correction image.
1186 result : `lsst.pipe.base.Struct`
1187 Result struct
with component:
1190 The fully ISR corrected exposure.
1195 Thumbnail image of the exposure after overscan subtraction.
1198 Thumbnail image of the exposure after flat-field correction.
1200 ``outputStatistics``
1201 Values of the additional statistics calculated.
1206 Raised
if a configuration option
is set to `
True`, but the
1207 required calibration data has
not been specified.
1211 The current processed exposure can be viewed by setting the
1212 appropriate `lsstDebug` entries
in the ``debug.display``
1213 dictionary. The names of these entries correspond to some of
1214 the `IsrTaskConfig` Boolean options,
with the value denoting the
1215 frame to use. The exposure
is shown inside the matching
1216 option check
and after the processing of that step has
1217 finished. The steps
with debug points are:
1228 In addition, setting the ``postISRCCD`` entry displays the
1229 exposure after all ISR processing has finished.
1232 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1237 ccd = ccdExposure.getDetector()
1238 filterLabel = ccdExposure.getFilter()
1239 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1242 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1243 ccd = [
FakeAmp(ccdExposure, self.config)]
1246 if self.config.doBias
and bias
is None:
1247 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1249 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1250 if self.config.doBrighterFatter
and bfKernel
is None:
1251 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1252 if self.config.doDark
and dark
is None:
1253 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1254 if self.config.doFlat
and flat
is None:
1255 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1256 if self.config.doDefect
and defects
is None:
1257 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1258 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1259 and fringes.fringes
is None):
1264 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1265 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1266 and illumMaskedImage
is None):
1267 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1268 if (self.config.doDeferredCharge
and deferredChargeCalib
is None):
1269 raise RuntimeError(
"Must supply a deferred charge calibration if config.doDeferredCharge=True.")
1271 if self.config.doHeaderProvenance:
1274 exposureMetadata = ccdExposure.getMetadata()
1275 if self.config.doBias:
1278 if self.config.doBrighterFatter:
1281 if self.config.doCrosstalk:
1282 exposureMetadata[
"LSST CALIB DATE CROSSTALK"] = self.
extractCalibDate(crosstalk)
1284 if self.config.doDark:
1287 if self.config.doDefect:
1288 exposureMetadata[
"LSST CALIB DATE DEFECTS"] = self.
extractCalibDate(defects)
1290 if self.config.doDeferredCharge:
1291 exposureMetadata[
"LSST CALIB DATE CTI"] = self.
extractCalibDate(deferredChargeCalib)
1293 if self.config.doFlat:
1296 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters):
1297 exposureMetadata[
"LSST CALIB DATE FRINGE"] = self.
extractCalibDate(fringes.fringes)
1299 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters):
1300 exposureMetadata[
"LSST CALIB DATE ILLUMINATION"] = self.
extractCalibDate(illumMaskedImage)
1303 exposureMetadata[
"LSST CALIB DATE LINEARIZER"] = self.
extractCalibDate(linearizer)
1305 if self.config.usePtcGains
or self.config.usePtcReadNoise:
1308 if self.config.doStrayLight:
1309 exposureMetadata[
"LSST CALIB DATE STRAYLIGHT"] = self.
extractCalibDate(strayLightData)
1311 if self.config.doAttachTransmissionCurve:
1312 exposureMetadata[
"LSST CALIB DATE OPTICS_TR"] = self.
extractCalibDate(opticsTransmission)
1313 exposureMetadata[
"LSST CALIB DATE FILTER_TR"] = self.
extractCalibDate(filterTransmission)
1314 exposureMetadata[
"LSST CALIB DATE SENSOR_TR"] = self.
extractCalibDate(sensorTransmission)
1315 exposureMetadata[
"LSST CALIB DATE ATMOSP_TR"] = self.
extractCalibDate(atmosphereTransmission)
1318 if self.config.doConvertIntToFloat:
1319 self.log.info(
"Converting exposure to floating point values.")
1322 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1323 self.log.info(
"Applying bias correction.")
1324 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1325 trimToFit=self.config.doTrimToMatchCalib)
1331 if self.config.doOverscan
and self.config.overscan.doParallelOverscan:
1333 self.overscan.maskParallelOverscan(ccdExposure, ccd)
1338 if ccdExposure.getBBox().contains(amp.getBBox()):
1343 if self.config.doOverscan
and not badAmp:
1346 self.log.debug(
"Corrected overscan for amplifier %s.", amp.getName())
1347 if overscanResults
is not None and \
1348 self.config.qa
is not None and self.config.qa.saveStats
is True:
1349 if isinstance(overscanResults.overscanMean, float):
1351 mean = overscanResults.overscanMean
1352 sigma = overscanResults.overscanSigma
1353 residMean = overscanResults.residualMean
1354 residSigma = overscanResults.residualSigma
1358 mean = overscanResults.overscanMean[0]
1359 sigma = overscanResults.overscanSigma[0]
1360 residMean = overscanResults.residualMean[0]
1361 residSigma = overscanResults.residualSigma[0]
1363 self.metadata[f
"FIT MEDIAN {amp.getName()}"] = mean
1364 self.metadata[f
"FIT STDEV {amp.getName()}"] = sigma
1365 self.log.debug(
" Overscan stats for amplifer %s: %f +/- %f",
1366 amp.getName(), mean, sigma)
1368 self.metadata[f
"RESIDUAL MEDIAN {amp.getName()}"] = residMean
1369 self.metadata[f
"RESIDUAL STDEV {amp.getName()}"] = residSigma
1370 self.log.debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1371 amp.getName(), residMean, residSigma)
1373 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
1376 self.log.warning(
"Amplifier %s is bad.", amp.getName())
1377 overscanResults =
None
1379 overscans.append(overscanResults
if overscanResults
is not None else None)
1381 self.log.info(
"Skipped OSCAN for %s.", amp.getName())
1383 if self.config.doDeferredCharge:
1384 self.log.info(
"Applying deferred charge/CTI correction.")
1385 self.deferredChargeCorrection.
run(ccdExposure, deferredChargeCalib)
1386 self.
debugView(ccdExposure,
"doDeferredCharge")
1388 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1389 self.log.info(
"Applying crosstalk correction.")
1390 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1391 crosstalkSources=crosstalkSources, camera=camera)
1392 self.
debugView(ccdExposure,
"doCrosstalk")
1394 if self.config.doAssembleCcd:
1395 self.log.info(
"Assembling CCD from amplifiers.")
1396 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1398 if self.config.expectWcs
and not ccdExposure.getWcs():
1399 self.log.warning(
"No WCS found in input exposure.")
1400 self.
debugView(ccdExposure,
"doAssembleCcd")
1403 if self.config.qa.doThumbnailOss:
1404 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1406 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1407 self.log.info(
"Applying bias correction.")
1408 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1409 trimToFit=self.config.doTrimToMatchCalib)
1412 if self.config.doVariance:
1413 for amp, overscanResults
in zip(ccd, overscans):
1414 if ccdExposure.getBBox().contains(amp.getBBox()):
1415 self.log.debug(
"Constructing variance map for amplifer %s.", amp.getName())
1416 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1417 if overscanResults
is not None:
1419 overscanImage=overscanResults.overscanImage,
1425 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1426 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1427 afwMath.MEDIAN | afwMath.STDEVCLIP)
1428 self.metadata[f
"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1429 qaStats.getValue(afwMath.MEDIAN)
1430 self.metadata[f
"ISR VARIANCE {amp.getName()} STDEV"] = \
1431 qaStats.getValue(afwMath.STDEVCLIP)
1432 self.log.debug(
" Variance stats for amplifer %s: %f +/- %f.",
1433 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1434 qaStats.getValue(afwMath.STDEVCLIP))
1435 if self.config.maskNegativeVariance:
1439 self.log.info(
"Applying linearizer.")
1440 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1441 detector=ccd, log=self.log)
1443 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1444 self.log.info(
"Applying crosstalk correction.")
1445 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1446 crosstalkSources=crosstalkSources, isTrimmed=
True)
1447 self.
debugView(ccdExposure,
"doCrosstalk")
1452 if self.config.doDefect:
1453 self.log.info(
"Masking defects.")
1456 if self.config.numEdgeSuspect > 0:
1457 self.log.info(
"Masking edges as SUSPECT.")
1458 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1459 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1461 if self.config.doNanMasking:
1462 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1465 if self.config.doWidenSaturationTrails:
1466 self.log.info(
"Widening saturation trails.")
1467 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1469 if self.config.doCameraSpecificMasking:
1470 self.log.info(
"Masking regions for camera specific reasons.")
1471 self.masking.
run(ccdExposure)
1473 if self.config.doBrighterFatter:
1483 interpExp = ccdExposure.clone()
1485 isrFunctions.interpolateFromMask(
1486 maskedImage=interpExp.getMaskedImage(),
1487 fwhm=self.config.fwhm,
1488 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1489 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1491 bfExp = interpExp.clone()
1493 self.log.info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1494 type(bfKernel), type(bfGains))
1495 if self.config.doFluxConservingBrighterFatterCorrection:
1496 bfResults = isrFunctions.fluxConservingBrighterFatterCorrection(
1499 self.config.brighterFatterMaxIter,
1500 self.config.brighterFatterThreshold,
1501 self.config.brighterFatterApplyGain,
1505 bfResults = isrFunctions.brighterFatterCorrection(
1508 self.config.brighterFatterMaxIter,
1509 self.config.brighterFatterThreshold,
1510 self.config.brighterFatterApplyGain,
1513 if bfResults[1] == self.config.brighterFatterMaxIter:
1514 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1517 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1519 image = ccdExposure.getMaskedImage().getImage()
1520 bfCorr = bfExp.getMaskedImage().getImage()
1521 bfCorr -= interpExp.getMaskedImage().getImage()
1530 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1531 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1534 if self.config.brighterFatterMaskGrowSize > 0:
1535 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1536 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1537 isrFunctions.growMasks(ccdExposure.getMask(),
1538 radius=self.config.brighterFatterMaskGrowSize,
1539 maskNameList=maskPlane,
1540 maskValue=maskPlane)
1542 self.
debugView(ccdExposure,
"doBrighterFatter")
1544 if self.config.doDark:
1545 self.log.info(
"Applying dark correction.")
1549 if self.config.doFringe
and not self.config.fringeAfterFlat:
1550 self.log.info(
"Applying fringe correction before flat.")
1551 self.fringe.
run(ccdExposure, **fringes.getDict())
1554 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1555 self.log.info(
"Checking strayLight correction.")
1556 self.strayLight.
run(ccdExposure, strayLightData)
1557 self.
debugView(ccdExposure,
"doStrayLight")
1559 if self.config.doFlat:
1560 self.log.info(
"Applying flat correction.")
1564 if self.config.doApplyGains:
1565 self.log.info(
"Applying gain correction instead of flat.")
1566 if self.config.usePtcGains:
1567 self.log.info(
"Using gains from the Photon Transfer Curve.")
1568 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1571 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1573 if self.config.doFringe
and self.config.fringeAfterFlat:
1574 self.log.info(
"Applying fringe correction after flat.")
1575 self.fringe.
run(ccdExposure, **fringes.getDict())
1577 if self.config.doVignette:
1578 if self.config.doMaskVignettePolygon:
1579 self.log.info(
"Constructing, attaching, and masking vignette polygon.")
1581 self.log.info(
"Constructing and attaching vignette polygon.")
1583 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1584 vignetteValue=self.config.vignetteValue, log=self.log)
1586 if self.config.doAttachTransmissionCurve:
1587 self.log.info(
"Adding transmission curves.")
1588 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1589 filterTransmission=filterTransmission,
1590 sensorTransmission=sensorTransmission,
1591 atmosphereTransmission=atmosphereTransmission)
1593 flattenedThumb =
None
1594 if self.config.qa.doThumbnailFlattened:
1595 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1597 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1598 self.log.info(
"Performing illumination correction.")
1599 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1600 illumMaskedImage, illumScale=self.config.illumScale,
1601 trimToFit=self.config.doTrimToMatchCalib)
1604 if self.config.doSaveInterpPixels:
1605 preInterpExp = ccdExposure.clone()
1620 if self.config.doSetBadRegions:
1621 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1622 if badPixelCount > 0:
1623 self.log.info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1625 if self.config.doInterpolate:
1626 self.log.info(
"Interpolating masked pixels.")
1627 isrFunctions.interpolateFromMask(
1628 maskedImage=ccdExposure.getMaskedImage(),
1629 fwhm=self.config.fwhm,
1630 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1631 maskNameList=list(self.config.maskListToInterpolate)
1637 if self.config.doAmpOffset:
1638 self.log.info(
"Correcting amp offsets.")
1639 self.ampOffset.
run(ccdExposure)
1641 if self.config.doMeasureBackground:
1642 self.log.info(
"Measuring background level.")
1645 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1647 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1648 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1649 afwMath.MEDIAN | afwMath.STDEVCLIP)
1650 self.metadata[f
"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1651 self.metadata[f
"ISR BACKGROUND {amp.getName()} STDEV"] = \
1652 qaStats.getValue(afwMath.STDEVCLIP)
1653 self.log.debug(
" Background stats for amplifer %s: %f +/- %f",
1654 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1655 qaStats.getValue(afwMath.STDEVCLIP))
1658 if self.config.doStandardStatistics:
1659 metadata = ccdExposure.getMetadata()
1661 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1662 ampName = amp.getName()
1663 metadata[f
"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels(
1664 ampExposure.getMaskedImage(),
1665 [self.config.saturatedMaskName]
1667 metadata[f
"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels(
1668 ampExposure.getMaskedImage(),
1671 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1672 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP)
1674 metadata[f
"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN)
1675 metadata[f
"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN)
1676 metadata[f
"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP)
1678 k1 = f
"LSST ISR FINAL MEDIAN {ampName}"
1679 k2 = f
"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}"
1680 if self.config.doOverscan
and k1
in metadata
and k2
in metadata:
1681 metadata[f
"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2]
1683 metadata[f
"LSST ISR LEVEL {ampName}"] = numpy.nan
1686 outputStatistics =
None
1687 if self.config.doCalculateStatistics:
1688 outputStatistics = self.isrStats.
run(ccdExposure, overscanResults=overscans,
1691 self.
debugView(ccdExposure,
"postISRCCD")
1693 return pipeBase.Struct(
1694 exposure=ccdExposure,
1696 flattenedThumb=flattenedThumb,
1698 preInterpExposure=preInterpExp,
1699 outputExposure=ccdExposure,
1700 outputOssThumbnail=ossThumb,
1701 outputFlattenedThumbnail=flattenedThumb,
1702 outputStatistics=outputStatistics,
1706 """Ensure that the data returned by Butler is a fully constructed exp.
1708 ISR requires exposure-level image data for historical reasons, so
if we
1709 did
not recieve that
from Butler, construct it
from what we have,
1710 modifying the input
in place.
1715 The input data structure obtained
from Butler.
1717 `lsst.afw.image.DecoratedImageU`,
1718 or `lsst.afw.image.ImageF`
1719 camera : `lsst.afw.cameraGeom.camera`, optional
1720 The camera associated
with the image. Used to find the appropriate
1721 detector
if detector
is not already set.
1722 detectorNum : `int`, optional
1723 The detector
in the camera to attach,
if the detector
is not
1729 The re-constructed exposure,
with appropriate detector parameters.
1734 Raised
if the input data cannot be used to construct an exposure.
1736 if isinstance(inputExp, afwImage.DecoratedImageU):
1737 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1738 elif isinstance(inputExp, afwImage.ImageF):
1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1740 elif isinstance(inputExp, afwImage.MaskedImageF):
1741 inputExp = afwImage.makeExposure(inputExp)
1742 elif isinstance(inputExp, afwImage.Exposure):
1744 elif inputExp
is None:
1748 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1751 if inputExp.getDetector()
is None:
1752 if camera
is None or detectorNum
is None:
1753 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1754 'without a detector set.')
1755 inputExp.setDetector(camera[detectorNum])
1761 """Extract common calibration metadata values that will be written to
1767 Calibration to pull date information
from.
1772 Calibration creation date string to add to header.
1774 if hasattr(calib,
"getMetadata"):
1775 if 'CALIB_CREATION_DATE' in calib.getMetadata():
1776 return " ".join((calib.getMetadata().get(
"CALIB_CREATION_DATE",
"Unknown"),
1777 calib.getMetadata().get(
"CALIB_CREATION_TIME",
"Unknown")))
1779 return " ".join((calib.getMetadata().get(
"CALIB_CREATE_DATE",
"Unknown"),
1780 calib.getMetadata().get(
"CALIB_CREATE_TIME",
"Unknown")))
1782 return "Unknown Unknown"
1785 """Compare header keywords to confirm camera states match.
1790 Header for the exposure being processed.
1792 Calibration to be applied.
1794 Calib type
for log message.
1797 calibMetadata = calib.getMetadata()
1798 except AttributeError:
1800 for keyword
in self.config.cameraKeywordsToCompare:
1801 if keyword
in exposureMetadata
and keyword
in calibMetadata:
1802 if exposureMetadata[keyword] != calibMetadata[keyword]:
1803 if self.config.doRaiseOnCalibMismatch:
1804 raise RuntimeError(
"Sequencer mismatch for %s [%s]: exposure: %s calib: %s",
1806 exposureMetadata[keyword], calibMetadata[keyword])
1808 self.log.warning(
"Sequencer mismatch for %s [%s]: exposure: %s calib: %s",
1810 exposureMetadata[keyword], calibMetadata[keyword])
1812 self.log.debug(
"Sequencer keyword %s not found.", keyword)
1815 """Convert exposure image from uint16 to float.
1817 If the exposure does not need to be converted, the input
is
1818 immediately returned. For exposures that are converted to use
1819 floating point pixels, the variance
is set to unity
and the
1825 The raw exposure to be converted.
1830 The input ``exposure``, converted to floating point pixels.
1835 Raised
if the exposure type cannot be converted to float.
1838 if isinstance(exposure, afwImage.ExposureF):
1840 self.log.debug(
"Exposure already of type float.")
1842 if not hasattr(exposure,
"convertF"):
1843 raise RuntimeError(
"Unable to convert exposure (%s) to float." % type(exposure))
1845 newexposure = exposure.convertF()
1846 newexposure.variance[:] = 1
1847 newexposure.mask[:] = 0x0
1852 """Identify bad amplifiers, saturated and suspect pixels.
1857 Input exposure to be masked.
1859 Catalog of parameters defining the amplifier on this
1862 List of defects. Used to determine if the entire
1868 If this
is true, the entire amplifier area
is covered by
1869 defects
and unusable.
1872 maskedImage = ccdExposure.getMaskedImage()
1879 if defects
is not None:
1880 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1886 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1888 maskView = dataView.getMask()
1889 maskView |= maskView.getPlaneBitMask(
"BAD")
1897 if self.config.doSaturation
and not badAmp:
1898 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1899 if self.config.doSuspect
and not badAmp:
1900 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1901 if math.isfinite(self.config.saturation):
1902 limits.update({self.config.saturatedMaskName: self.config.saturation})
1904 for maskName, maskThreshold
in limits.items():
1905 if not math.isnan(maskThreshold):
1906 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1907 isrFunctions.makeThresholdMask(
1908 maskedImage=dataView,
1909 threshold=maskThreshold,
1916 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1918 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1919 self.config.suspectMaskName])
1920 if numpy.all(maskView.getArray() & maskVal > 0):
1922 maskView |= maskView.getPlaneBitMask(
"BAD")
1927 """Apply overscan correction in place.
1929 This method does initial pixel rejection of the overscan
1930 region. The overscan can also be optionally segmented to
1931 allow for discontinuous overscan responses to be fit
1932 separately. The actual overscan subtraction
is performed by
1933 the `lsst.ip.isr.overscan.OverscanTask`, which
is called here
1934 after the amplifier
is preprocessed.
1939 Exposure to have overscan correction performed.
1940 amp : `lsst.afw.cameraGeom.Amplifer`
1941 The amplifier to consider
while correcting the overscan.
1945 overscanResults : `lsst.pipe.base.Struct`
1946 Result struct
with components:
1949 Value
or fit subtracted
from the amplifier image data.
1952 Value
or fit subtracted
from the overscan image data.
1955 Image of the overscan region
with the overscan
1956 correction applied. This quantity
is used to estimate
1957 the amplifier read noise empirically.
1962 Median overscan fit value. (`float`)
1964 Clipped standard deviation of the overscan after
1965 correction. (`float`)
1970 Raised
if the ``amp`` does
not contain raw pixel information.
1974 lsst.ip.isr.overscan.OverscanTask
1976 if amp.getRawHorizontalOverscanBBox().isEmpty():
1977 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1981 overscanResults = self.overscan.
run(ccdExposure, amp)
1983 metadata = ccdExposure.getMetadata()
1984 ampName = amp.getName()
1986 keyBase =
"LSST ISR OVERSCAN"
1988 if isinstance(overscanResults.overscanMean, float):
1990 metadata[f
"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean
1991 metadata[f
"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian
1992 metadata[f
"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma
1994 metadata[f
"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean
1995 metadata[f
"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian
1996 metadata[f
"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma
1997 elif isinstance(overscanResults.overscanMean, tuple):
1999 metadata[f
"{keyBase} SERIAL MEAN {ampName}"] = overscanResults.overscanMean[0]
2000 metadata[f
"{keyBase} SERIAL MEDIAN {ampName}"] = overscanResults.overscanMedian[0]
2001 metadata[f
"{keyBase} SERIAL STDEV {ampName}"] = overscanResults.overscanSigma[0]
2003 metadata[f
"{keyBase} PARALLEL MEAN {ampName}"] = overscanResults.overscanMean[1]
2004 metadata[f
"{keyBase} PARALLEL MEDIAN {ampName}"] = overscanResults.overscanMedian[1]
2005 metadata[f
"{keyBase} PARALLEL STDEV {ampName}"] = overscanResults.overscanSigma[1]
2007 metadata[f
"{keyBase} RESIDUAL SERIAL MEAN {ampName}"] = overscanResults.residualMean[0]
2008 metadata[f
"{keyBase} RESIDUAL SERIAL MEDIAN {ampName}"] = overscanResults.residualMedian[0]
2009 metadata[f
"{keyBase} RESIDUAL SERIAL STDEV {ampName}"] = overscanResults.residualSigma[0]
2011 metadata[f
"{keyBase} RESIDUAL PARALLEL MEAN {ampName}"] = overscanResults.residualMean[1]
2012 metadata[f
"{keyBase} RESIDUAL PARALLEL MEDIAN {ampName}"] = overscanResults.residualMedian[1]
2013 metadata[f
"{keyBase} RESIDUAL PARALLEL STDEV {ampName}"] = overscanResults.residualSigma[1]
2015 self.log.warning(
"Unexpected type for overscan values; none added to header.")
2017 return overscanResults
2020 """Set the variance plane using the gain and read noise
2022 The read noise is calculated
from the ``overscanImage``
if the
2023 ``doEmpiricalReadNoise`` option
is set
in the configuration; otherwise
2024 the value
from the amplifier data
is used.
2029 Exposure to process.
2031 Amplifier detector data.
2033 Image of overscan, required only
for empirical read noise.
2035 PTC dataset containing the gains
and read noise.
2040 Raised
if either ``usePtcGains`` of ``usePtcReadNoise``
2041 are ``
True``, but ptcDataset
is not provided.
2043 Raised
if ```doEmpiricalReadNoise``
is ``
True`` but
2044 ``overscanImage``
is ``
None``.
2048 lsst.ip.isr.isrFunctions.updateVariance
2050 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2051 if self.config.usePtcGains:
2052 if ptcDataset
is None:
2053 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2055 gain = ptcDataset.gain[amp.getName()]
2056 self.log.info(
"Using gain from Photon Transfer Curve.")
2058 gain = amp.getGain()
2060 if math.isnan(gain):
2062 self.log.warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2065 self.log.warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2066 amp.getName(), gain, patchedGain)
2069 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2070 badPixels = isrFunctions.countMaskedPixels(ampExposure.getMaskedImage(),
2071 [self.config.saturatedMaskName,
2072 self.config.suspectMaskName,
2074 allPixels = ampExposure.getWidth() * ampExposure.getHeight()
2075 if allPixels == badPixels:
2077 self.log.info(
"Skipping empirical read noise for amp %s. No good pixels.",
2080 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2082 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2083 stats = afwMath.StatisticsControl()
2084 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2085 readNoise = afwMath.makeStatistics(overscanImage.getImage(),
2086 afwMath.STDEVCLIP, stats).getValue()
2087 self.log.info(
"Calculated empirical read noise for amp %s: %f.",
2088 amp.getName(), readNoise)
2089 elif self.config.usePtcReadNoise:
2090 if ptcDataset
is None:
2091 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2093 readNoise = ptcDataset.noise[amp.getName()]
2094 self.log.info(
"Using read noise from Photon Transfer Curve.")
2096 readNoise = amp.getReadNoise()
2098 metadata = ampExposure.getMetadata()
2099 metadata[f
'LSST GAIN {amp.getName()}'] = gain
2100 metadata[f
'LSST READNOISE {amp.getName()}'] = readNoise
2102 isrFunctions.updateVariance(
2103 maskedImage=ampExposure.getMaskedImage(),
2105 readNoise=readNoise,
2109 """Identify and mask pixels with negative variance values.
2114 Exposure to process.
2118 lsst.ip.isr.isrFunctions.updateVariance
2120 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2121 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2122 exposure.mask.array[bad] |= maskPlane
2125 """Apply dark correction in place.
2130 Exposure to process.
2132 Dark exposure of the same size as ``exposure``.
2133 invert : `Bool`, optional
2134 If
True, re-add the dark to an already corrected image.
2139 Raised
if either ``exposure``
or ``darkExposure`` do
not
2140 have their dark time defined.
2144 lsst.ip.isr.isrFunctions.darkCorrection
2146 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2147 if math.isnan(expScale):
2148 raise RuntimeError(
"Exposure darktime is NAN.")
2149 if darkExposure.getInfo().getVisitInfo()
is not None \
2150 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2151 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2155 self.log.warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2158 isrFunctions.darkCorrection(
2159 maskedImage=exposure.getMaskedImage(),
2160 darkMaskedImage=darkExposure.getMaskedImage(),
2162 darkScale=darkScale,
2164 trimToFit=self.config.doTrimToMatchCalib
2168 """Check if linearization is needed for the detector cameraGeom.
2170 Checks config.doLinearize and the linearity type of the first
2176 Detector to get linearity type
from.
2180 doLinearize : `Bool`
2181 If
True, linearization should be performed.
2183 return self.config.doLinearize
and \
2184 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2187 """Apply flat correction in place.
2192 Exposure to process.
2194 Flat exposure of the same size as ``exposure``.
2195 invert : `Bool`, optional
2196 If
True, unflatten an already flattened image.
2200 lsst.ip.isr.isrFunctions.flatCorrection
2202 isrFunctions.flatCorrection(
2203 maskedImage=exposure.getMaskedImage(),
2204 flatMaskedImage=flatExposure.getMaskedImage(),
2205 scalingType=self.config.flatScalingType,
2206 userScale=self.config.flatUserScale,
2208 trimToFit=self.config.doTrimToMatchCalib
2212 """Detect and mask saturated pixels in config.saturatedMaskName.
2217 Exposure to process. Only the amplifier DataSec is processed.
2219 Amplifier detector data.
2223 lsst.ip.isr.isrFunctions.makeThresholdMask
2225 if not math.isnan(amp.getSaturation()):
2226 maskedImage = exposure.getMaskedImage()
2227 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2228 isrFunctions.makeThresholdMask(
2229 maskedImage=dataView,
2230 threshold=amp.getSaturation(),
2232 maskName=self.config.saturatedMaskName,
2236 """Interpolate over saturated pixels, in place.
2238 This method should be called after `saturationDetection`, to
2239 ensure that the saturated pixels have been identified in the
2240 SAT mask. It should also be called after `assembleCcd`, since
2241 saturated regions may cross amplifier boundaries.
2246 Exposure to process.
2250 lsst.ip.isr.isrTask.saturationDetection
2251 lsst.ip.isr.isrFunctions.interpolateFromMask
2253 isrFunctions.interpolateFromMask(
2254 maskedImage=exposure.getMaskedImage(),
2255 fwhm=self.config.fwhm,
2256 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2257 maskNameList=list(self.config.saturatedMaskName),
2261 """Detect and mask suspect pixels in config.suspectMaskName.
2266 Exposure to process. Only the amplifier DataSec is processed.
2268 Amplifier detector data.
2272 lsst.ip.isr.isrFunctions.makeThresholdMask
2276 Suspect pixels are pixels whose value
is greater than
2277 amp.getSuspectLevel(). This
is intended to indicate pixels that may be
2278 affected by unknown systematics;
for example
if non-linearity
2279 corrections above a certain level are unstable then that would be a
2280 useful value
for suspectLevel. A value of `nan` indicates that no such
2281 level exists
and no pixels are to be masked
as suspicious.
2283 suspectLevel = amp.getSuspectLevel()
2284 if math.isnan(suspectLevel):
2287 maskedImage = exposure.getMaskedImage()
2288 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2289 isrFunctions.makeThresholdMask(
2290 maskedImage=dataView,
2291 threshold=suspectLevel,
2293 maskName=self.config.suspectMaskName,
2297 """Mask defects using mask plane "BAD", in place.
2302 Exposure to process.
2303 defectBaseList : defect-type
2309 Call this after CCD assembly, since defects may cross amplifier
2312 maskedImage = exposure.getMaskedImage()
2313 if not isinstance(defectBaseList, Defects):
2315 defectList =
Defects(defectBaseList)
2317 defectList = defectBaseList
2318 defectList.maskPixels(maskedImage, maskName=
"BAD")
2320 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2321 """Mask edge pixels with applicable mask plane.
2326 Exposure to process.
2327 numEdgePixels : `int`, optional
2328 Number of edge pixels to mask.
2329 maskPlane : `str`, optional
2330 Mask plane name to use.
2331 level : `str`, optional
2332 Level at which to mask edges.
2334 maskedImage = exposure.getMaskedImage()
2335 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2337 if numEdgePixels > 0:
2338 if level ==
'DETECTOR':
2339 boxes = [maskedImage.getBBox()]
2340 elif level ==
'AMP':
2341 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2346 subImage = maskedImage[box]
2347 box.grow(-numEdgePixels)
2349 SourceDetectionTask.setEdgeBits(
2355 """Mask and interpolate defects using mask plane "BAD", in place.
2360 Exposure to process.
2361 defectBaseList : defects-like
2362 List of defects to mask and interpolate. Can be
2367 lsst.ip.isr.isrTask.maskDefect
2370 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2371 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2372 isrFunctions.interpolateFromMask(
2373 maskedImage=exposure.getMaskedImage(),
2374 fwhm=self.config.fwhm,
2375 growSaturatedFootprints=0,
2376 maskNameList=[
"BAD"],
2380 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2385 Exposure to process.
2389 We mask over all non-finite values (NaN, inf), including those
2390 that are masked with other bits (because those may
or may
not be
2391 interpolated over later,
and we want to remove all NaN/infs).
2392 Despite this behaviour, the
"UNMASKEDNAN" mask plane
is used to
2393 preserve the historical name.
2395 maskedImage = exposure.getMaskedImage()
2398 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2399 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2400 numNans = maskNans(maskedImage, maskVal)
2401 self.metadata[
"NUMNANS"] = numNans
2403 self.log.warning(
"There were %d unmasked NaNs.", numNans)
2406 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2412 Exposure to process.
2416 lsst.ip.isr.isrTask.maskNan
2419 isrFunctions.interpolateFromMask(
2420 maskedImage=exposure.getMaskedImage(),
2421 fwhm=self.config.fwhm,
2422 growSaturatedFootprints=0,
2423 maskNameList=["UNMASKEDNAN"],
2427 """Measure the image background in subgrids, for quality control.
2432 Exposure to process.
2434 Configuration object containing parameters on which background
2435 statistics and subgrids to use.
2437 if IsrQaConfig
is not None:
2438 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2439 IsrQaConfig.flatness.nIter)
2440 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2441 statsControl.setAndMask(maskVal)
2442 maskedImage = exposure.getMaskedImage()
2443 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2444 skyLevel = stats.getValue(afwMath.MEDIAN)
2445 skySigma = stats.getValue(afwMath.STDEVCLIP)
2446 self.log.info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2447 metadata = exposure.getMetadata()
2448 metadata[
"SKYLEVEL"] = skyLevel
2449 metadata[
"SKYSIGMA"] = skySigma
2452 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2453 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2454 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2455 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2456 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2457 skyLevels = numpy.zeros((nX, nY))
2460 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2462 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2464 xLLC = xc - meshXHalf
2465 yLLC = yc - meshYHalf
2466 xURC = xc + meshXHalf - 1
2467 yURC = yc + meshYHalf - 1
2470 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2472 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2474 good = numpy.where(numpy.isfinite(skyLevels))
2475 skyMedian = numpy.median(skyLevels[good])
2476 flatness = (skyLevels[good] - skyMedian) / skyMedian
2477 flatness_rms = numpy.std(flatness)
2478 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2480 self.log.info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2481 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2482 nX, nY, flatness_pp, flatness_rms)
2484 metadata[
"FLATNESS_PP"] = float(flatness_pp)
2485 metadata[
"FLATNESS_RMS"] = float(flatness_rms)
2486 metadata[
"FLATNESS_NGRIDS"] =
'%dx%d' % (nX, nY)
2487 metadata[
"FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2488 metadata[
"FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2491 """Set an approximate magnitude zero point for the exposure.
2496 Exposure to process.
2498 filterLabel = exposure.getFilter()
2499 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2501 if physicalFilter
in self.config.fluxMag0T1:
2502 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2504 self.log.warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2505 fluxMag0 = self.config.defaultFluxMag0T1
2507 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2509 self.log.warning(
"Non-positive exposure time; skipping rough zero point.")
2512 self.log.info(
"Setting rough magnitude zero point for filter %s: %f",
2513 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2514 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2518 """Context manager that applies and removes flats and darks,
2519 if the task
is configured to apply them.
2524 Exposure to process.
2526 Flat exposure the same size
as ``exp``.
2528 Dark exposure the same size
as ``exp``.
2533 The flat
and dark corrected exposure.
2535 if self.config.doDark
and dark
is not None:
2537 if self.config.doFlat:
2542 if self.config.doFlat:
2544 if self.config.doDark
and dark
is not None:
2548 """Utility function to examine ISR exposure at different stages.
2555 State of processing to view.
2557 frame = getDebugFrame(self._display, stepname)
2559 display = getDisplay(frame)
2560 display.scale(
'asinh',
'zscale')
2561 display.mtv(exposure)
2562 prompt =
"Press Enter to continue [c]... "
2564 ans = input(prompt).lower()
2565 if ans
in (
"",
"c",):
2570 """A Detector-like object that supports returning gain and saturation level
2572 This is used when the input exposure does
not have a detector.
2577 Exposure to generate a fake amplifier
for.
2578 config : `lsst.ip.isr.isrTaskConfig`
2579 Configuration to apply to the fake amplifier.
2583 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2585 self.
_gain = config.gain
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def extractCalibDate(calib)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def maskNegativeVariance(self, exposure)
def compareCameraKeywords(self, exposureMetadata, calib, calibName)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, deferredChargeCalib=None)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)