33 from contextlib
import contextmanager
34 from lsstDebug
import getDebugFrame
42 from .
import isrFunctions
45 from .assembleCcdTask
import AssembleCcdTask
46 from .crosstalk
import CrosstalkTask
47 from .fringe
import FringeTask
48 from .isr
import maskNans
49 from .masking
import MaskingTask
50 from .straylight
import StrayLightTask
51 from .vignette
import VignetteTask
53 __all__ = [
"IsrTask",
"RunIsrTask"]
57 """Configuration parameters for IsrTask. 59 Items are grouped in the order in which they are executed by the task. 62 datasetType = pexConfig.Field(
64 doc=
"Dataset type for input data; users will typically leave this alone, " 65 "but camera-specific ISR tasks will override it",
68 fallbackFilterName = pexConfig.Field(
70 doc=
"Fallback default filter name for calibrations.",
72 expectWcs = pexConfig.Field(
75 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)." 77 fwhm = pexConfig.Field(
79 doc=
"FWHM of PSF in arcseconds.",
82 qa = pexConfig.ConfigField(
84 doc=
"QA related configuration options.",
88 doConvertIntToFloat = pexConfig.Field(
90 doc=
"Convert integer raw images to floating point values?",
95 doSaturation = pexConfig.Field(
97 doc=
"Mask saturated pixels?",
100 saturatedMaskName = pexConfig.Field(
102 doc=
"Name of mask plane to use in saturation detection and interpolation",
105 saturation = pexConfig.Field(
107 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
108 default=float(
"NaN"),
110 growSaturationFootprintSize = pexConfig.Field(
112 doc=
"Number of pixels by which to grow the saturation footprints",
117 doSuspect = pexConfig.Field(
119 doc=
"Mask suspect pixels?",
122 suspectMaskName = pexConfig.Field(
124 doc=
"Name of mask plane to use for suspect pixels",
127 numEdgeSuspect = pexConfig.Field(
129 doc=
"Number of edge pixels to be flagged as untrustworthy.",
134 doSetBadRegions = pexConfig.Field(
136 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
139 badStatistic = pexConfig.ChoiceField(
141 doc=
"How to estimate the average value for BAD regions.",
144 "MEANCLIP":
"Correct using the (clipped) mean of good data",
145 "MEDIAN":
"Correct using the median of the good data",
150 doOverscan = pexConfig.Field(
152 doc=
"Do overscan subtraction?",
155 overscanFitType = pexConfig.ChoiceField(
157 doc=
"The method for fitting the overscan bias level.",
160 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
161 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
162 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
163 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
164 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
165 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
166 "MEAN":
"Correct using the mean of the overscan region",
167 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
168 "MEDIAN":
"Correct using the median of the overscan region",
171 overscanOrder = pexConfig.Field(
173 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
174 "or number of spline knots if overscan fit type is a spline."),
177 overscanNumSigmaClip = pexConfig.Field(
179 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
182 overscanIsInt = pexConfig.Field(
184 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
187 overscanNumLeadingColumnsToSkip = pexConfig.Field(
189 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
192 overscanNumTrailingColumnsToSkip = pexConfig.Field(
194 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
197 overscanMaxDev = pexConfig.Field(
199 doc=
"Maximum deviation from the median for overscan",
200 default=1000.0, check=
lambda x: x > 0
202 overscanBiasJump = pexConfig.Field(
204 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
207 overscanBiasJumpKeyword = pexConfig.Field(
209 doc=
"Header keyword containing information about devices.",
210 default=
"NO_SUCH_KEY",
212 overscanBiasJumpDevices = pexConfig.ListField(
214 doc=
"List of devices that need piecewise overscan correction.",
217 overscanBiasJumpLocation = pexConfig.Field(
219 doc=
"Location of bias jump along y-axis.",
224 doAssembleCcd = pexConfig.Field(
227 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 229 assembleCcd = pexConfig.ConfigurableField(
230 target=AssembleCcdTask,
231 doc=
"CCD assembly task",
235 doAssembleIsrExposures = pexConfig.Field(
238 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 240 doTrimToMatchCalib = pexConfig.Field(
243 doc=
"Trim raw data to match calibration bounding boxes?" 247 doBias = pexConfig.Field(
249 doc=
"Apply bias frame correction?",
252 biasDataProductName = pexConfig.Field(
254 doc=
"Name of the bias data product",
259 doVariance = pexConfig.Field(
261 doc=
"Calculate variance?",
264 gain = pexConfig.Field(
266 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
267 default=float(
"NaN"),
269 readNoise = pexConfig.Field(
271 doc=
"The read noise to use if no Detector is present in the Exposure",
274 doEmpiricalReadNoise = pexConfig.Field(
277 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 281 doLinearize = pexConfig.Field(
283 doc=
"Correct for nonlinearity of the detector's response?",
288 doCrosstalk = pexConfig.Field(
290 doc=
"Apply intra-CCD crosstalk correction?",
293 doCrosstalkBeforeAssemble = pexConfig.Field(
295 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
298 crosstalk = pexConfig.ConfigurableField(
299 target=CrosstalkTask,
300 doc=
"Intra-CCD crosstalk correction",
304 doWidenSaturationTrails = pexConfig.Field(
306 doc=
"Widen bleed trails based on their width?",
311 doBrighterFatter = pexConfig.Field(
314 doc=
"Apply the brighter fatter correction" 316 brighterFatterLevel = pexConfig.ChoiceField(
319 doc=
"The level at which to correct for brighter-fatter.",
321 "AMP":
"Every amplifier treated separately.",
322 "DETECTOR":
"One kernel per detector",
325 brighterFatterKernelFile = pexConfig.Field(
328 doc=
"Kernel file used for the brighter fatter correction" 330 brighterFatterMaxIter = pexConfig.Field(
333 doc=
"Maximum number of iterations for the brighter fatter correction" 335 brighterFatterThreshold = pexConfig.Field(
338 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 339 " absolute value of the difference between the current corrected image and the one" 340 " from the previous iteration summed over all the pixels." 342 brighterFatterApplyGain = pexConfig.Field(
345 doc=
"Should the gain be applied when applying the brighter fatter correction?" 349 doDefect = pexConfig.Field(
351 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
354 doSaturationInterpolation = pexConfig.Field(
356 doc=
"Perform interpolation over pixels masked as saturated?",
359 numEdgeSuspect = pexConfig.Field(
361 doc=
"Number of edge pixels to be flagged as untrustworthy.",
366 doDark = pexConfig.Field(
368 doc=
"Apply dark frame correction?",
371 darkDataProductName = pexConfig.Field(
373 doc=
"Name of the dark data product",
378 doStrayLight = pexConfig.Field(
380 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
383 strayLight = pexConfig.ConfigurableField(
384 target=StrayLightTask,
385 doc=
"y-band stray light correction" 389 doFlat = pexConfig.Field(
391 doc=
"Apply flat field correction?",
394 flatDataProductName = pexConfig.Field(
396 doc=
"Name of the flat data product",
399 flatScalingType = pexConfig.ChoiceField(
401 doc=
"The method for scaling the flat on the fly.",
404 "USER":
"Scale by flatUserScale",
405 "MEAN":
"Scale by the inverse of the mean",
406 "MEDIAN":
"Scale by the inverse of the median",
409 flatUserScale = pexConfig.Field(
411 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
414 doTweakFlat = pexConfig.Field(
416 doc=
"Tweak flats to match observed amplifier ratios?",
421 doApplyGains = pexConfig.Field(
423 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
426 normalizeGains = pexConfig.Field(
428 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
433 doFringe = pexConfig.Field(
435 doc=
"Apply fringe correction?",
438 fringe = pexConfig.ConfigurableField(
440 doc=
"Fringe subtraction task",
442 fringeAfterFlat = pexConfig.Field(
444 doc=
"Do fringe subtraction after flat-fielding?",
449 doNanInterpAfterFlat = pexConfig.Field(
451 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 452 "also have to interpolate them before flat-fielding."),
457 doAddDistortionModel = pexConfig.Field(
459 doc=
"Apply a distortion model based on camera geometry to the WCS?",
464 doMeasureBackground = pexConfig.Field(
466 doc=
"Measure the background level on the reduced image?",
471 doCameraSpecificMasking = pexConfig.Field(
473 doc=
"Mask camera-specific bad regions?",
476 masking = pexConfig.ConfigurableField(
482 fluxMag0T1 = pexConfig.DictField(
485 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
486 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
489 defaultFluxMag0T1 = pexConfig.Field(
491 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
492 default=pow(10.0, 0.4*28.0)
496 doVignette = pexConfig.Field(
498 doc=
"Apply vignetting parameters?",
501 vignette = pexConfig.ConfigurableField(
503 doc=
"Vignetting task.",
507 doAttachTransmissionCurve = pexConfig.Field(
510 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 512 doUseOpticsTransmission = pexConfig.Field(
515 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 517 doUseFilterTransmission = pexConfig.Field(
520 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 522 doUseSensorTransmission = pexConfig.Field(
525 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 527 doUseAtmosphereTransmission = pexConfig.Field(
530 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 534 doWrite = pexConfig.Field(
536 doc=
"Persist postISRCCD?",
543 raise ValueError(
"You may not specify both doFlat and doApplyGains")
547 r"""Apply common instrument signature correction algorithms to a raw frame. 549 The process for correcting imaging data is very similar from 550 camera to camera. This task provides a vanilla implementation of 551 doing these corrections, including the ability to turn certain 552 corrections off if they are not needed. The inputs to the primary 553 method, `run()`, are a raw exposure to be corrected and the 554 calibration data products. The raw input is a single chip sized 555 mosaic of all amps including overscans and other non-science 556 pixels. The method `runDataRef()` identifies and defines the 557 calibration data products, and is intended for use by a 558 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a 559 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be 560 subclassed for different camera, although the most camera specific 561 methods have been split into subtasks that can be redirected 564 The __init__ method sets up the subtasks for ISR processing, using 565 the defaults from `lsst.ip.isr`. 570 Positional arguments passed to the Task constructor. None used at this time. 571 kwargs : `dict`, optional 572 Keyword arguments passed on to the Task constructor. None used at this time. 574 ConfigClass = IsrTaskConfig
578 pipeBase.Task.__init__(self, *args, **kwargs)
580 self.makeSubtask(
"assembleCcd")
581 self.makeSubtask(
"crosstalk")
582 self.makeSubtask(
"strayLight")
583 self.makeSubtask(
"fringe")
584 self.makeSubtask(
"masking")
585 self.makeSubtask(
"vignette")
588 """!Retrieve necessary frames for instrument signature removal. 590 Pre-fetching all required ISR data products limits the IO 591 required by the ISR. Any conflict between the calibration data 592 available and that needed for ISR is also detected prior to 593 doing processing, allowing it to fail quickly. 597 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 598 Butler reference of the detector data to be processed 599 rawExposure : `afw.image.Exposure` 600 The raw exposure that will later be corrected with the 601 retrieved calibration data; should not be modified in this 606 result : `lsst.pipe.base.Struct` 607 Result struct with components (which may be `None`): 608 - ``bias``: bias calibration frame (`afw.image.Exposure`) 609 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`) 610 - ``crosstalkSources``: list of possible crosstalk sources (`list`) 611 - ``dark``: dark calibration frame (`afw.image.Exposure`) 612 - ``flat``: flat calibration frame (`afw.image.Exposure`) 613 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`) 614 - ``defects``: list of defects (`list`) 615 - ``fringes``: `lsst.pipe.base.Struct` with components: 616 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 617 - ``seed``: random seed derived from the ccdExposureId for random 618 number generator (`uint32`) 619 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve` 620 A ``TransmissionCurve`` that represents the throughput of the optics, 621 to be evaluated in focal-plane coordinates. 622 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve` 623 A ``TransmissionCurve`` that represents the throughput of the filter 624 itself, to be evaluated in focal-plane coordinates. 625 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve` 626 A ``TransmissionCurve`` that represents the throughput of the sensor 627 itself, to be evaluated in post-assembly trimmed detector coordinates. 628 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve` 629 A ``TransmissionCurve`` that represents the throughput of the 630 atmosphere, assumed to be spatially constant. 633 ccd = rawExposure.getDetector()
634 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
635 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
636 if self.config.doBias
else None)
638 linearizer = (dataRef.get(
"linearizer", immediate=
True)
640 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
641 if self.config.doCrosstalk
else None)
642 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
643 if self.config.doDark
else None)
644 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName)
645 if self.config.doFlat
else None)
646 brighterFatterKernel = (dataRef.get(
"bfKernel")
647 if self.config.doBrighterFatter
else None)
648 defectList = (dataRef.get(
"defects")
649 if self.config.doDefect
else None)
650 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
651 if self.config.doAssembleIsrExposures
else None)
652 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
653 else pipeBase.Struct(fringes=
None))
655 if self.config.doAttachTransmissionCurve:
656 opticsTransmission = (dataRef.get(
"transmission_optics")
657 if self.config.doUseOpticsTransmission
else None)
658 filterTransmission = (dataRef.get(
"transmission_filter")
659 if self.config.doUseFilterTransmission
else None)
660 sensorTransmission = (dataRef.get(
"transmission_sensor")
661 if self.config.doUseSensorTransmission
else None)
662 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
663 if self.config.doUseAtmosphereTransmission
else None)
665 opticsTransmission =
None 666 filterTransmission =
None 667 sensorTransmission =
None 668 atmosphereTransmission =
None 671 return pipeBase.Struct(bias=biasExposure,
672 linearizer=linearizer,
673 crosstalkSources=crosstalkSources,
676 bfKernel=brighterFatterKernel,
678 fringes=fringeStruct,
679 opticsTransmission=opticsTransmission,
680 filterTransmission=filterTransmission,
681 sensorTransmission=sensorTransmission,
682 atmosphereTransmission=atmosphereTransmission,
686 def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
687 dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
688 opticsTransmission=None, filterTransmission=None,
689 sensorTransmission=None, atmosphereTransmission=None,
691 """!Perform instrument signature removal on an exposure. 693 Steps included in the ISR processing, in order performed, are: 694 - saturation and suspect pixel masking 695 - overscan subtraction 696 - CCD assembly of individual amplifiers 698 - variance image construction 699 - linearization of non-linear response 701 - brighter-fatter correction 704 - stray light subtraction 706 - masking of known defects and camera specific features 707 - vignette calculation 708 - appending transmission curve and distortion model 712 ccdExposure : `lsst.afw.image.Exposure` 713 The raw exposure that is to be run through ISR. The 714 exposure is modified by this method. 715 camera : `lsst.afw.cameraGeom.Camera`, optional 716 The camera geometry for this exposure. Used to select the 717 distortion model appropriate for this data. 718 bias : `lsst.afw.image.Exposure`, optional 719 Bias calibration frame. 720 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 721 Functor for linearization. 722 crosstalkSources : `list`, optional 723 List of possible crosstalk sources. 724 dark : `lsst.afw.image.Exposure`, optional 725 Dark calibration frame. 726 flat : `lsst.afw.image.Exposure`, optional 727 Flat calibration frame. 728 bfKernel : `numpy.ndarray`, optional 729 Brighter-fatter kernel. 730 defects : `list`, optional 732 fringes : `lsst.pipe.base.Struct`, optional 733 Struct containing the fringe correction data, with 735 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 736 - ``seed``: random seed derived from the ccdExposureId for random 737 number generator (`uint32`) 738 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 739 A ``TransmissionCurve`` that represents the throughput of the optics, 740 to be evaluated in focal-plane coordinates. 741 filterTransmission : `lsst.afw.image.TransmissionCurve` 742 A ``TransmissionCurve`` that represents the throughput of the filter 743 itself, to be evaluated in focal-plane coordinates. 744 sensorTransmission : `lsst.afw.image.TransmissionCurve` 745 A ``TransmissionCurve`` that represents the throughput of the sensor 746 itself, to be evaluated in post-assembly trimmed detector coordinates. 747 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 748 A ``TransmissionCurve`` that represents the throughput of the 749 atmosphere, assumed to be spatially constant. 753 result : `lsst.pipe.base.Struct` 754 Result struct with component: 755 - ``exposure`` : `afw.image.Exposure` 756 The fully ISR corrected exposure. 757 - ``ossThumb`` : `numpy.ndarray` 758 Thumbnail image of the exposure after overscan subtraction. 759 - ``flattenedThumb`` : `numpy.ndarray` 760 Thumbnail image of the exposure after flat-field correction. 765 Raised if a configuration option is set to True, but the 766 required calibration data has not been specified. 770 The current processed exposure can be viewed by setting the 771 appropriate lsstDebug entries in the `debug.display` 772 dictionary. The names of these entries correspond to some of 773 the IsrTaskConfig Boolean options, with the value denoting the 774 frame to use. The exposure is shown inside the matching 775 option check and after the processing of that step has 776 finished. The steps with debug points are: 787 In addition, setting the "postISRCCD" entry displays the 788 exposure after all ISR processing has finished. 791 if isinstance(ccdExposure, ButlerDataRef):
794 self.log.info(
"Performing ISR on exposure %s" %
795 (ccdExposure.getInfo().getVisitInfo().getExposureId()))
796 ccd = ccdExposure.getDetector()
799 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 800 ccd = [
FakeAmp(ccdExposure, self.config)]
803 if self.config.doBias
and bias
is None:
804 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
806 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
807 if self.config.doBrighterFatter
and bfKernel
is None:
808 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
809 if self.config.doDark
and dark
is None:
810 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
812 fringes = pipeBase.Struct(fringes=
None)
813 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
814 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
815 if self.config.doFlat
and flat
is None:
816 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
817 if self.config.doDefect
and defects
is None:
818 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
819 if self.config.doAddDistortionModel
and camera
is None:
820 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel=True.")
823 if self.config.doConvertIntToFloat:
824 self.log.info(
"Converting exposure to floating point values")
831 if ccdExposure.getBBox().contains(amp.getBBox()):
835 if self.config.doOverscan
and not badAmp:
838 self.log.info(
"Corrected overscan for amplifier %s" % (amp.getName()))
839 if self.config.qa
is not None and self.config.qa.saveStats
is True:
840 if isinstance(overscanResults.overscanFit, float):
841 qaMedian = overscanResults.overscanFit
842 qaStdev = float(
"NaN")
844 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
845 afwMath.MEDIAN | afwMath.STDEVCLIP)
846 qaMedian = qaStats.getValue(afwMath.MEDIAN)
847 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
849 self.metadata.set(
"ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
850 self.metadata.set(
"ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
851 self.log.info(
" Overscan stats for amplifer %s: %f +/- %f" %
852 (amp.getName(), qaMedian, qaStdev))
853 ccdExposure.getMetadata().set(
'OVERSCAN',
"Overscan corrected")
855 self.log.warn(
"Amplifier %s is bad." % (amp.getName()))
856 overscanResults =
None 858 overscans.append(overscanResults
if overscanResults
is not None else None)
860 self.log.info(
"Skipped OSCAN")
862 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
863 self.log.info(
"Applying crosstalk correction.")
864 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
865 self.
debugView(ccdExposure,
"doCrosstalk")
867 if self.config.doAssembleCcd:
868 self.log.info(
"Assembling CCD from amplifiers")
869 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
871 if self.config.expectWcs
and not ccdExposure.getWcs():
872 self.log.warn(
"No WCS found in input exposure")
873 self.
debugView(ccdExposure,
"doAssembleCcd")
876 if self.config.qa.doThumbnailOss:
877 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
879 if self.config.doBias:
880 self.log.info(
"Applying bias correction.")
881 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
882 trimToFit=self.config.doTrimToMatchCalib)
885 if self.config.doVariance:
886 for amp, overscanResults
in zip(ccd, overscans):
887 if ccdExposure.getBBox().contains(amp.getBBox()):
888 self.log.info(
"Constructing variance map for amplifer %s" % (amp.getName()))
889 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
890 if overscanResults
is not None:
892 overscanImage=overscanResults.overscanImage)
896 if self.config.qa
is not None and self.config.qa.saveStats
is True:
897 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
898 afwMath.MEDIAN | afwMath.STDEVCLIP)
899 self.metadata.set(
"ISR VARIANCE {} MEDIAN".format(amp.getName()),
900 qaStats.getValue(afwMath.MEDIAN))
901 self.metadata.set(
"ISR VARIANCE {} STDEV".format(amp.getName()),
902 qaStats.getValue(afwMath.STDEVCLIP))
903 self.log.info(
" Variance stats for amplifer %s: %f +/- %f" %
904 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
905 qaStats.getValue(afwMath.STDEVCLIP)))
908 self.log.info(
"Applying linearizer.")
909 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
911 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
912 self.log.info(
"Applying crosstalk correction.")
913 self.crosstalk.
run(ccdExposure, crosstalkSources=crosstalkSources)
914 self.
debugView(ccdExposure,
"doCrosstalk")
916 if self.config.doWidenSaturationTrails:
917 self.log.info(
"Widening saturation trails.")
918 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
920 interpolationDone =
False 921 if self.config.doBrighterFatter:
927 if self.config.doDefect:
930 if self.config.doSaturationInterpolation:
934 interpolationDone =
True 936 if self.config.brighterFatterLevel ==
'DETECTOR':
937 kernelElement = bfKernel
940 raise NotImplementedError(
"per-amplifier brighter-fatter correction not yet implemented")
941 self.log.info(
"Applying brighter fatter correction.")
942 isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
943 self.config.brighterFatterMaxIter,
944 self.config.brighterFatterThreshold,
945 self.config.brighterFatterApplyGain,
947 self.
debugView(ccdExposure,
"doBrighterFatter")
949 if self.config.doDark:
950 self.log.info(
"Applying dark correction.")
954 if self.config.doFringe
and not self.config.fringeAfterFlat:
955 self.log.info(
"Applying fringe correction before flat.")
956 self.fringe.
run(ccdExposure, **fringes.getDict())
959 if self.config.doStrayLight:
960 self.log.info(
"Applying stray light correction.")
961 self.strayLight.
run(ccdExposure)
962 self.
debugView(ccdExposure,
"doStrayLight")
964 if self.config.doFlat:
965 self.log.info(
"Applying flat correction.")
969 if self.config.doApplyGains:
970 self.log.info(
"Applying gain correction instead of flat.")
971 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
973 if self.config.doDefect
and not interpolationDone:
974 self.log.info(
"Masking and interpolating defects.")
977 if self.config.doSaturation
and not interpolationDone:
978 self.log.info(
"Interpolating saturated pixels.")
981 if self.config.doNanInterpAfterFlat
or not interpolationDone:
982 self.log.info(
"Masking and interpolating NAN value pixels.")
985 if self.config.doFringe
and self.config.fringeAfterFlat:
986 self.log.info(
"Applying fringe correction after flat.")
987 self.fringe.
run(ccdExposure, **fringes.getDict())
989 if self.config.doSetBadRegions:
990 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
991 self.log.info(
"Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
993 flattenedThumb =
None 994 if self.config.qa.doThumbnailFlattened:
995 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
997 if self.config.doCameraSpecificMasking:
998 self.log.info(
"Masking regions for camera specific reasons.")
999 self.masking.
run(ccdExposure)
1003 if self.config.doVignette:
1004 self.log.info(
"Constructing Vignette polygon.")
1007 if self.config.vignette.doWriteVignettePolygon:
1010 if self.config.doAttachTransmissionCurve:
1011 self.log.info(
"Adding transmission curves.")
1012 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1013 filterTransmission=filterTransmission,
1014 sensorTransmission=sensorTransmission,
1015 atmosphereTransmission=atmosphereTransmission)
1017 if self.config.doAddDistortionModel:
1018 self.log.info(
"Adding a distortion model to the WCS.")
1019 isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1021 if self.config.doMeasureBackground:
1022 self.log.info(
"Measuring background level:")
1025 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1027 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1028 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1029 afwMath.MEDIAN | afwMath.STDEVCLIP)
1030 self.metadata.set(
"ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1031 qaStats.getValue(afwMath.MEDIAN))
1032 self.metadata.set(
"ISR BACKGROUND {} STDEV".format(amp.getName()),
1033 qaStats.getValue(afwMath.STDEVCLIP))
1034 self.log.info(
" Background stats for amplifer %s: %f +/- %f" %
1035 (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1036 qaStats.getValue(afwMath.STDEVCLIP)))
1038 self.
debugView(ccdExposure,
"postISRCCD")
1040 return pipeBase.Struct(
1041 exposure=ccdExposure,
1043 flattenedThumb=flattenedThumb
1046 @pipeBase.timeMethod
1048 """Perform instrument signature removal on a ButlerDataRef of a Sensor. 1050 This method contains the `CmdLineTask` interface to the ISR 1051 processing. All IO is handled here, freeing the `run()` method 1052 to manage only pixel-level calculations. The steps performed 1054 - Read in necessary detrending/isr/calibration data. 1055 - Process raw exposure in `run()`. 1056 - Persist the ISR-corrected exposure as "postISRCCD" if 1057 config.doWrite=True. 1061 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 1062 DataRef of the detector data to be processed 1066 result : `lsst.pipe.base.Struct` 1067 Result struct with component: 1068 - ``exposure`` : `afw.image.Exposure` 1069 The fully ISR corrected exposure. 1074 Raised if a configuration option is set to True, but the 1075 required calibration data does not exist. 1078 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
1079 ccdExposure = sensorRef.get(self.config.datasetType)
1081 camera = sensorRef.get(
"camera")
1082 if camera
is None and self.config.doAddDistortionModel:
1083 raise RuntimeError(
"config.doAddDistortionModel is True " 1084 "but could not get a camera from the butler")
1085 isrData = self.
readIsrData(sensorRef, ccdExposure)
1087 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1089 if self.config.doWrite:
1090 sensorRef.put(result.exposure,
"postISRCCD")
1091 if result.ossThumb
is not None:
1092 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1093 if result.flattenedThumb
is not None:
1094 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1099 """!Retrieve a calibration dataset for removing instrument signature. 1104 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 1105 DataRef of the detector data to find calibration datasets 1108 Type of dataset to retrieve (e.g. 'bias', 'flat', etc). 1110 If True, disable butler proxies to enable error handling 1111 within this routine. 1115 exposure : `lsst.afw.image.Exposure` 1116 Requested calibration frame. 1121 Raised if no matching calibration frame can be found. 1124 exp = dataRef.get(datasetType, immediate=immediate)
1125 except Exception
as exc1:
1126 if not self.config.fallbackFilterName:
1127 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1129 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1130 except Exception
as exc2:
1131 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1132 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1133 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1135 if self.config.doAssembleIsrExposures:
1136 exp = self.assembleCcd.assembleCcd(exp)
1140 """Convert exposure image from uint16 to float. 1142 If the exposure does not need to be converted, the input is 1143 immediately returned. For exposures that are converted to use 1144 floating point pixels, the variance is set to unity and the 1149 exposure : `lsst.afw.image.Exposure` 1150 The raw exposure to be converted. 1154 newexposure : `lsst.afw.image.Exposure` 1155 The input ``exposure``, converted to floating point pixels. 1160 Raised if the exposure type cannot be converted to float. 1163 if isinstance(exposure, afwImage.ExposureF):
1166 if not hasattr(exposure,
"convertF"):
1167 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
1169 newexposure = exposure.convertF()
1170 newexposure.variance[:] = 1
1171 newexposure.mask[:] = 0x0
1176 """Identify bad amplifiers, saturated and suspect pixels. 1180 ccdExposure : `lsst.afw.image.Exposure` 1181 Input exposure to be masked. 1182 amp : `lsst.afw.table.AmpInfoCatalog` 1183 Catalog of parameters defining the amplifier on this 1186 List of defects. Used to determine if the entire 1192 If this is true, the entire amplifier area is covered by 1193 defects and unusable. 1196 maskedImage = ccdExposure.getMaskedImage()
1202 if defects
is not None:
1203 badAmp = bool(sum([v.getBBox().contains(amp.getBBox())
for v
in defects]))
1208 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1210 maskView = dataView.getMask()
1211 maskView |= maskView.getPlaneBitMask(
"BAD")
1218 if self.config.doSaturation
and not badAmp:
1219 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1220 if self.config.doSuspect
and not badAmp:
1221 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1223 for maskName, maskThreshold
in limits.items():
1224 if not math.isnan(maskThreshold):
1225 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1226 isrFunctions.makeThresholdMask(
1227 maskedImage=dataView,
1228 threshold=maskThreshold,
1234 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1236 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1237 self.config.suspectMaskName])
1238 if numpy.all(maskView.getArray() & maskVal > 0):
1244 """Apply overscan correction in place. 1246 This method does initial pixel rejection of the overscan 1247 region. The overscan can also be optionally segmented to 1248 allow for discontinuous overscan responses to be fit 1249 separately. The actual overscan subtraction is performed by 1250 the `lsst.ip.isr.isrFunctions.overscanCorrection` function, 1251 which is called here after the amplifier is preprocessed. 1255 ccdExposure : `lsst.afw.image.Exposure` 1256 Exposure to have overscan correction performed. 1257 amp : `lsst.afw.table.AmpInfoCatalog` 1258 The amplifier to consider while correcting the overscan. 1262 overscanResults : `lsst.pipe.base.Struct` 1263 Result struct with components: 1264 - ``imageFit`` : scalar or `lsst.afw.image.Image` 1265 Value or fit subtracted from the amplifier image data. 1266 - ``overscanFit`` : scalar or `lsst.afw.image.Image` 1267 Value or fit subtracted from the overscan image data. 1268 - ``overscanImage`` : `lsst.afw.image.Image` 1269 Image of the overscan region with the overscan 1270 correction applied. This quantity is used to estimate 1271 the amplifier read noise empirically. 1276 Raised if the ``amp`` does not contain raw pixel information. 1280 lsst.ip.isr.isrFunctions.overscanCorrection 1282 if not amp.getHasRawInfo():
1283 raise RuntimeError(
"This method must be executed on an amp with raw information.")
1285 if amp.getRawHorizontalOverscanBBox().isEmpty():
1286 self.log.info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1290 ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(),
1292 overscanImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
1293 amp.getRawHorizontalOverscanBBox(),
1295 overscanArray = overscanImage.getImage().getArray()
1297 statControl = afwMath.StatisticsControl()
1298 statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask(
"SAT"))
1301 dataBBox = amp.getRawDataBBox()
1302 oscanBBox = amp.getRawHorizontalOverscanBBox()
1306 prescanBBox = amp.getRawPrescanBBox()
1307 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1308 x0 += self.config.overscanNumLeadingColumnsToSkip
1309 x1 -= self.config.overscanNumTrailingColumnsToSkip
1311 x0 += self.config.overscanNumTrailingColumnsToSkip
1312 x1 -= self.config.overscanNumLeadingColumnsToSkip
1318 if ((self.config.overscanBiasJump
and 1319 self.config.overscanBiasJumpLocation)
and 1320 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
and 1321 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in 1322 self.config.overscanBiasJumpDevices)):
1323 if amp.getReadoutCorner()
in (afwTable.LL, afwTable.LR):
1324 yLower = self.config.overscanBiasJumpLocation
1325 yUpper = dataBBox.getHeight() - yLower
1327 yUpper = self.config.overscanBiasJumpLocation
1328 yLower = dataBBox.getHeight() - yUpper
1330 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1331 afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1332 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1333 afwGeom.Extent2I(x0, 0),
1334 afwGeom.Extent2I(oscanBBox.getWidth() + x1, yLower)))
1336 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1337 afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1339 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(x0, yLower),
1340 afwGeom.Extent2I(oscanBBox.getWidth() + x1, yUpper)))
1342 imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1343 afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1345 overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(x0, 0),
1346 afwGeom.Extent2I(oscanBBox.getWidth() + x1,
1347 oscanBBox.getHeight())))
1350 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1351 ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), imageBBox,
1353 overscanImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), overscanBBox,
1356 overscanArray = overscanImage.getImage().getArray()
1357 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.getMask().getArray(),
1359 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1360 overscanImage.getMask().getArray()[bad] = overscanImage.getMask().getPlaneBitMask(
"SAT")
1362 statControl = afwMath.StatisticsControl()
1363 statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask(
"SAT"))
1365 overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1366 overscanImage=overscanImage,
1367 fitType=self.config.overscanFitType,
1368 order=self.config.overscanOrder,
1369 collapseRej=self.config.overscanNumSigmaClip,
1370 statControl=statControl,
1371 overscanIsInt=self.config.overscanIsInt
1375 levelStat = afwMath.MEDIAN
1376 sigmaStat = afwMath.STDEVCLIP
1378 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1379 self.config.qa.flatness.nIter)
1380 metadata = ccdExposure.getMetadata()
1381 ampNum = amp.getName()
1382 if self.config.overscanFitType
in (
"MEDIAN",
"MEAN",
"MEANCLIP"):
1383 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1384 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1386 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1387 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1388 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1390 return overscanResults
1393 """Set the variance plane using the amplifier gain and read noise 1395 The read noise is calculated from the ``overscanImage`` if the 1396 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 1397 the value from the amplifier data is used. 1401 ampExposure : `lsst.afw.image.Exposure` 1402 Exposure to process. 1403 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 1404 Amplifier detector data. 1405 overscanImage : `lsst.afw.image.MaskedImage`, optional. 1406 Image of overscan, required only for empirical read noise. 1410 lsst.ip.isr.isrFunctions.updateVariance 1412 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1413 gain = amp.getGain()
1415 if math.isnan(gain):
1417 self.log.warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1420 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
1421 (amp.getName(), gain, patchedGain))
1424 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
1425 self.log.info(
"Overscan is none for EmpiricalReadNoise")
1427 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
1428 stats = afwMath.StatisticsControl()
1429 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1430 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1431 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1433 readNoise = amp.getReadNoise()
1435 isrFunctions.updateVariance(
1436 maskedImage=ampExposure.getMaskedImage(),
1438 readNoise=readNoise,
1442 """!Apply dark correction in place. 1446 exposure : `lsst.afw.image.Exposure` 1447 Exposure to process. 1448 darkExposure : `lsst.afw.image.Exposure` 1449 Dark exposure of the same size as ``exposure``. 1450 invert : `Bool`, optional 1451 If True, re-add the dark to an already corrected image. 1456 Raised if either ``exposure`` or ``darkExposure`` do not 1457 have their dark time defined. 1461 lsst.ip.isr.isrFunctions.darkCorrection 1463 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1464 if math.isnan(expScale):
1465 raise RuntimeError(
"Exposure darktime is NAN")
1466 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1467 if math.isnan(darkScale):
1468 raise RuntimeError(
"Dark calib darktime is NAN")
1469 isrFunctions.darkCorrection(
1470 maskedImage=exposure.getMaskedImage(),
1471 darkMaskedImage=darkExposure.getMaskedImage(),
1473 darkScale=darkScale,
1475 trimToFit=self.config.doTrimToMatchCalib
1479 """!Check if linearization is needed for the detector cameraGeom. 1481 Checks config.doLinearize and the linearity type of the first 1486 detector : `lsst.afw.cameraGeom.Detector` 1487 Detector to get linearity type from. 1491 doLinearize : `Bool` 1492 If True, linearization should be performed. 1494 return self.config.doLinearize
and \
1495 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1498 """!Apply flat correction in place. 1502 exposure : `lsst.afw.image.Exposure` 1503 Exposure to process. 1504 flatExposure : `lsst.afw.image.Exposure` 1505 Flat exposure of the same size as ``exposure``. 1506 invert : `Bool`, optional 1507 If True, unflatten an already flattened image. 1511 lsst.ip.isr.isrFunctions.flatCorrection 1513 isrFunctions.flatCorrection(
1514 maskedImage=exposure.getMaskedImage(),
1515 flatMaskedImage=flatExposure.getMaskedImage(),
1516 scalingType=self.config.flatScalingType,
1517 userScale=self.config.flatUserScale,
1519 trimToFit=self.config.doTrimToMatchCalib
1523 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place. 1527 exposure : `lsst.afw.image.Exposure` 1528 Exposure to process. Only the amplifier DataSec is processed. 1529 amp : `lsst.afw.table.AmpInfoCatalog` 1530 Amplifier detector data. 1534 lsst.ip.isr.isrFunctions.makeThresholdMask 1536 if not math.isnan(amp.getSaturation()):
1537 maskedImage = exposure.getMaskedImage()
1538 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1539 isrFunctions.makeThresholdMask(
1540 maskedImage=dataView,
1541 threshold=amp.getSaturation(),
1543 maskName=self.config.saturatedMaskName,
1547 """!Interpolate over saturated pixels, in place. 1549 This method should be called after `saturationDetection`, to 1550 ensure that the saturated pixels have been identified in the 1551 SAT mask. It should also be called after `assembleCcd`, since 1552 saturated regions may cross amplifier boundaries. 1556 exposure : `lsst.afw.image.Exposure` 1557 Exposure to process. 1561 lsst.ip.isr.isrTask.saturationDetection 1562 lsst.ip.isr.isrFunctions.interpolateFromMask 1564 isrFunctions.interpolateFromMask(
1565 maskedImage=ccdExposure.getMaskedImage(),
1566 fwhm=self.config.fwhm,
1567 growFootprints=self.config.growSaturationFootprintSize,
1568 maskName=self.config.saturatedMaskName,
1572 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place. 1576 exposure : `lsst.afw.image.Exposure` 1577 Exposure to process. Only the amplifier DataSec is processed. 1578 amp : `lsst.afw.table.AmpInfoCatalog` 1579 Amplifier detector data. 1583 lsst.ip.isr.isrFunctions.makeThresholdMask 1587 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 1588 This is intended to indicate pixels that may be affected by unknown systematics; 1589 for example if non-linearity corrections above a certain level are unstable 1590 then that would be a useful value for suspectLevel. A value of `nan` indicates 1591 that no such level exists and no pixels are to be masked as suspicious. 1593 suspectLevel = amp.getSuspectLevel()
1594 if math.isnan(suspectLevel):
1597 maskedImage = exposure.getMaskedImage()
1598 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1599 isrFunctions.makeThresholdMask(
1600 maskedImage=dataView,
1601 threshold=suspectLevel,
1603 maskName=self.config.suspectMaskName,
1607 """!Mask defects using mask plane "BAD" and interpolate over them, in place. 1611 ccdExposure : `lsst.afw.image.Exposure` 1612 Exposure to process. 1613 defectBaseList : `List` 1614 List of defects to mask and interpolate. 1618 Call this after CCD assembly, since defects may cross amplifier boundaries. 1620 maskedImage = ccdExposure.getMaskedImage()
1622 for d
in defectBaseList:
1624 nd = measAlg.Defect(bbox)
1625 defectList.append(nd)
1626 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
1627 isrFunctions.interpolateDefectList(
1628 maskedImage=maskedImage,
1629 defectList=defectList,
1630 fwhm=self.config.fwhm,
1633 if self.config.numEdgeSuspect > 0:
1634 goodBBox = maskedImage.getBBox()
1636 goodBBox.grow(-self.config.numEdgeSuspect)
1638 SourceDetectionTask.setEdgeBits(
1641 maskedImage.getMask().getPlaneBitMask(
"SUSPECT")
1645 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place. 1649 exposure : `lsst.afw.image.Exposure` 1650 Exposure to process. 1654 We mask and interpolate over all NaNs, including those 1655 that are masked with other bits (because those may or may 1656 not be interpolated over later, and we want to remove all 1657 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 1658 is used to preserve the historical name. 1660 maskedImage = exposure.getMaskedImage()
1663 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
1664 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
1665 numNans =
maskNans(maskedImage, maskVal)
1666 self.metadata.set(
"NUMNANS", numNans)
1670 self.log.warn(
"There were %i unmasked NaNs", numNans)
1671 nanDefectList = isrFunctions.getDefectListFromMask(
1672 maskedImage=maskedImage,
1673 maskName=
'UNMASKEDNAN',
1675 isrFunctions.interpolateDefectList(
1676 maskedImage=exposure.getMaskedImage(),
1677 defectList=nanDefectList,
1678 fwhm=self.config.fwhm,
1682 """Measure the image background in subgrids, for quality control purposes. 1686 exposure : `lsst.afw.image.Exposure` 1687 Exposure to process. 1688 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig` 1689 Configuration object containing parameters on which background 1690 statistics and subgrids to use. 1692 if IsrQaConfig
is not None:
1693 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1694 IsrQaConfig.flatness.nIter)
1695 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
1696 statsControl.setAndMask(maskVal)
1697 maskedImage = exposure.getMaskedImage()
1698 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
1699 skyLevel = stats.getValue(afwMath.MEDIAN)
1700 skySigma = stats.getValue(afwMath.STDEVCLIP)
1701 self.log.info(
"Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
1702 metadata = exposure.getMetadata()
1703 metadata.set(
'SKYLEVEL', skyLevel)
1704 metadata.set(
'SKYSIGMA', skySigma)
1707 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
1708 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
1709 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
1710 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
1711 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
1712 skyLevels = numpy.zeros((nX, nY))
1715 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
1717 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
1719 xLLC = xc - meshXHalf
1720 yLLC = yc - meshYHalf
1721 xURC = xc + meshXHalf - 1
1722 yURC = yc + meshYHalf - 1
1724 bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
1725 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
1727 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
1729 good = numpy.where(numpy.isfinite(skyLevels))
1730 skyMedian = numpy.median(skyLevels[good])
1731 flatness = (skyLevels[good] - skyMedian) / skyMedian
1732 flatness_rms = numpy.std(flatness)
1733 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
1735 self.log.info(
"Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
1736 self.log.info(
"Sky flatness in %dx%d grids - pp: %f rms: %f" %
1737 (nX, nY, flatness_pp, flatness_rms))
1739 metadata.set(
'FLATNESS_PP', float(flatness_pp))
1740 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
1741 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
1742 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
1743 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
1746 """Set an approximate magnitude zero point for the exposure. 1750 exposure : `lsst.afw.image.Exposure` 1751 Exposure to process. 1753 filterName = afwImage.Filter(exposure.getFilter().getId()).getName()
1754 if filterName
in self.config.fluxMag0T1:
1755 fluxMag0 = self.config.fluxMag0T1[filterName]
1757 self.log.warn(
"No rough magnitude zero point set for filter %s" % filterName)
1758 fluxMag0 = self.config.defaultFluxMag0T1
1760 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
1762 self.log.warn(
"Non-positive exposure time; skipping rough zero point")
1765 self.log.info(
"Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
1766 exposure.getCalib().setFluxMag0(fluxMag0*expTime)
1769 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners. 1773 ccdExposure : `lsst.afw.image.Exposure` 1774 Exposure to process. 1775 fpPolygon : `lsst.afw.geom.Polygon` 1776 Polygon in focal plane coordinates. 1779 ccd = ccdExposure.getDetector()
1780 fpCorners = ccd.getCorners(FOCAL_PLANE)
1781 ccdPolygon = Polygon(fpCorners)
1784 intersect = ccdPolygon.intersectionSingle(fpPolygon)
1787 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
1788 validPolygon = Polygon(ccdPoints)
1789 ccdExposure.getInfo().setValidPolygon(validPolygon)
1793 """Context manager that applies and removes flats and darks, 1794 if the task is configured to apply them. 1798 exp : `lsst.afw.image.Exposure` 1799 Exposure to process. 1800 flat : `lsst.afw.image.Exposure` 1801 Flat exposure the same size as ``exp``. 1802 dark : `lsst.afw.image.Exposure`, optional 1803 Dark exposure the same size as ``exp``. 1807 exp : `lsst.afw.image.Exposure` 1808 The flat and dark corrected exposure. 1810 if self.config.doDark
and dark
is not None:
1812 if self.config.doFlat:
1817 if self.config.doFlat:
1819 if self.config.doDark
and dark
is not None:
1823 """Utility function to examine ISR exposure at different stages. 1827 exposure : `lsst.afw.image.Exposure` 1830 State of processing to view. 1832 frame = getDebugFrame(self._display, stepname)
1834 display = getDisplay(frame)
1835 display.scale(
'asinh',
'zscale')
1836 display.mtv(exposure)
1840 """A Detector-like object that supports returning gain and saturation level 1842 This is used when the input exposure does not have a detector. 1846 exposure : `lsst.afw.image.Exposure` 1847 Exposure to generate a fake amplifier for. 1848 config : `lsst.ip.isr.isrTaskConfig` 1849 Configuration to apply to the fake amplifier. 1853 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1855 self.
_gain = config.gain
1885 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
1889 """Task to wrap the default IsrTask to allow it to be retargeted. 1891 The standard IsrTask can be called directly from a command line 1892 program, but doing so removes the ability of the task to be 1893 retargeted. As most cameras override some set of the IsrTask 1894 methods, this would remove those data-specific methods in the 1895 output post-ISR images. This wrapping class fixes the issue, 1896 allowing identical post-ISR images to be generated by both the 1897 processCcd and isrTask code. 1899 ConfigClass = RunIsrConfig
1900 _DefaultName =
"runIsr" 1904 self.makeSubtask(
"isr")
1910 dataRef : `lsst.daf.persistence.ButlerDataRef` 1911 data reference of the detector data to be processed 1915 result : `pipeBase.Struct` 1916 Result struct with component: 1918 - exposure : `lsst.afw.image.Exposure` 1919 Post-ISR processed exposure.
def runDataRef(self, sensorRef)
def measureBackground(self, exposure, IsrQaConfig=None)
def debugView(self, exposure, stepname)
def __init__(self, args, kwargs)
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
def roughZeroPoint(self, exposure)
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Perform instrument signature removal on an exposure.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def overscanCorrection(self, ccdExposure, amp)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def maskAmplifier(self, ccdExposure, amp, defects)
def flatContext(self, exp, flat, dark=None)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)