32 from lsstDebug
import getDebugFrame
34 from .
import isrFunctions
35 from .assembleCcdTask
import AssembleCcdTask
36 from .fringe
import FringeTask
40 from contextlib
import contextmanager
41 from .isr
import maskNans
42 from .crosstalk
import CrosstalkTask
46 doBias = pexConfig.Field(
48 doc=
"Apply bias frame correction?",
51 doDark = pexConfig.Field(
53 doc=
"Apply dark frame correction?",
56 doFlat = pexConfig.Field(
58 doc=
"Apply flat field correction?",
61 doFringe = pexConfig.Field(
63 doc=
"Apply fringe correction?",
66 doDefect = pexConfig.Field(
68 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
71 doAddDistortionModel = pexConfig.Field(
73 doc=
"Apply a distortion model based on camera geometry to the WCS?",
76 doWrite = pexConfig.Field(
78 doc=
"Persist postISRCCD?",
81 biasDataProductName = pexConfig.Field(
83 doc=
"Name of the bias data product",
86 darkDataProductName = pexConfig.Field(
88 doc=
"Name of the dark data product",
91 flatDataProductName = pexConfig.Field(
93 doc=
"Name of the flat data product",
96 assembleCcd = pexConfig.ConfigurableField(
97 target=AssembleCcdTask,
98 doc=
"CCD assembly task",
100 gain = pexConfig.Field(
102 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
103 default=float(
"NaN"),
105 readNoise = pexConfig.Field(
107 doc=
"The read noise to use if no Detector is present in the Exposure",
110 saturation = pexConfig.Field(
112 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
113 default=float(
"NaN"),
115 fringeAfterFlat = pexConfig.Field(
117 doc=
"Do fringe subtraction after flat-fielding?",
120 fringe = pexConfig.ConfigurableField(
122 doc=
"Fringe subtraction task",
124 fwhm = pexConfig.Field(
126 doc=
"FWHM of PSF (arcsec)",
129 saturatedMaskName = pexConfig.Field(
131 doc=
"Name of mask plane to use in saturation detection and interpolation",
134 suspectMaskName = pexConfig.Field(
136 doc=
"Name of mask plane to use for suspect pixels",
139 flatScalingType = pexConfig.ChoiceField(
141 doc=
"The method for scaling the flat on the fly.",
144 "USER":
"Scale by flatUserScale",
145 "MEAN":
"Scale by the inverse of the mean",
146 "MEDIAN":
"Scale by the inverse of the median",
149 flatUserScale = pexConfig.Field(
151 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
154 overscanFitType = pexConfig.ChoiceField(
156 doc=
"The method for fitting the overscan bias level.",
159 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
160 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
161 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
162 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
163 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
164 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
165 "MEAN":
"Correct using the mean of the overscan region",
166 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
167 "MEDIAN":
"Correct using the median of the overscan region",
170 overscanOrder = pexConfig.Field(
172 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
173 "or number of spline knots if overscan fit type is a spline."),
176 overscanNumSigmaClip = pexConfig.Field(
178 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
182 overscanNumLeadingColumnsToSkip = pexConfig.Field(
184 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
187 overscanNumTrailingColumnsToSkip = pexConfig.Field(
189 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
192 growSaturationFootprintSize = pexConfig.Field(
194 doc=
"Number of pixels by which to grow the saturation footprints",
197 doSaturationInterpolation = pexConfig.Field(
199 doc=
"Perform interpolation over pixels masked as saturated?",
202 doNanInterpAfterFlat = pexConfig.Field(
204 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 205 "also have to interpolate them before flat-fielding."),
208 fluxMag0T1 = pexConfig.Field(
210 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure",
213 keysToRemoveFromAssembledCcd = pexConfig.ListField(
215 doc=
"fields to remove from the metadata of the assembled ccd.",
218 doAssembleIsrExposures = pexConfig.Field(
221 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 223 doAssembleCcd = pexConfig.Field(
226 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 228 expectWcs = pexConfig.Field(
231 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)" 233 doLinearize = pexConfig.Field(
235 doc=
"Correct for nonlinearity of the detector's response?",
238 doCrosstalk = pexConfig.Field(
240 doc=
"Apply intra-CCD crosstalk correction?",
243 crosstalk = pexConfig.ConfigurableField(
244 target=CrosstalkTask,
245 doc=
"Intra-CCD crosstalk correction",
247 doBrighterFatter = pexConfig.Field(
250 doc=
"Apply the brighter fatter correction" 252 brighterFatterLevel = pexConfig.ChoiceField(
253 doc=
"The level at which to correct for brighter-fatter",
254 dtype=str, default=
"DETECTOR",
256 "AMP":
"Every amplifier treated separately",
257 "DETECTOR":
"One kernel per detector",
260 brighterFatterKernelFile = pexConfig.Field(
263 doc=
"Kernel file used for the brighter fatter correction" 265 brighterFatterMaxIter = pexConfig.Field(
268 doc=
"Maximum number of iterations for the brighter fatter correction" 270 brighterFatterThreshold = pexConfig.Field(
273 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 274 " absolute value of the difference between the current corrected image and the one" 275 " from the previous iteration summed over all the pixels." 277 brighterFatterApplyGain = pexConfig.Field(
280 doc=
"Should the gain be applied when applying the brighter fatter correction?" 282 datasetType = pexConfig.Field(
284 doc=
"Dataset type for input data; users will typically leave this alone, " 285 "but camera-specific ISR tasks will override it",
288 fallbackFilterName = pexConfig.Field(dtype=str,
289 doc=
"Fallback default filter name for calibrations", optional=
True)
290 doAttachTransmissionCurve = pexConfig.Field(
293 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 295 doUseOpticsTransmission = pexConfig.Field(
298 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 300 doUseFilterTransmission = pexConfig.Field(
303 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 305 doUseSensorTransmission = pexConfig.Field(
308 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 310 doUseAtmosphereTransmission = pexConfig.Field(
313 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 315 doEmpiricalReadNoise = pexConfig.Field(
318 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 333 @brief Apply common instrument signature correction algorithms to a raw frame. 335 @section ip_isr_isr_Contents Contents 337 - @ref ip_isr_isr_Purpose 338 - @ref ip_isr_isr_Initialize 340 - @ref ip_isr_isr_Config 341 - @ref ip_isr_isr_Debug 344 @section ip_isr_isr_Purpose Description 346 The process for correcting imaging data is very similar from camera to camera. 347 This task provides a vanilla implementation of doing these corrections, including 348 the ability to turn certain corrections off if they are not needed. 349 The inputs to the primary method, run, are a raw exposure to be corrected and the 350 calibration data products. The raw input is a single chip sized mosaic of all amps 351 including overscans and other non-science pixels. 352 The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 353 and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 354 This task may not meet all needs and it is expected that it will be subclassed for 355 specific applications. 357 @section ip_isr_isr_Initialize Task initialization 359 @copydoc \_\_init\_\_ 361 @section ip_isr_isr_IO Inputs/Outputs to the run method 365 @section ip_isr_isr_Config Configuration parameters 367 See @ref IsrTaskConfig 369 @section ip_isr_isr_Debug Debug variables 371 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 372 flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 373 href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 374 Using lsstDebug to control debugging output</a> for more about @b debug.py files. 376 The available variables in IsrTask are: 379 <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 382 <DD> display exposure after ISR has been applied 386 For example, put something like 390 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 391 if name == "lsst.ip.isr.isrTask": 392 di.display = {'postISRCCD':2} 394 lsstDebug.Info = DebugInfo 396 into your debug.py file and run the commandline task with the @c --debug flag. 400 ConfigClass = IsrTaskConfig
404 '''!Constructor for IsrTask 405 @param[in] *args a list of positional arguments passed on to the Task constructor 406 @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 407 Call the lsst.pipe.base.task.Task.__init__ method 408 Then setup the assembly and fringe correction subtasks 410 pipeBase.Task.__init__(self, *args, **kwargs)
411 self.makeSubtask(
"assembleCcd")
412 self.makeSubtask(
"fringe")
413 self.makeSubtask(
"crosstalk")
416 """!Retrieve necessary frames for instrument signature removal 417 @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 418 of the detector data to be processed 419 @param[in] rawExposure a reference raw exposure that will later be 420 corrected with the retrieved calibration data; 421 should not be modified in this method. 422 @return a pipeBase.Struct with fields containing kwargs expected by run() 423 - bias: exposure of bias frame 424 - dark: exposure of dark frame 425 - flat: exposure of flat field 426 - defects: list of detects 427 - fringeStruct: a pipeBase.Struct with field fringes containing 428 exposure of fringe frame or list of fringe exposure 430 ccd = rawExposure.getDetector()
432 biasExposure = self.
getIsrExposure(dataRef, self.config.biasDataProductName) \
433 if self.config.doBias
else None 435 linearizer = dataRef.get(
"linearizer", immediate=
True)
if self.
doLinearize(ccd)
else None 436 darkExposure = self.
getIsrExposure(dataRef, self.config.darkDataProductName) \
437 if self.config.doDark
else None 438 flatExposure = self.
getIsrExposure(dataRef, self.config.flatDataProductName) \
439 if self.config.doFlat
else None 440 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
if self.config.doBrighterFatter
else None 441 defectList = dataRef.get(
"defects")
if self.config.doDefect
else None 443 if self.config.doCrosstalk:
444 crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
446 crosstalkSources =
None 448 if self.config.doFringe
and self.fringe.checkFilter(rawExposure):
449 fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
450 if self.config.doAssembleIsrExposures
else None)
452 fringeStruct = pipeBase.Struct(fringes=
None)
454 if self.config.doAttachTransmissionCurve:
455 opticsTransmission = (dataRef.get(
"transmission_optics")
456 if self.config.doUseOpticsTransmission
else None)
457 filterTransmission = (dataRef.get(
"transmission_filter")
458 if self.config.doUseFilterTransmission
else None)
459 sensorTransmission = (dataRef.get(
"transmission_sensor")
460 if self.config.doUseSensorTransmission
else None)
461 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
462 if self.config.doUseAtmosphereTransmission
else None)
464 opticsTransmission =
None 465 filterTransmission =
None 466 sensorTransmission =
None 467 atmosphereTransmission =
None 470 return pipeBase.Struct(bias=biasExposure,
471 linearizer=linearizer,
475 fringes=fringeStruct,
476 bfKernel=brighterFatterKernel,
477 opticsTransmission=opticsTransmission,
478 filterTransmission=filterTransmission,
479 sensorTransmission=sensorTransmission,
480 atmosphereTransmission=atmosphereTransmission,
481 crosstalkSources=crosstalkSources,
485 def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
486 fringes=None, bfKernel=None, camera=None,
487 opticsTransmission=None, filterTransmission=None,
488 sensorTransmission=None, atmosphereTransmission=None,
489 crosstalkSources=None):
490 """!Perform instrument signature removal on an exposure 493 - Detect saturation, apply overscan correction, bias, dark and flat 494 - Perform CCD assembly 495 - Interpolate over defects, saturated pixels and all NaNs 497 @param[in] ccdExposure lsst.afw.image.exposure of detector data 498 @param[in] bias exposure of bias frame 499 @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 500 @param[in] dark exposure of dark frame 501 @param[in] flat exposure of flatfield 502 @param[in] defects list of detects 503 @param[in] fringes a pipeBase.Struct with field fringes containing 504 exposure of fringe frame or list of fringe exposure 505 @param[in] bfKernel kernel for brighter-fatter correction, an 506 lsst.cp.pipe.makeBrighterFatterKernel.BrighterFatterKernel object 507 @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 508 used by addDistortionModel 509 @param[in] opticsTransmission a TransmissionCurve for the optics 510 @param[in] filterTransmission a TransmissionCurve for the filter 511 @param[in] sensorTransmission a TransmissionCurve for the sensor 512 @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 513 @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk 515 @return a pipeBase.Struct with field: 519 if isinstance(ccdExposure, ButlerDataRef):
522 ccd = ccdExposure.getDetector()
525 if self.config.doBias
and bias
is None:
526 raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
528 raise RuntimeError(
"Must supply a linearizer if config.doBias True")
529 if self.config.doDark
and dark
is None:
530 raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
531 if self.config.doFlat
and flat
is None:
532 raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
533 if self.config.doBrighterFatter
and bfKernel
is None:
534 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter True")
536 fringes = pipeBase.Struct(fringes=
None)
537 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
538 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
539 if self.config.doDefect
and defects
is None:
540 raise RuntimeError(
"Must supply defects if config.doDefect True")
541 if self.config.doAddDistortionModel
and camera
is None:
542 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel True")
547 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 548 ccd = [
FakeAmp(ccdExposure, self.config)]
553 if ccdExposure.getBBox().contains(amp.getBBox()):
557 overscans.append(overscanResults.overscanImage
if overscanResults
is not None else None)
559 overscans.append(
None)
561 if self.config.doCrosstalk:
562 self.crosstalk.
run(ccdExposure, crosstalkSources)
564 if self.config.doAssembleCcd:
565 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
566 if self.config.expectWcs
and not ccdExposure.getWcs():
567 self.log.warn(
"No WCS found in input exposure")
569 if self.config.doBias:
573 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
575 assert len(ccd) == len(overscans)
576 for amp, overscanImage
in zip(ccd, overscans):
578 if ccdExposure.getBBox().contains(amp.getBBox()):
579 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
582 interpolationDone =
False 584 if self.config.doBrighterFatter:
591 if self.config.doDefect:
593 if self.config.doSaturationInterpolation:
596 interpolationDone =
True 598 if self.config.brighterFatterLevel ==
'DETECTOR':
599 kernelElement = bfKernel.kernel[ccdExposure.getDetector().getId()]
602 raise NotImplementedError(
"per-amplifier brighter-fatter correction not yet implemented")
604 self.config.brighterFatterMaxIter,
605 self.config.brighterFatterThreshold,
606 self.config.brighterFatterApplyGain,
609 if self.config.doDark:
612 if self.config.doFringe
and not self.config.fringeAfterFlat:
613 self.fringe.
run(ccdExposure, **fringes.getDict())
615 if self.config.doFlat:
618 if not interpolationDone:
619 if self.config.doDefect:
621 if self.config.doSaturationInterpolation:
623 if not interpolationDone
or self.config.doNanInterpAfterFlat:
626 if self.config.doFringe
and self.config.fringeAfterFlat:
627 self.fringe.
run(ccdExposure, **fringes.getDict())
629 exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
630 ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
632 if self.config.doAddDistortionModel:
635 if self.config.doAttachTransmissionCurve:
637 filterTransmission=filterTransmission,
638 sensorTransmission=sensorTransmission,
639 atmosphereTransmission=atmosphereTransmission)
641 frame = getDebugFrame(self._display,
"postISRCCD")
643 display = getDisplay(frame)
644 display.scale(
'asinh',
'zscale')
645 display.mtv(ccdExposure)
647 return pipeBase.Struct(
648 exposure=ccdExposure,
653 """Perform instrument signature removal on a ButlerDataRef of a Sensor 655 - Read in necessary detrending/isr/calibration data 656 - Process raw exposure in run() 657 - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 661 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 662 DataRef of the detector data to be processed 666 result : `pipeBase.Struct` 667 Struct contains field "exposure," which is the exposure after application of ISR 669 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
670 ccdExposure = sensorRef.get(
'raw')
671 camera = sensorRef.get(
"camera")
672 if camera
is None and self.config.doAddDistortionModel:
673 raise RuntimeError(
"config.doAddDistortionModel is True " 674 "but could not get a camera from the butler")
677 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
679 if self.config.doWrite:
680 sensorRef.put(result.exposure,
"postISRCCD")
685 """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 687 if isinstance(exposure, afwImage.ExposureF):
690 if not hasattr(exposure,
"convertF"):
691 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
693 newexposure = exposure.convertF()
694 newexposure.variance[:] = 1
695 newexposure.mask[:] = 0x0
700 """!Apply bias correction in place 702 @param[in,out] exposure exposure to process 703 @param[in] biasExposure bias exposure of same size as exposure 705 isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
708 """!Apply dark correction in place 710 @param[in,out] exposure exposure to process 711 @param[in] darkExposure dark exposure of same size as exposure 712 @param[in] invert if True, remove the dark from an already-corrected image 714 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
715 if math.isnan(expScale):
716 raise RuntimeError(
"Exposure darktime is NAN")
717 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
718 if math.isnan(darkScale):
719 raise RuntimeError(
"Dark calib darktime is NAN")
720 isrFunctions.darkCorrection(
721 maskedImage=exposure.getMaskedImage(),
722 darkMaskedImage=darkExposure.getMaskedImage(),
729 """!Is linearization wanted for this detector? 731 Checks config.doLinearize and the linearity type of the first amplifier. 733 @param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 735 return self.config.doLinearize
and \
736 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
739 """Set the variance plane using the amplifier gain and read noise 741 The read noise is calculated from the ``overscanImage`` if the 742 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 743 the value from the amplifier data is used. 747 ampExposure : `lsst.afw.image.Exposure` 749 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 750 Amplifier detector data. 751 overscanImage : `lsst.afw.image.MaskedImage`, optional. 752 Image of overscan, required only for empirical read noise. 754 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
756 if not math.isnan(gain):
759 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
760 (amp.getName(), gain, patchedGain))
763 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
764 stats = afwMath.StatisticsControl()
765 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
766 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
767 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
769 readNoise = amp.getReadNoise()
771 isrFunctions.updateVariance(
772 maskedImage=ampExposure.getMaskedImage(),
778 """!Apply flat correction in place 780 @param[in,out] exposure exposure to process 781 @param[in] flatExposure flatfield exposure same size as exposure 782 @param[in] invert if True, unflatten an already-flattened image instead. 784 isrFunctions.flatCorrection(
785 maskedImage=exposure.getMaskedImage(),
786 flatMaskedImage=flatExposure.getMaskedImage(),
787 scalingType=self.config.flatScalingType,
788 userScale=self.config.flatUserScale,
793 """!Retrieve a calibration dataset for removing instrument signature 795 @param[in] dataRef data reference for exposure 796 @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 797 @param[in] immediate if True, disable butler proxies to enable error 798 handling within this routine 802 exp = dataRef.get(datasetType, immediate=immediate)
803 except Exception
as exc1:
804 if not self.config.fallbackFilterName:
805 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
807 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
808 except Exception
as exc2:
809 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
810 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
811 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
813 if self.config.doAssembleIsrExposures:
814 exp = self.assembleCcd.assembleCcd(exp)
818 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 820 @param[in,out] exposure exposure to process; only the amp DataSec is processed 821 @param[in] amp amplifier device data 823 if not math.isnan(amp.getSaturation()):
824 maskedImage = exposure.getMaskedImage()
825 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
826 isrFunctions.makeThresholdMask(
827 maskedImage=dataView,
828 threshold=amp.getSaturation(),
830 maskName=self.config.saturatedMaskName,
834 """!Interpolate over saturated pixels, in place 836 @param[in,out] ccdExposure exposure to process 839 - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 840 - Call this after CCD assembly, since saturated regions may cross amplifier boundaries 842 isrFunctions.interpolateFromMask(
843 maskedImage=ccdExposure.getMaskedImage(),
844 fwhm=self.config.fwhm,
845 growFootprints=self.config.growSaturationFootprintSize,
846 maskName=self.config.saturatedMaskName,
850 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 852 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 853 This is intended to indicate pixels that may be affected by unknown systematics; 854 for example if non-linearity corrections above a certain level are unstable 855 then that would be a useful value for suspectLevel. A value of `nan` indicates 856 that no such level exists and no pixels are to be masked as suspicious. 858 @param[in,out] exposure exposure to process; only the amp DataSec is processed 859 @param[in] amp amplifier device data 861 suspectLevel = amp.getSuspectLevel()
862 if math.isnan(suspectLevel):
865 maskedImage = exposure.getMaskedImage()
866 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
867 isrFunctions.makeThresholdMask(
868 maskedImage=dataView,
869 threshold=suspectLevel,
871 maskName=self.config.suspectMaskName,
875 """!Mask defects using mask plane "BAD" and interpolate over them, in place 877 @param[in,out] ccdExposure exposure to process 878 @param[in] defectBaseList a list of defects to mask and interpolate 880 @warning: call this after CCD assembly, since defects may cross amplifier boundaries 882 maskedImage = ccdExposure.getMaskedImage()
884 for d
in defectBaseList:
886 nd = measAlg.Defect(bbox)
887 defectList.append(nd)
888 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
889 isrFunctions.interpolateDefectList(
890 maskedImage=maskedImage,
891 defectList=defectList,
892 fwhm=self.config.fwhm,
896 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 898 We mask and interpolate over all NaNs, including those 899 that are masked with other bits (because those may or may 900 not be interpolated over later, and we want to remove all 901 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 902 is used to preserve the historical name. 904 @param[in,out] exposure exposure to process 906 maskedImage = exposure.getMaskedImage()
909 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
910 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
911 numNans =
maskNans(maskedImage, maskVal)
912 self.metadata.set(
"NUMNANS", numNans)
916 self.log.warn(
"There were %i unmasked NaNs", numNans)
917 nanDefectList = isrFunctions.getDefectListFromMask(
918 maskedImage=maskedImage,
919 maskName=
'UNMASKEDNAN',
921 isrFunctions.interpolateDefectList(
922 maskedImage=exposure.getMaskedImage(),
923 defectList=nanDefectList,
924 fwhm=self.config.fwhm,
928 """Apply overscan correction, in-place 932 exposure : `lsst.afw.image.Exposure` 933 Exposure to process; must include both data and bias regions. 934 amp : `lsst.afw.table.AmpInfoRecord` 935 Amplifier device data. 939 result : `lsst.pipe.base.Struct` or `NoneType` 940 `None` if there is no overscan; otherwise, this is a 941 result struct with components: 943 - ``imageFit``: Value(s) removed from image (scalar or 944 `lsst.afw.image.Image`). 945 - ``overscanFit``: Value(s) removed from overscan (scalar or 946 `lsst.afw.image.Image`). 947 - ``overscanImage``: Image of the overscan, post-subtraction 948 (`lsst.afw.image.Image`). 950 if not amp.getHasRawInfo():
951 raise RuntimeError(
"This method must be executed on an amp with raw information.")
953 if amp.getRawHorizontalOverscanBBox().isEmpty():
954 self.log.info(
"No Overscan region. Not performing Overscan Correction.")
957 oscanBBox = amp.getRawHorizontalOverscanBBox()
960 x0, x1 = oscanBBox.getBeginX(), oscanBBox.getEndX()
962 prescanBBox = amp.getRawPrescanBBox()
963 if oscanBBox.getBeginX() > prescanBBox.getBeginX():
964 x0 += self.config.overscanNumLeadingColumnsToSkip
965 x1 -= self.config.overscanNumTrailingColumnsToSkip
967 x0 += self.config.overscanNumTrailingColumnsToSkip
968 x1 -= self.config.overscanNumLeadingColumnsToSkip
970 oscanBBox = afwGeom.BoxI(afwGeom.PointI(x0, oscanBBox.getBeginY()),
971 afwGeom.PointI(x1 - 1, oscanBBox.getEndY() - 1))
973 maskedImage = exposure.maskedImage
974 dataView = maskedImage[amp.getRawDataBBox()]
975 overscanImage = maskedImage[oscanBBox]
977 sctrl = afwMath.StatisticsControl()
978 sctrl.setNumSigmaClip(self.config.overscanNumSigmaClip)
980 results = isrFunctions.overscanCorrection(
981 ampMaskedImage=dataView,
982 overscanImage=overscanImage,
983 fitType=self.config.overscanFitType,
984 order=self.config.overscanOrder,
987 results.overscanImage = overscanImage
991 """!Update the WCS in exposure with a distortion model based on camera geometry 993 Add a model for optical distortion based on geometry found in `camera` 994 and the `exposure`'s detector. The raw input exposure is assumed 995 have a TAN WCS that has no compensation for optical distortion. 996 Two other possibilities are: 997 - The raw input exposure already has a model for optical distortion, 998 as is the case for raw DECam data. 999 In that case you should set config.doAddDistortionModel False. 1000 - The raw input exposure has a model for distortion, but it has known 1001 deficiencies severe enough to be worth fixing (e.g. because they 1002 cause problems for fitting a better WCS). In that case you should 1003 override this method with a version suitable for your raw data. 1005 @param[in,out] exposure exposure to process; must include a Detector and a WCS; 1006 the WCS of the exposure is modified in place 1007 @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 1009 self.log.info(
"Adding a distortion model to the WCS")
1010 wcs = exposure.getWcs()
1012 raise RuntimeError(
"exposure has no WCS")
1014 raise RuntimeError(
"camera is None")
1015 detector = exposure.getDetector()
1016 if detector
is None:
1017 raise RuntimeError(
"exposure has no Detector")
1018 pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
1019 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
1020 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
1021 exposure.setWcs(distortedWcs)
1024 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners 1026 @param[in,out] ccdExposure exposure to process 1027 @param[in] fpPolygon Polygon in focal plane coordinates 1030 ccd = ccdExposure.getDetector()
1031 fpCorners = ccd.getCorners(FOCAL_PLANE)
1032 ccdPolygon = Polygon(fpCorners)
1035 intersect = ccdPolygon.intersectionSingle(fpPolygon)
1038 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
1039 validPolygon = Polygon(ccdPoints)
1040 ccdExposure.getInfo().setValidPolygon(validPolygon)
1043 """Apply brighter fatter correction in place for the image 1045 This correction takes a kernel that has been derived from flat field images to 1046 redistribute the charge. The gradient of the kernel is the deflection 1047 field due to the accumulated charge. 1049 Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 1050 using the following equation: 1052 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 1054 To evaluate the derivative term we expand it as follows: 1056 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 1058 Because we use the measured counts instead of the incident counts we apply the correction 1059 iteratively to reconstruct the original counts and the correction. We stop iterating when the 1060 summed difference between the current corrected image and the one from the previous iteration 1061 is below the threshold. We do not require convergence because the number of iterations is 1062 too large a computational cost. How we define the threshold still needs to be evaluated, the 1063 current default was shown to work reasonably well on a small set of images. For more information 1064 on the method see DocuShare Document-19407. 1066 The edges as defined by the kernel are not corrected because they have spurious values 1067 due to the convolution. 1069 self.log.info(
"Applying brighter fatter correction")
1071 image = exposure.getMaskedImage().getImage()
1074 with self.
gainContext(exposure, image, applyGain):
1076 kLx = numpy.shape(kernel)[0]
1077 kLy = numpy.shape(kernel)[1]
1078 kernelImage = afwImage.ImageD(kLx, kLy)
1079 kernelImage.getArray()[:, :] = kernel
1080 tempImage = image.clone()
1082 nanIndex = numpy.isnan(tempImage.getArray())
1083 tempImage.getArray()[nanIndex] = 0.
1085 outImage = afwImage.ImageF(image.getDimensions())
1086 corr = numpy.zeros_like(image.getArray())
1087 prev_image = numpy.zeros_like(image.getArray())
1088 convCntrl = afwMath.ConvolutionControl(
False,
True, 1)
1089 fixedKernel = afwMath.FixedKernel(kernelImage)
1099 for iteration
in range(maxIter):
1101 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1102 tmpArray = tempImage.getArray()
1103 outArray = outImage.getArray()
1105 with numpy.errstate(invalid=
"ignore", over=
"ignore"):
1107 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1108 gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1109 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1112 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1113 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1114 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1116 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1118 tmpArray[:, :] = image.getArray()[:, :]
1119 tmpArray[nanIndex] = 0.
1120 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1123 diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1125 if diff < threshold:
1127 prev_image[:, :] = tmpArray[:, :]
1129 if iteration == maxIter - 1:
1130 self.log.warn(
"Brighter fatter correction did not converge, final difference %f" % diff)
1132 self.log.info(
"Finished brighter fatter in %d iterations" % (iteration + 1))
1133 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1134 corr[startY + 1:endY - 1, startX + 1:endX - 1]
1137 sensorTransmission=None, atmosphereTransmission=None):
1138 """Attach a TransmissionCurve to an Exposure, given separate curves for 1139 different components. 1143 exposure : `lsst.afw.image.Exposure` 1144 Exposure object to modify by attaching the product of all given 1145 ``TransmissionCurves`` in post-assembly trimmed detector 1146 coordinates. Must have a valid ``Detector`` attached that matches 1147 the detector associated with sensorTransmission. 1148 opticsTransmission : `lsst.afw.image.TransmissionCurve` 1149 A ``TransmissionCurve`` that represents the throughput of the 1150 optics, to be evaluated in focal-plane coordinates. 1151 filterTransmission : `lsst.afw.image.TransmissionCurve` 1152 A ``TransmissionCurve`` that represents the throughput of the 1153 filter itself, to be evaluated in focal-plane coordinates. 1154 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1155 A ``TransmissionCurve`` that represents the throughput of the 1156 sensor itself, to be evaluated in post-assembly trimmed detector 1158 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1159 A ``TransmissionCurve`` that represents the throughput of the 1160 atmosphere, assumed to be spatially constant. 1162 All ``TransmissionCurve`` arguments are optional; if none are provided, 1163 the attached ``TransmissionCurve`` will have unit transmission 1168 combined : ``lsst.afw.image.TransmissionCurve`` 1169 The TransmissionCurve attached to the exposure. 1171 return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1172 filterTransmission=filterTransmission,
1173 sensorTransmission=sensorTransmission,
1174 atmosphereTransmission=atmosphereTransmission)
1178 """Context manager that applies and removes gain 1181 ccd = exp.getDetector()
1183 sim = image.Factory(image, amp.getBBox())
1184 sim *= amp.getGain()
1190 ccd = exp.getDetector()
1192 sim = image.Factory(image, amp.getBBox())
1193 sim /= amp.getGain()
1197 """Context manager that applies and removes flats and darks, 1198 if the task is configured to apply them. 1200 if self.config.doDark
and dark
is not None:
1202 if self.config.doFlat:
1207 if self.config.doFlat:
1209 if self.config.doDark
and dark
is not None:
1214 """A Detector-like object that supports returning gain and saturation level""" 1217 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1219 self.
_gain = config.gain
1249 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
1260 """Task to wrap the default IsrTask to allow it to be retargeted. 1262 The standard IsrTask can be called directly from a command line 1263 program, but doing so removes the ability of the task to be 1264 retargeted. As most cameras override some set of the IsrTask 1265 methods, this would remove those data-specific methods in the 1266 output post-ISR images. This wrapping class fixes the issue, 1267 allowing identical post-ISR images to be generated by both the 1268 processCcd and isrTask code. 1270 ConfigClass = RunIsrConfig
1271 _DefaultName =
"runIsr" 1275 self.makeSubtask(
"isr")
1281 dataRef : `lsst.daf.persistence.ButlerDataRef` 1282 data reference of the detector data to be processed 1286 result : `pipeBase.Struct` 1287 Result struct with component: 1289 - exposure : `lsst.afw.image.Exposure` 1290 Post-ISR processed exposure.
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
def runDataRef(self, sensorRef)
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
def gainContext(self, exp, image, apply)
def __init__(self, args, kwargs)
Constructor for IsrTask.
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Apply common instrument signature correction algorithms to a raw frame.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Is linearization wanted for this detector?
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
def flatContext(self, exp, flat, dark=None)
def overscanCorrection(self, exposure, amp)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)