32 from lsstDebug
import getDebugFrame
34 from .
import isrFunctions
35 from .assembleCcdTask
import AssembleCcdTask
36 from .fringe
import FringeTask
40 from contextlib
import contextmanager
41 from .isr
import maskNans
42 from .crosstalk
import CrosstalkTask
46 doBias = pexConfig.Field(
48 doc=
"Apply bias frame correction?",
51 doDark = pexConfig.Field(
53 doc=
"Apply dark frame correction?",
56 doFlat = pexConfig.Field(
58 doc=
"Apply flat field correction?",
61 doFringe = pexConfig.Field(
63 doc=
"Apply fringe correction?",
66 doDefect = pexConfig.Field(
68 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
71 doAddDistortionModel = pexConfig.Field(
73 doc=
"Apply a distortion model based on camera geometry to the WCS?",
76 doWrite = pexConfig.Field(
78 doc=
"Persist postISRCCD?",
81 biasDataProductName = pexConfig.Field(
83 doc=
"Name of the bias data product",
86 darkDataProductName = pexConfig.Field(
88 doc=
"Name of the dark data product",
91 flatDataProductName = pexConfig.Field(
93 doc=
"Name of the flat data product",
96 assembleCcd = pexConfig.ConfigurableField(
97 target=AssembleCcdTask,
98 doc=
"CCD assembly task",
100 gain = pexConfig.Field(
102 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
103 default=float(
"NaN"),
105 readNoise = pexConfig.Field(
107 doc=
"The read noise to use if no Detector is present in the Exposure",
110 saturation = pexConfig.Field(
112 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
113 default=float(
"NaN"),
115 fringeAfterFlat = pexConfig.Field(
117 doc=
"Do fringe subtraction after flat-fielding?",
120 fringe = pexConfig.ConfigurableField(
122 doc=
"Fringe subtraction task",
124 fwhm = pexConfig.Field(
126 doc=
"FWHM of PSF (arcsec)",
129 saturatedMaskName = pexConfig.Field(
131 doc=
"Name of mask plane to use in saturation detection and interpolation",
134 suspectMaskName = pexConfig.Field(
136 doc=
"Name of mask plane to use for suspect pixels",
139 flatScalingType = pexConfig.ChoiceField(
141 doc=
"The method for scaling the flat on the fly.",
144 "USER":
"Scale by flatUserScale",
145 "MEAN":
"Scale by the inverse of the mean",
146 "MEDIAN":
"Scale by the inverse of the median",
149 flatUserScale = pexConfig.Field(
151 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
154 overscanFitType = pexConfig.ChoiceField(
156 doc=
"The method for fitting the overscan bias level.",
159 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
160 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
161 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
162 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
163 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
164 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
165 "MEAN":
"Correct using the mean of the overscan region",
166 "MEDIAN":
"Correct using the median of the overscan region",
169 overscanOrder = pexConfig.Field(
171 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
172 "or number of spline knots if overscan fit type is a spline."),
175 overscanRej = pexConfig.Field(
177 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
181 overscanNumLeadingColumnsToSkip = pexConfig.Field(
183 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
186 overscanNumTrailingColumnsToSkip = pexConfig.Field(
188 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
191 growSaturationFootprintSize = pexConfig.Field(
193 doc=
"Number of pixels by which to grow the saturation footprints",
196 doSaturationInterpolation = pexConfig.Field(
198 doc=
"Perform interpolation over pixels masked as saturated?",
201 doNanInterpAfterFlat = pexConfig.Field(
203 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 204 "also have to interpolate them before flat-fielding."),
207 fluxMag0T1 = pexConfig.Field(
209 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure",
212 keysToRemoveFromAssembledCcd = pexConfig.ListField(
214 doc=
"fields to remove from the metadata of the assembled ccd.",
217 doAssembleIsrExposures = pexConfig.Field(
220 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 222 doAssembleCcd = pexConfig.Field(
225 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 227 expectWcs = pexConfig.Field(
230 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)" 232 doLinearize = pexConfig.Field(
234 doc=
"Correct for nonlinearity of the detector's response?",
237 doCrosstalk = pexConfig.Field(
239 doc=
"Apply intra-CCD crosstalk correction?",
242 crosstalk = pexConfig.ConfigurableField(
243 target=CrosstalkTask,
244 doc=
"Intra-CCD crosstalk correction",
246 doBrighterFatter = pexConfig.Field(
249 doc=
"Apply the brighter fatter correction" 251 brighterFatterLevel = pexConfig.ChoiceField(
252 doc=
"The level at which to correct for brighter-fatter",
253 dtype=str, default=
"DETECTOR",
255 "AMP":
"Every amplifier treated separately",
256 "DETECTOR":
"One kernel per detector",
259 brighterFatterKernelFile = pexConfig.Field(
262 doc=
"Kernel file used for the brighter fatter correction" 264 brighterFatterMaxIter = pexConfig.Field(
267 doc=
"Maximum number of iterations for the brighter fatter correction" 269 brighterFatterThreshold = pexConfig.Field(
272 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 273 " absolute value of the difference between the current corrected image and the one" 274 " from the previous iteration summed over all the pixels." 276 brighterFatterApplyGain = pexConfig.Field(
279 doc=
"Should the gain be applied when applying the brighter fatter correction?" 281 datasetType = pexConfig.Field(
283 doc=
"Dataset type for input data; users will typically leave this alone, " 284 "but camera-specific ISR tasks will override it",
287 fallbackFilterName = pexConfig.Field(dtype=str,
288 doc=
"Fallback default filter name for calibrations", optional=
True)
289 doAttachTransmissionCurve = pexConfig.Field(
292 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 294 doUseOpticsTransmission = pexConfig.Field(
297 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 299 doUseFilterTransmission = pexConfig.Field(
302 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 304 doUseSensorTransmission = pexConfig.Field(
307 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 309 doUseAtmosphereTransmission = pexConfig.Field(
312 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 314 doEmpiricalReadNoise = pexConfig.Field(
317 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 332 @brief Apply common instrument signature correction algorithms to a raw frame. 334 @section ip_isr_isr_Contents Contents 336 - @ref ip_isr_isr_Purpose 337 - @ref ip_isr_isr_Initialize 339 - @ref ip_isr_isr_Config 340 - @ref ip_isr_isr_Debug 343 @section ip_isr_isr_Purpose Description 345 The process for correcting imaging data is very similar from camera to camera. 346 This task provides a vanilla implementation of doing these corrections, including 347 the ability to turn certain corrections off if they are not needed. 348 The inputs to the primary method, run, are a raw exposure to be corrected and the 349 calibration data products. The raw input is a single chip sized mosaic of all amps 350 including overscans and other non-science pixels. 351 The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 352 and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 353 This task may not meet all needs and it is expected that it will be subclassed for 354 specific applications. 356 @section ip_isr_isr_Initialize Task initialization 358 @copydoc \_\_init\_\_ 360 @section ip_isr_isr_IO Inputs/Outputs to the run method 364 @section ip_isr_isr_Config Configuration parameters 366 See @ref IsrTaskConfig 368 @section ip_isr_isr_Debug Debug variables 370 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 371 flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 372 href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 373 Using lsstDebug to control debugging output</a> for more about @b debug.py files. 375 The available variables in IsrTask are: 378 <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 381 <DD> display exposure after ISR has been applied 385 For example, put something like 389 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 390 if name == "lsst.ip.isrFunctions.isrTask": 391 di.display = {'postISRCCD':2} 393 lsstDebug.Info = DebugInfo 395 into your debug.py file and run the commandline task with the @c --debug flag. 399 ConfigClass = IsrTaskConfig
403 '''!Constructor for IsrTask 404 @param[in] *args a list of positional arguments passed on to the Task constructor 405 @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 406 Call the lsst.pipe.base.task.Task.__init__ method 407 Then setup the assembly and fringe correction subtasks 409 pipeBase.Task.__init__(self, *args, **kwargs)
410 self.makeSubtask(
"assembleCcd")
411 self.makeSubtask(
"fringe")
412 self.makeSubtask(
"crosstalk")
415 """!Retrieve necessary frames for instrument signature removal 416 @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 417 of the detector data to be processed 418 @param[in] rawExposure a reference raw exposure that will later be 419 corrected with the retrieved calibration data; 420 should not be modified in this method. 421 @return a pipeBase.Struct with fields containing kwargs expected by run() 422 - bias: exposure of bias frame 423 - dark: exposure of dark frame 424 - flat: exposure of flat field 425 - defects: list of detects 426 - fringeStruct: a pipeBase.Struct with field fringes containing 427 exposure of fringe frame or list of fringe exposure 429 ccd = rawExposure.getDetector()
431 biasExposure = self.
getIsrExposure(dataRef, self.config.biasDataProductName) \
432 if self.config.doBias
else None 434 linearizer = dataRef.get(
"linearizer", immediate=
True)
if self.
doLinearize(ccd)
else None 435 darkExposure = self.
getIsrExposure(dataRef, self.config.darkDataProductName) \
436 if self.config.doDark
else None 437 flatExposure = self.
getIsrExposure(dataRef, self.config.flatDataProductName) \
438 if self.config.doFlat
else None 439 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
if self.config.doBrighterFatter
else None 440 defectList = dataRef.get(
"defects")
if self.config.doDefect
else None 442 if self.config.doCrosstalk:
443 crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
445 crosstalkSources =
None 447 if self.config.doFringe
and self.fringe.checkFilter(rawExposure):
448 fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
449 if self.config.doAssembleIsrExposures
else None)
451 fringeStruct = pipeBase.Struct(fringes=
None)
453 if self.config.doAttachTransmissionCurve:
454 opticsTransmission = (dataRef.get(
"transmission_optics")
455 if self.config.doUseOpticsTransmission
else None)
456 filterTransmission = (dataRef.get(
"transmission_filter")
457 if self.config.doUseFilterTransmission
else None)
458 sensorTransmission = (dataRef.get(
"transmission_sensor")
459 if self.config.doUseSensorTransmission
else None)
460 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
461 if self.config.doUseAtmosphereTransmission
else None)
463 opticsTransmission =
None 464 filterTransmission =
None 465 sensorTransmission =
None 466 atmosphereTransmission =
None 469 return pipeBase.Struct(bias=biasExposure,
470 linearizer=linearizer,
474 fringes=fringeStruct,
475 bfKernel=brighterFatterKernel,
476 opticsTransmission=opticsTransmission,
477 filterTransmission=filterTransmission,
478 sensorTransmission=sensorTransmission,
479 atmosphereTransmission=atmosphereTransmission,
480 crosstalkSources=crosstalkSources,
484 def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
485 fringes=None, bfKernel=None, camera=None,
486 opticsTransmission=None, filterTransmission=None,
487 sensorTransmission=None, atmosphereTransmission=None,
488 crosstalkSources=None):
489 """!Perform instrument signature removal on an exposure 492 - Detect saturation, apply overscan correction, bias, dark and flat 493 - Perform CCD assembly 494 - Interpolate over defects, saturated pixels and all NaNs 496 @param[in] ccdExposure lsst.afw.image.exposure of detector data 497 @param[in] bias exposure of bias frame 498 @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 499 @param[in] dark exposure of dark frame 500 @param[in] flat exposure of flatfield 501 @param[in] defects list of detects 502 @param[in] fringes a pipeBase.Struct with field fringes containing 503 exposure of fringe frame or list of fringe exposure 504 @param[in] bfKernel kernel for brighter-fatter correction, an 505 lsst.cp.pipe.makeBrighterFatterKernel.BrighterFatterKernel object 506 @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 507 used by addDistortionModel 508 @param[in] opticsTransmission a TransmissionCurve for the optics 509 @param[in] filterTransmission a TransmissionCurve for the filter 510 @param[in] sensorTransmission a TransmissionCurve for the sensor 511 @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 512 @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk 514 @return a pipeBase.Struct with field: 518 if isinstance(ccdExposure, ButlerDataRef):
521 ccd = ccdExposure.getDetector()
524 if self.config.doBias
and bias
is None:
525 raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
527 raise RuntimeError(
"Must supply a linearizer if config.doBias True")
528 if self.config.doDark
and dark
is None:
529 raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
530 if self.config.doFlat
and flat
is None:
531 raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
532 if self.config.doBrighterFatter
and bfKernel
is None:
533 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter True")
535 fringes = pipeBase.Struct(fringes=
None)
536 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
537 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
538 if self.config.doDefect
and defects
is None:
539 raise RuntimeError(
"Must supply defects if config.doDefect True")
540 if self.config.doAddDistortionModel
and camera
is None:
541 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel True")
546 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 547 ccd = [
FakeAmp(ccdExposure, self.config)]
552 if ccdExposure.getBBox().contains(amp.getBBox()):
556 overscans.append(overscanResults.overscanImage
if overscanResults
is not None else None)
558 overscans.append(
None)
560 if self.config.doCrosstalk:
561 self.crosstalk.
run(ccdExposure, crosstalkSources)
563 if self.config.doAssembleCcd:
564 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
565 if self.config.expectWcs
and not ccdExposure.getWcs():
566 self.log.warn(
"No WCS found in input exposure")
568 if self.config.doBias:
572 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
574 assert len(ccd) == len(overscans)
575 for amp, overscanImage
in zip(ccd, overscans):
577 if ccdExposure.getBBox().contains(amp.getBBox()):
578 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
581 interpolationDone =
False 583 if self.config.doBrighterFatter:
590 if self.config.doDefect:
592 if self.config.doSaturationInterpolation:
595 interpolationDone =
True 597 if self.config.brighterFatterLevel ==
'DETECTOR':
598 kernelElement = bfKernel.kernel[ccdExposure.getDetector().getId()]
601 raise NotImplementedError(
"per-amplifier brighter-fatter correction not yet implemented")
603 self.config.brighterFatterMaxIter,
604 self.config.brighterFatterThreshold,
605 self.config.brighterFatterApplyGain,
608 if self.config.doDark:
611 if self.config.doFringe
and not self.config.fringeAfterFlat:
612 self.fringe.
run(ccdExposure, **fringes.getDict())
614 if self.config.doFlat:
617 if not interpolationDone:
618 if self.config.doDefect:
620 if self.config.doSaturationInterpolation:
622 if not interpolationDone
or self.config.doNanInterpAfterFlat:
625 if self.config.doFringe
and self.config.fringeAfterFlat:
626 self.fringe.
run(ccdExposure, **fringes.getDict())
628 exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
629 ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
631 if self.config.doAddDistortionModel:
634 if self.config.doAttachTransmissionCurve:
636 filterTransmission=filterTransmission,
637 sensorTransmission=sensorTransmission,
638 atmosphereTransmission=atmosphereTransmission)
640 frame = getDebugFrame(self._display,
"postISRCCD")
642 getDisplay(frame).mtv(ccdExposure)
644 return pipeBase.Struct(
645 exposure=ccdExposure,
650 """Perform instrument signature removal on a ButlerDataRef of a Sensor 652 - Read in necessary detrending/isr/calibration data 653 - Process raw exposure in run() 654 - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 658 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 659 DataRef of the detector data to be processed 663 result : `pipeBase.Struct` 664 Struct contains field "exposure," which is the exposure after application of ISR 666 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
667 ccdExposure = sensorRef.get(
'raw')
668 camera = sensorRef.get(
"camera")
669 if camera
is None and self.config.doAddDistortionModel:
670 raise RuntimeError(
"config.doAddDistortionModel is True " 671 "but could not get a camera from the butler")
674 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
676 if self.config.doWrite:
677 sensorRef.put(result.exposure,
"postISRCCD")
682 """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 684 if isinstance(exposure, afwImage.ExposureF):
687 if not hasattr(exposure,
"convertF"):
688 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
690 newexposure = exposure.convertF()
691 maskedImage = newexposure.getMaskedImage()
692 varArray = maskedImage.getVariance().getArray()
694 maskArray = maskedImage.getMask().getArray()
699 """!Apply bias correction in place 701 @param[in,out] exposure exposure to process 702 @param[in] biasExposure bias exposure of same size as exposure 704 isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
707 """!Apply dark correction in place 709 @param[in,out] exposure exposure to process 710 @param[in] darkExposure dark exposure of same size as exposure 711 @param[in] invert if True, remove the dark from an already-corrected image 713 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
714 if math.isnan(expScale):
715 raise RuntimeError(
"Exposure darktime is NAN")
716 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
717 if math.isnan(darkScale):
718 raise RuntimeError(
"Dark calib darktime is NAN")
719 isrFunctions.darkCorrection(
720 maskedImage=exposure.getMaskedImage(),
721 darkMaskedImage=darkExposure.getMaskedImage(),
728 """!Is linearization wanted for this detector? 730 Checks config.doLinearize and the linearity type of the first amplifier. 732 @param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 734 return self.config.doLinearize
and \
735 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
738 """Set the variance plane using the amplifier gain and read noise 740 The read noise is calculated from the ``overscanImage`` if the 741 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 742 the value from the amplifier data is used. 746 ampExposure : `lsst.afw.image.Exposure` 748 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 749 Amplifier detector data. 750 overscanImage : `lsst.afw.image.MaskedImage`, optional. 751 Image of overscan, required only for empirical read noise. 753 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
755 if not math.isnan(gain):
758 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
759 (amp.getName(), gain, patchedGain))
762 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
763 stats = afwMath.StatisticsControl()
764 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
765 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
766 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
768 readNoise = amp.getReadNoise()
770 isrFunctions.updateVariance(
771 maskedImage=ampExposure.getMaskedImage(),
777 """!Apply flat correction in place 779 @param[in,out] exposure exposure to process 780 @param[in] flatExposure flatfield exposure same size as exposure 781 @param[in] invert if True, unflatten an already-flattened image instead. 783 isrFunctions.flatCorrection(
784 maskedImage=exposure.getMaskedImage(),
785 flatMaskedImage=flatExposure.getMaskedImage(),
786 scalingType=self.config.flatScalingType,
787 userScale=self.config.flatUserScale,
792 """!Retrieve a calibration dataset for removing instrument signature 794 @param[in] dataRef data reference for exposure 795 @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 796 @param[in] immediate if True, disable butler proxies to enable error 797 handling within this routine 801 exp = dataRef.get(datasetType, immediate=immediate)
802 except Exception
as exc1:
803 if not self.config.fallbackFilterName:
804 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
806 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
807 except Exception
as exc2:
808 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
809 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
810 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
812 if self.config.doAssembleIsrExposures:
813 exp = self.assembleCcd.assembleCcd(exp)
817 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 819 @param[in,out] exposure exposure to process; only the amp DataSec is processed 820 @param[in] amp amplifier device data 822 if not math.isnan(amp.getSaturation()):
823 maskedImage = exposure.getMaskedImage()
824 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
825 isrFunctions.makeThresholdMask(
826 maskedImage=dataView,
827 threshold=amp.getSaturation(),
829 maskName=self.config.saturatedMaskName,
833 """!Interpolate over saturated pixels, in place 835 @param[in,out] ccdExposure exposure to process 838 - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 839 - Call this after CCD assembly, since saturated regions may cross amplifier boundaries 841 isrFunctions.interpolateFromMask(
842 maskedImage=ccdExposure.getMaskedImage(),
843 fwhm=self.config.fwhm,
844 growFootprints=self.config.growSaturationFootprintSize,
845 maskName=self.config.saturatedMaskName,
849 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 851 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 852 This is intended to indicate pixels that may be affected by unknown systematics; 853 for example if non-linearity corrections above a certain level are unstable 854 then that would be a useful value for suspectLevel. A value of `nan` indicates 855 that no such level exists and no pixels are to be masked as suspicious. 857 @param[in,out] exposure exposure to process; only the amp DataSec is processed 858 @param[in] amp amplifier device data 860 suspectLevel = amp.getSuspectLevel()
861 if math.isnan(suspectLevel):
864 maskedImage = exposure.getMaskedImage()
865 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
866 isrFunctions.makeThresholdMask(
867 maskedImage=dataView,
868 threshold=suspectLevel,
870 maskName=self.config.suspectMaskName,
874 """!Mask defects using mask plane "BAD" and interpolate over them, in place 876 @param[in,out] ccdExposure exposure to process 877 @param[in] defectBaseList a list of defects to mask and interpolate 879 @warning: call this after CCD assembly, since defects may cross amplifier boundaries 881 maskedImage = ccdExposure.getMaskedImage()
883 for d
in defectBaseList:
885 nd = measAlg.Defect(bbox)
886 defectList.append(nd)
887 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
888 isrFunctions.interpolateDefectList(
889 maskedImage=maskedImage,
890 defectList=defectList,
891 fwhm=self.config.fwhm,
895 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 897 We mask and interpolate over all NaNs, including those 898 that are masked with other bits (because those may or may 899 not be interpolated over later, and we want to remove all 900 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 901 is used to preserve the historical name. 903 @param[in,out] exposure exposure to process 905 maskedImage = exposure.getMaskedImage()
908 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
909 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
910 numNans =
maskNans(maskedImage, maskVal)
911 self.metadata.set(
"NUMNANS", numNans)
915 self.log.warn(
"There were %i unmasked NaNs", numNans)
916 nanDefectList = isrFunctions.getDefectListFromMask(
917 maskedImage=maskedImage,
918 maskName=
'UNMASKEDNAN',
920 isrFunctions.interpolateDefectList(
921 maskedImage=exposure.getMaskedImage(),
922 defectList=nanDefectList,
923 fwhm=self.config.fwhm,
927 """Apply overscan correction, in-place 931 exposure : `lsst.afw.image.Exposure` 932 Exposure to process; must include both data and bias regions. 933 amp : `lsst.afw.table.AmpInfoRecord` 934 Amplifier device data. 938 result : `lsst.pipe.base.Struct` or `NoneType` 939 `None` if there is no overscan; otherwise, this is a 940 result struct with components: 942 - ``imageFit``: Value(s) removed from image (scalar or 943 `lsst.afw.image.Image`). 944 - ``overscanFit``: Value(s) removed from overscan (scalar or 945 `lsst.afw.image.Image`). 946 - ``overscanImage``: Image of the overscan, post-subtraction 947 (`lsst.afw.image.Image`). 949 if not amp.getHasRawInfo():
950 raise RuntimeError(
"This method must be executed on an amp with raw information.")
952 if amp.getRawHorizontalOverscanBBox().isEmpty():
953 self.log.info(
"No Overscan region. Not performing Overscan Correction.")
956 oscanBBox = amp.getRawHorizontalOverscanBBox()
959 x0, x1 = oscanBBox.getBeginX(), oscanBBox.getEndX()
961 prescanBBox = amp.getRawPrescanBBox()
962 if oscanBBox.getBeginX() > prescanBBox.getBeginX():
963 x0 += self.config.overscanNumLeadingColumnsToSkip
964 x1 -= self.config.overscanNumTrailingColumnsToSkip
966 x0 += self.config.overscanNumTrailingColumnsToSkip
967 x1 -= self.config.overscanNumLeadingColumnsToSkip
969 oscanBBox = afwGeom.BoxI(afwGeom.PointI(x0, oscanBBox.getBeginY()),
970 afwGeom.PointI(x1 - 1, oscanBBox.getEndY() - 1))
972 maskedImage = exposure.maskedImage
973 dataView = maskedImage[amp.getRawDataBBox()]
974 overscanImage = maskedImage[oscanBBox]
976 results = isrFunctions.overscanCorrection(
977 ampMaskedImage=dataView,
978 overscanImage=overscanImage,
979 fitType=self.config.overscanFitType,
980 order=self.config.overscanOrder,
981 collapseRej=self.config.overscanRej,
983 results.overscanImage = overscanImage
987 """!Update the WCS in exposure with a distortion model based on camera geometry 989 Add a model for optical distortion based on geometry found in `camera` 990 and the `exposure`'s detector. The raw input exposure is assumed 991 have a TAN WCS that has no compensation for optical distortion. 992 Two other possibilities are: 993 - The raw input exposure already has a model for optical distortion, 994 as is the case for raw DECam data. 995 In that case you should set config.doAddDistortionModel False. 996 - The raw input exposure has a model for distortion, but it has known 997 deficiencies severe enough to be worth fixing (e.g. because they 998 cause problems for fitting a better WCS). In that case you should 999 override this method with a version suitable for your raw data. 1001 @param[in,out] exposure exposure to process; must include a Detector and a WCS; 1002 the WCS of the exposure is modified in place 1003 @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 1005 self.log.info(
"Adding a distortion model to the WCS")
1006 wcs = exposure.getWcs()
1008 raise RuntimeError(
"exposure has no WCS")
1010 raise RuntimeError(
"camera is None")
1011 detector = exposure.getDetector()
1012 if detector
is None:
1013 raise RuntimeError(
"exposure has no Detector")
1014 pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
1015 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
1016 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
1017 exposure.setWcs(distortedWcs)
1020 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners 1022 @param[in,out] ccdExposure exposure to process 1023 @param[in] fpPolygon Polygon in focal plane coordinates 1026 ccd = ccdExposure.getDetector()
1027 fpCorners = ccd.getCorners(FOCAL_PLANE)
1028 ccdPolygon = Polygon(fpCorners)
1031 intersect = ccdPolygon.intersectionSingle(fpPolygon)
1034 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
1035 validPolygon = Polygon(ccdPoints)
1036 ccdExposure.getInfo().setValidPolygon(validPolygon)
1039 """Apply brighter fatter correction in place for the image 1041 This correction takes a kernel that has been derived from flat field images to 1042 redistribute the charge. The gradient of the kernel is the deflection 1043 field due to the accumulated charge. 1045 Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 1046 using the following equation: 1048 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 1050 To evaluate the derivative term we expand it as follows: 1052 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 1054 Because we use the measured counts instead of the incident counts we apply the correction 1055 iteratively to reconstruct the original counts and the correction. We stop iterating when the 1056 summed difference between the current corrected image and the one from the previous iteration 1057 is below the threshold. We do not require convergence because the number of iterations is 1058 too large a computational cost. How we define the threshold still needs to be evaluated, the 1059 current default was shown to work reasonably well on a small set of images. For more information 1060 on the method see DocuShare Document-19407. 1062 The edges as defined by the kernel are not corrected because they have spurious values 1063 due to the convolution. 1065 self.log.info(
"Applying brighter fatter correction")
1067 image = exposure.getMaskedImage().getImage()
1070 with self.
gainContext(exposure, image, applyGain):
1072 kLx = numpy.shape(kernel)[0]
1073 kLy = numpy.shape(kernel)[1]
1074 kernelImage = afwImage.ImageD(kLx, kLy)
1075 kernelImage.getArray()[:, :] = kernel
1076 tempImage = image.clone()
1078 nanIndex = numpy.isnan(tempImage.getArray())
1079 tempImage.getArray()[nanIndex] = 0.
1081 outImage = afwImage.ImageF(image.getDimensions())
1082 corr = numpy.zeros_like(image.getArray())
1083 prev_image = numpy.zeros_like(image.getArray())
1084 convCntrl = afwMath.ConvolutionControl(
False,
True, 1)
1085 fixedKernel = afwMath.FixedKernel(kernelImage)
1095 for iteration
in range(maxIter):
1097 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1098 tmpArray = tempImage.getArray()
1099 outArray = outImage.getArray()
1101 with numpy.errstate(invalid=
"ignore", over=
"ignore"):
1103 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1104 gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1105 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1108 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1109 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1110 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1112 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1114 tmpArray[:, :] = image.getArray()[:, :]
1115 tmpArray[nanIndex] = 0.
1116 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1119 diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1121 if diff < threshold:
1123 prev_image[:, :] = tmpArray[:, :]
1125 if iteration == maxIter - 1:
1126 self.log.warn(
"Brighter fatter correction did not converge, final difference %f" % diff)
1128 self.log.info(
"Finished brighter fatter in %d iterations" % (iteration + 1))
1129 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1130 corr[startY + 1:endY - 1, startX + 1:endX - 1]
1133 sensorTransmission=None, atmosphereTransmission=None):
1134 """Attach a TransmissionCurve to an Exposure, given separate curves for 1135 different components. 1139 exposure : `lsst.afw.image.Exposure` 1140 Exposure object to modify by attaching the product of all given 1141 ``TransmissionCurves`` in post-assembly trimmed detector 1142 coordinates. Must have a valid ``Detector`` attached that matches 1143 the detector associated with sensorTransmission. 1144 opticsTransmission : `lsst.afw.image.TransmissionCurve` 1145 A ``TransmissionCurve`` that represents the throughput of the 1146 optics, to be evaluated in focal-plane coordinates. 1147 filterTransmission : `lsst.afw.image.TransmissionCurve` 1148 A ``TransmissionCurve`` that represents the throughput of the 1149 filter itself, to be evaluated in focal-plane coordinates. 1150 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1151 A ``TransmissionCurve`` that represents the throughput of the 1152 sensor itself, to be evaluated in post-assembly trimmed detector 1154 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1155 A ``TransmissionCurve`` that represents the throughput of the 1156 atmosphere, assumed to be spatially constant. 1158 All ``TransmissionCurve`` arguments are optional; if none are provided, 1159 the attached ``TransmissionCurve`` will have unit transmission 1164 combined : ``lsst.afw.image.TransmissionCurve`` 1165 The TransmissionCurve attached to the exposure. 1167 return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1168 filterTransmission=filterTransmission,
1169 sensorTransmission=sensorTransmission,
1170 atmosphereTransmission=atmosphereTransmission)
1174 """Context manager that applies and removes gain 1177 ccd = exp.getDetector()
1179 sim = image.Factory(image, amp.getBBox())
1180 sim *= amp.getGain()
1186 ccd = exp.getDetector()
1188 sim = image.Factory(image, amp.getBBox())
1189 sim /= amp.getGain()
1193 """Context manager that applies and removes flats and darks, 1194 if the task is configured to apply them. 1196 if self.config.doDark
and dark
is not None:
1198 if self.config.doFlat:
1203 if self.config.doFlat:
1205 if self.config.doDark
and dark
is not None:
1210 """A Detector-like object that supports returning gain and saturation level""" 1213 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1215 self.
_gain = config.gain
1245 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
1256 """Task to wrap the default IsrTask to allow it to be retargeted. 1258 The standard IsrTask can be called directly from a command line 1259 program, but doing so removes the ability of the task to be 1260 retargeted. As most cameras override some set of the IsrTask 1261 methods, this would remove those data-specific methods in the 1262 output post-ISR images. This wrapping class fixes the issue, 1263 allowing identical post-ISR images to be generated by both the 1264 processCcd and isrTask code. 1266 ConfigClass = RunIsrConfig
1267 _DefaultName =
"runIsr" 1271 self.makeSubtask(
"isr")
1277 dataRef : `lsst.daf.persistence.ButlerDataRef` 1278 data reference of the detector data to be processed 1282 result : `pipeBase.Struct` 1283 Result struct with component: 1285 - exposure : `lsst.afw.image.Exposure` 1286 Post-ISR processed exposure.
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
def runDataRef(self, sensorRef)
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
def gainContext(self, exp, image, apply)
def __init__(self, args, kwargs)
Constructor for IsrTask.
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def runDataRef(self, dataRef)
def __init__(self, args, kwargs)
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Apply common instrument signature correction algorithms to a raw frame.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Is linearization wanted for this detector?
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
def flatContext(self, exp, flat, dark=None)
def overscanCorrection(self, exposure, amp)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)