32 from lsstDebug
import getDebugFrame
34 from .
import isrFunctions
35 from .assembleCcdTask
import AssembleCcdTask
36 from .fringe
import FringeTask
40 from contextlib
import contextmanager
41 from .isr
import maskNans
42 from .crosstalk
import CrosstalkTask
46 doBias = pexConfig.Field(
48 doc=
"Apply bias frame correction?",
51 doDark = pexConfig.Field(
53 doc=
"Apply dark frame correction?",
56 doFlat = pexConfig.Field(
58 doc=
"Apply flat field correction?",
61 doFringe = pexConfig.Field(
63 doc=
"Apply fringe correction?",
66 doDefect = pexConfig.Field(
68 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
71 doAddDistortionModel = pexConfig.Field(
73 doc=
"Apply a distortion model based on camera geometry to the WCS?",
76 doWrite = pexConfig.Field(
78 doc=
"Persist postISRCCD?",
81 biasDataProductName = pexConfig.Field(
83 doc=
"Name of the bias data product",
86 darkDataProductName = pexConfig.Field(
88 doc=
"Name of the dark data product",
91 flatDataProductName = pexConfig.Field(
93 doc=
"Name of the flat data product",
96 assembleCcd = pexConfig.ConfigurableField(
97 target=AssembleCcdTask,
98 doc=
"CCD assembly task",
100 gain = pexConfig.Field(
102 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
103 default=float(
"NaN"),
105 readNoise = pexConfig.Field(
107 doc=
"The read noise to use if no Detector is present in the Exposure",
110 saturation = pexConfig.Field(
112 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
113 default=float(
"NaN"),
115 fringeAfterFlat = pexConfig.Field(
117 doc=
"Do fringe subtraction after flat-fielding?",
120 fringe = pexConfig.ConfigurableField(
122 doc=
"Fringe subtraction task",
124 fwhm = pexConfig.Field(
126 doc=
"FWHM of PSF (arcsec)",
129 saturatedMaskName = pexConfig.Field(
131 doc=
"Name of mask plane to use in saturation detection and interpolation",
134 suspectMaskName = pexConfig.Field(
136 doc=
"Name of mask plane to use for suspect pixels",
139 flatScalingType = pexConfig.ChoiceField(
141 doc=
"The method for scaling the flat on the fly.",
144 "USER":
"Scale by flatUserScale",
145 "MEAN":
"Scale by the inverse of the mean",
146 "MEDIAN":
"Scale by the inverse of the median",
149 flatUserScale = pexConfig.Field(
151 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
154 overscanFitType = pexConfig.ChoiceField(
156 doc=
"The method for fitting the overscan bias level.",
159 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
160 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
161 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
162 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
163 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
164 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
165 "MEAN":
"Correct using the mean of the overscan region",
166 "MEDIAN":
"Correct using the median of the overscan region",
169 overscanOrder = pexConfig.Field(
171 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
172 "or number of spline knots if overscan fit type is a spline."),
175 overscanRej = pexConfig.Field(
177 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
181 overscanNumLeadingColumnsToSkip = pexConfig.Field(
183 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
186 overscanNumTrailingColumnsToSkip = pexConfig.Field(
188 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
191 growSaturationFootprintSize = pexConfig.Field(
193 doc=
"Number of pixels by which to grow the saturation footprints",
196 doSaturationInterpolation = pexConfig.Field(
198 doc=
"Perform interpolation over pixels masked as saturated?",
201 doNanInterpAfterFlat = pexConfig.Field(
203 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 204 "also have to interpolate them before flat-fielding."),
207 fluxMag0T1 = pexConfig.Field(
209 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure",
212 keysToRemoveFromAssembledCcd = pexConfig.ListField(
214 doc=
"fields to remove from the metadata of the assembled ccd.",
217 doAssembleIsrExposures = pexConfig.Field(
220 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 222 doAssembleCcd = pexConfig.Field(
225 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 227 expectWcs = pexConfig.Field(
230 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)" 232 doLinearize = pexConfig.Field(
234 doc=
"Correct for nonlinearity of the detector's response?",
237 doCrosstalk = pexConfig.Field(
239 doc=
"Apply intra-CCD crosstalk correction?",
242 crosstalk = pexConfig.ConfigurableField(
243 target=CrosstalkTask,
244 doc=
"Intra-CCD crosstalk correction",
246 doBrighterFatter = pexConfig.Field(
249 doc=
"Apply the brighter fatter correction" 251 brighterFatterKernelFile = pexConfig.Field(
254 doc=
"Kernel file used for the brighter fatter correction" 256 brighterFatterMaxIter = pexConfig.Field(
259 doc=
"Maximum number of iterations for the brighter fatter correction" 261 brighterFatterThreshold = pexConfig.Field(
264 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 265 " absolute value of the difference between the current corrected image and the one" 266 " from the previous iteration summed over all the pixels." 268 brighterFatterApplyGain = pexConfig.Field(
271 doc=
"Should the gain be applied when applying the brighter fatter correction?" 273 datasetType = pexConfig.Field(
275 doc=
"Dataset type for input data; users will typically leave this alone, " 276 "but camera-specific ISR tasks will override it",
279 fallbackFilterName = pexConfig.Field(dtype=str,
280 doc=
"Fallback default filter name for calibrations", optional=
True)
281 doAttachTransmissionCurve = pexConfig.Field(
284 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 286 doUseOpticsTransmission = pexConfig.Field(
289 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 291 doUseFilterTransmission = pexConfig.Field(
294 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 296 doUseSensorTransmission = pexConfig.Field(
299 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 301 doUseAtmosphereTransmission = pexConfig.Field(
304 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 306 doEmpiricalReadNoise = pexConfig.Field(
309 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 324 @brief Apply common instrument signature correction algorithms to a raw frame. 326 @section ip_isr_isr_Contents Contents 328 - @ref ip_isr_isr_Purpose 329 - @ref ip_isr_isr_Initialize 331 - @ref ip_isr_isr_Config 332 - @ref ip_isr_isr_Debug 335 @section ip_isr_isr_Purpose Description 337 The process for correcting imaging data is very similar from camera to camera. 338 This task provides a vanilla implementation of doing these corrections, including 339 the ability to turn certain corrections off if they are not needed. 340 The inputs to the primary method, run, are a raw exposure to be corrected and the 341 calibration data products. The raw input is a single chip sized mosaic of all amps 342 including overscans and other non-science pixels. 343 The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 344 and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 345 This task may not meet all needs and it is expected that it will be subclassed for 346 specific applications. 348 @section ip_isr_isr_Initialize Task initialization 350 @copydoc \_\_init\_\_ 352 @section ip_isr_isr_IO Inputs/Outputs to the run method 356 @section ip_isr_isr_Config Configuration parameters 358 See @ref IsrTaskConfig 360 @section ip_isr_isr_Debug Debug variables 362 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 363 flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 364 href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 365 Using lsstDebug to control debugging output</a> for more about @b debug.py files. 367 The available variables in IsrTask are: 370 <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 373 <DD> display exposure after ISR has been applied 377 For example, put something like 381 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 382 if name == "lsst.ip.isrFunctions.isrTask": 383 di.display = {'postISRCCD':2} 385 lsstDebug.Info = DebugInfo 387 into your debug.py file and run the commandline task with the @c --debug flag. 391 ConfigClass = IsrTaskConfig
395 '''!Constructor for IsrTask 396 @param[in] *args a list of positional arguments passed on to the Task constructor 397 @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 398 Call the lsst.pipe.base.task.Task.__init__ method 399 Then setup the assembly and fringe correction subtasks 401 pipeBase.Task.__init__(self, *args, **kwargs)
402 self.makeSubtask(
"assembleCcd")
403 self.makeSubtask(
"fringe")
404 self.makeSubtask(
"crosstalk")
407 """!Retrieve necessary frames for instrument signature removal 408 @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 409 of the detector data to be processed 410 @param[in] rawExposure a reference raw exposure that will later be 411 corrected with the retrieved calibration data; 412 should not be modified in this method. 413 @return a pipeBase.Struct with fields containing kwargs expected by run() 414 - bias: exposure of bias frame 415 - dark: exposure of dark frame 416 - flat: exposure of flat field 417 - defects: list of detects 418 - fringeStruct: a pipeBase.Struct with field fringes containing 419 exposure of fringe frame or list of fringe exposure 421 ccd = rawExposure.getDetector()
423 biasExposure = self.
getIsrExposure(dataRef, self.config.biasDataProductName) \
424 if self.config.doBias
else None 426 linearizer = dataRef.get(
"linearizer", immediate=
True)
if self.
doLinearize(ccd)
else None 427 darkExposure = self.
getIsrExposure(dataRef, self.config.darkDataProductName) \
428 if self.config.doDark
else None 429 flatExposure = self.
getIsrExposure(dataRef, self.config.flatDataProductName) \
430 if self.config.doFlat
else None 431 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
if self.config.doBrighterFatter
else None 432 defectList = dataRef.get(
"defects")
if self.config.doDefect
else None 434 if self.config.doCrosstalk:
435 crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
437 crosstalkSources =
None 439 if self.config.doFringe
and self.fringe.checkFilter(rawExposure):
440 fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
441 if self.config.doAssembleIsrExposures
else None)
443 fringeStruct = pipeBase.Struct(fringes=
None)
445 if self.config.doAttachTransmissionCurve:
446 opticsTransmission = (dataRef.get(
"transmission_optics")
447 if self.config.doUseOpticsTransmission
else None)
448 filterTransmission = (dataRef.get(
"transmission_filter")
449 if self.config.doUseFilterTransmission
else None)
450 sensorTransmission = (dataRef.get(
"transmission_sensor")
451 if self.config.doUseSensorTransmission
else None)
452 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
453 if self.config.doUseAtmosphereTransmission
else None)
455 opticsTransmission =
None 456 filterTransmission =
None 457 sensorTransmission =
None 458 atmosphereTransmission =
None 461 return pipeBase.Struct(bias=biasExposure,
462 linearizer=linearizer,
466 fringes=fringeStruct,
467 bfKernel=brighterFatterKernel,
468 opticsTransmission=opticsTransmission,
469 filterTransmission=filterTransmission,
470 sensorTransmission=sensorTransmission,
471 atmosphereTransmission=atmosphereTransmission,
472 crosstalkSources=crosstalkSources,
476 def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
477 fringes=None, bfKernel=None, camera=None,
478 opticsTransmission=None, filterTransmission=None,
479 sensorTransmission=None, atmosphereTransmission=None,
480 crosstalkSources=None):
481 """!Perform instrument signature removal on an exposure 484 - Detect saturation, apply overscan correction, bias, dark and flat 485 - Perform CCD assembly 486 - Interpolate over defects, saturated pixels and all NaNs 488 @param[in] ccdExposure lsst.afw.image.exposure of detector data 489 @param[in] bias exposure of bias frame 490 @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 491 @param[in] dark exposure of dark frame 492 @param[in] flat exposure of flatfield 493 @param[in] defects list of detects 494 @param[in] fringes a pipeBase.Struct with field fringes containing 495 exposure of fringe frame or list of fringe exposure 496 @param[in] bfKernel kernel for brighter-fatter correction 497 @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 498 used by addDistortionModel 499 @param[in] opticsTransmission a TransmissionCurve for the optics 500 @param[in] filterTransmission a TransmissionCurve for the filter 501 @param[in] sensorTransmission a TransmissionCurve for the sensor 502 @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 503 @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk 505 @return a pipeBase.Struct with field: 509 if isinstance(ccdExposure, ButlerDataRef):
512 ccd = ccdExposure.getDetector()
515 if self.config.doBias
and bias
is None:
516 raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
518 raise RuntimeError(
"Must supply a linearizer if config.doBias True")
519 if self.config.doDark
and dark
is None:
520 raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
521 if self.config.doFlat
and flat
is None:
522 raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
523 if self.config.doBrighterFatter
and bfKernel
is None:
524 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter True")
526 fringes = pipeBase.Struct(fringes=
None)
527 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
528 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
529 if self.config.doDefect
and defects
is None:
530 raise RuntimeError(
"Must supply defects if config.doDefect True")
531 if self.config.doAddDistortionModel
and camera
is None:
532 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel True")
537 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 538 ccd = [
FakeAmp(ccdExposure, self.config)]
543 if ccdExposure.getBBox().contains(amp.getBBox()):
547 overscans.append(overscanResults.overscanImage
if overscanResults
is not None else None)
549 overscans.append(
None)
551 if self.config.doCrosstalk:
552 self.crosstalk.
run(ccdExposure, crosstalkSources)
554 if self.config.doAssembleCcd:
555 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
556 if self.config.expectWcs
and not ccdExposure.getWcs():
557 self.log.warn(
"No WCS found in input exposure")
559 if self.config.doBias:
563 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
565 assert len(ccd) == len(overscans)
566 for amp, overscanImage
in zip(ccd, overscans):
568 if ccdExposure.getBBox().contains(amp.getBBox()):
569 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
572 interpolationDone =
False 574 if self.config.doBrighterFatter:
581 if self.config.doDefect:
583 if self.config.doSaturationInterpolation:
586 interpolationDone =
True 589 self.config.brighterFatterMaxIter,
590 self.config.brighterFatterThreshold,
591 self.config.brighterFatterApplyGain,
594 if self.config.doDark:
597 if self.config.doFringe
and not self.config.fringeAfterFlat:
598 self.fringe.
run(ccdExposure, **fringes.getDict())
600 if self.config.doFlat:
603 if not interpolationDone:
604 if self.config.doDefect:
606 if self.config.doSaturationInterpolation:
608 if not interpolationDone
or self.config.doNanInterpAfterFlat:
611 if self.config.doFringe
and self.config.fringeAfterFlat:
612 self.fringe.
run(ccdExposure, **fringes.getDict())
614 exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
615 ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
617 if self.config.doAddDistortionModel:
620 if self.config.doAttachTransmissionCurve:
622 filterTransmission=filterTransmission,
623 sensorTransmission=sensorTransmission,
624 atmosphereTransmission=atmosphereTransmission)
626 frame = getDebugFrame(self._display,
"postISRCCD")
628 getDisplay(frame).mtv(ccdExposure)
630 return pipeBase.Struct(
631 exposure=ccdExposure,
636 """Perform instrument signature removal on a ButlerDataRef of a Sensor 638 - Read in necessary detrending/isr/calibration data 639 - Process raw exposure in run() 640 - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 644 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 645 DataRef of the detector data to be processed 649 result : `pipeBase.Struct` 650 Struct contains field "exposure," which is the exposure after application of ISR 652 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
653 ccdExposure = sensorRef.get(
'raw')
654 camera = sensorRef.get(
"camera")
655 if camera
is None and self.config.doAddDistortionModel:
656 raise RuntimeError(
"config.doAddDistortionModel is True " 657 "but could not get a camera from the butler")
660 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
662 if self.config.doWrite:
663 sensorRef.put(result.exposure,
"postISRCCD")
668 """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 670 if isinstance(exposure, afwImage.ExposureF):
673 if not hasattr(exposure,
"convertF"):
674 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
676 newexposure = exposure.convertF()
677 maskedImage = newexposure.getMaskedImage()
678 varArray = maskedImage.getVariance().getArray()
680 maskArray = maskedImage.getMask().getArray()
685 """!Apply bias correction in place 687 @param[in,out] exposure exposure to process 688 @param[in] biasExposure bias exposure of same size as exposure 690 isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
693 """!Apply dark correction in place 695 @param[in,out] exposure exposure to process 696 @param[in] darkExposure dark exposure of same size as exposure 697 @param[in] invert if True, remove the dark from an already-corrected image 699 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
700 if math.isnan(expScale):
701 raise RuntimeError(
"Exposure darktime is NAN")
702 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
703 if math.isnan(darkScale):
704 raise RuntimeError(
"Dark calib darktime is NAN")
705 isrFunctions.darkCorrection(
706 maskedImage=exposure.getMaskedImage(),
707 darkMaskedImage=darkExposure.getMaskedImage(),
714 """!Is linearization wanted for this detector? 716 Checks config.doLinearize and the linearity type of the first amplifier. 718 @param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 720 return self.config.doLinearize
and \
721 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
724 """Set the variance plane using the amplifier gain and read noise 726 The read noise is calculated from the ``overscanImage`` if the 727 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 728 the value from the amplifier data is used. 732 ampExposure : `lsst.afw.image.Exposure` 734 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 735 Amplifier detector data. 736 overscanImage : `lsst.afw.image.MaskedImage`, optional. 737 Image of overscan, required only for empirical read noise. 739 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
741 if not math.isnan(gain):
744 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
745 (amp.getName(), gain, patchedGain))
748 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
749 stats = afwMath.StatisticsControl()
750 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
751 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
752 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
754 readNoise = amp.getReadNoise()
756 isrFunctions.updateVariance(
757 maskedImage=ampExposure.getMaskedImage(),
763 """!Apply flat correction in place 765 @param[in,out] exposure exposure to process 766 @param[in] flatExposure flatfield exposure same size as exposure 767 @param[in] invert if True, unflatten an already-flattened image instead. 769 isrFunctions.flatCorrection(
770 maskedImage=exposure.getMaskedImage(),
771 flatMaskedImage=flatExposure.getMaskedImage(),
772 scalingType=self.config.flatScalingType,
773 userScale=self.config.flatUserScale,
778 """!Retrieve a calibration dataset for removing instrument signature 780 @param[in] dataRef data reference for exposure 781 @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 782 @param[in] immediate if True, disable butler proxies to enable error 783 handling within this routine 787 exp = dataRef.get(datasetType, immediate=immediate)
788 except Exception
as exc1:
789 if not self.config.fallbackFilterName:
790 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
792 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
793 except Exception
as exc2:
794 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
795 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
796 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
798 if self.config.doAssembleIsrExposures:
799 exp = self.assembleCcd.assembleCcd(exp)
803 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 805 @param[in,out] exposure exposure to process; only the amp DataSec is processed 806 @param[in] amp amplifier device data 808 if not math.isnan(amp.getSaturation()):
809 maskedImage = exposure.getMaskedImage()
810 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
811 isrFunctions.makeThresholdMask(
812 maskedImage=dataView,
813 threshold=amp.getSaturation(),
815 maskName=self.config.saturatedMaskName,
819 """!Interpolate over saturated pixels, in place 821 @param[in,out] ccdExposure exposure to process 824 - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 825 - Call this after CCD assembly, since saturated regions may cross amplifier boundaries 827 isrFunctions.interpolateFromMask(
828 maskedImage=ccdExposure.getMaskedImage(),
829 fwhm=self.config.fwhm,
830 growFootprints=self.config.growSaturationFootprintSize,
831 maskName=self.config.saturatedMaskName,
835 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 837 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 838 This is intended to indicate pixels that may be affected by unknown systematics; 839 for example if non-linearity corrections above a certain level are unstable 840 then that would be a useful value for suspectLevel. A value of `nan` indicates 841 that no such level exists and no pixels are to be masked as suspicious. 843 @param[in,out] exposure exposure to process; only the amp DataSec is processed 844 @param[in] amp amplifier device data 846 suspectLevel = amp.getSuspectLevel()
847 if math.isnan(suspectLevel):
850 maskedImage = exposure.getMaskedImage()
851 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
852 isrFunctions.makeThresholdMask(
853 maskedImage=dataView,
854 threshold=suspectLevel,
856 maskName=self.config.suspectMaskName,
860 """!Mask defects using mask plane "BAD" and interpolate over them, in place 862 @param[in,out] ccdExposure exposure to process 863 @param[in] defectBaseList a list of defects to mask and interpolate 865 @warning: call this after CCD assembly, since defects may cross amplifier boundaries 867 maskedImage = ccdExposure.getMaskedImage()
869 for d
in defectBaseList:
871 nd = measAlg.Defect(bbox)
872 defectList.append(nd)
873 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
874 isrFunctions.interpolateDefectList(
875 maskedImage=maskedImage,
876 defectList=defectList,
877 fwhm=self.config.fwhm,
881 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 883 We mask and interpolate over all NaNs, including those 884 that are masked with other bits (because those may or may 885 not be interpolated over later, and we want to remove all 886 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 887 is used to preserve the historical name. 889 @param[in,out] exposure exposure to process 891 maskedImage = exposure.getMaskedImage()
894 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
895 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
896 numNans =
maskNans(maskedImage, maskVal)
897 self.metadata.set(
"NUMNANS", numNans)
901 self.log.warn(
"There were %i unmasked NaNs", numNans)
902 nanDefectList = isrFunctions.getDefectListFromMask(
903 maskedImage=maskedImage,
904 maskName=
'UNMASKEDNAN',
906 isrFunctions.interpolateDefectList(
907 maskedImage=exposure.getMaskedImage(),
908 defectList=nanDefectList,
909 fwhm=self.config.fwhm,
913 """Apply overscan correction, in-place 917 exposure : `lsst.afw.image.Exposure` 918 Exposure to process; must include both data and bias regions. 919 amp : `lsst.afw.table.AmpInfoRecord` 920 Amplifier device data. 924 result : `lsst.pipe.base.Struct` or `NoneType` 925 `None` if there is no overscan; otherwise, this is a 926 result struct with components: 928 - ``imageFit``: Value(s) removed from image (scalar or 929 `lsst.afw.image.Image`). 930 - ``overscanFit``: Value(s) removed from overscan (scalar or 931 `lsst.afw.image.Image`). 932 - ``overscanImage``: Image of the overscan, post-subtraction 933 (`lsst.afw.image.Image`). 935 if not amp.getHasRawInfo():
936 raise RuntimeError(
"This method must be executed on an amp with raw information.")
938 if amp.getRawHorizontalOverscanBBox().isEmpty():
939 self.log.info(
"No Overscan region. Not performing Overscan Correction.")
942 oscanBBox = amp.getRawHorizontalOverscanBBox()
945 x0, x1 = oscanBBox.getBeginX(), oscanBBox.getEndX()
947 prescanBBox = amp.getRawPrescanBBox()
948 if oscanBBox.getBeginX() > prescanBBox.getBeginX():
949 x0 += self.config.overscanNumLeadingColumnsToSkip
950 x1 -= self.config.overscanNumTrailingColumnsToSkip
952 x0 += self.config.overscanNumTrailingColumnsToSkip
953 x1 -= self.config.overscanNumLeadingColumnsToSkip
955 oscanBBox = afwGeom.BoxI(afwGeom.PointI(x0, oscanBBox.getBeginY()),
956 afwGeom.PointI(x1 - 1, oscanBBox.getEndY() - 1))
958 maskedImage = exposure.maskedImage
959 dataView = maskedImage[amp.getRawDataBBox()]
960 overscanImage = maskedImage[oscanBBox]
962 results = isrFunctions.overscanCorrection(
963 ampMaskedImage=dataView,
964 overscanImage=overscanImage,
965 fitType=self.config.overscanFitType,
966 order=self.config.overscanOrder,
967 collapseRej=self.config.overscanRej,
969 results.overscanImage = overscanImage
973 """!Update the WCS in exposure with a distortion model based on camera geometry 975 Add a model for optical distortion based on geometry found in `camera` 976 and the `exposure`'s detector. The raw input exposure is assumed 977 have a TAN WCS that has no compensation for optical distortion. 978 Two other possibilities are: 979 - The raw input exposure already has a model for optical distortion, 980 as is the case for raw DECam data. 981 In that case you should set config.doAddDistortionModel False. 982 - The raw input exposure has a model for distortion, but it has known 983 deficiencies severe enough to be worth fixing (e.g. because they 984 cause problems for fitting a better WCS). In that case you should 985 override this method with a version suitable for your raw data. 987 @param[in,out] exposure exposure to process; must include a Detector and a WCS; 988 the WCS of the exposure is modified in place 989 @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 991 self.log.info(
"Adding a distortion model to the WCS")
992 wcs = exposure.getWcs()
994 raise RuntimeError(
"exposure has no WCS")
996 raise RuntimeError(
"camera is None")
997 detector = exposure.getDetector()
999 raise RuntimeError(
"exposure has no Detector")
1000 pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
1001 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
1002 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
1003 exposure.setWcs(distortedWcs)
1006 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners 1008 @param[in,out] ccdExposure exposure to process 1009 @param[in] fpPolygon Polygon in focal plane coordinates 1012 ccd = ccdExposure.getDetector()
1013 fpCorners = ccd.getCorners(FOCAL_PLANE)
1014 ccdPolygon = Polygon(fpCorners)
1017 intersect = ccdPolygon.intersectionSingle(fpPolygon)
1020 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
1021 validPolygon = Polygon(ccdPoints)
1022 ccdExposure.getInfo().setValidPolygon(validPolygon)
1025 """Apply brighter fatter correction in place for the image 1027 This correction takes a kernel that has been derived from flat field images to 1028 redistribute the charge. The gradient of the kernel is the deflection 1029 field due to the accumulated charge. 1031 Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 1032 using the following equation: 1034 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 1036 To evaluate the derivative term we expand it as follows: 1038 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 1040 Because we use the measured counts instead of the incident counts we apply the correction 1041 iteratively to reconstruct the original counts and the correction. We stop iterating when the 1042 summed difference between the current corrected image and the one from the previous iteration 1043 is below the threshold. We do not require convergence because the number of iterations is 1044 too large a computational cost. How we define the threshold still needs to be evaluated, the 1045 current default was shown to work reasonably well on a small set of images. For more information 1046 on the method see DocuShare Document-19407. 1048 The edges as defined by the kernel are not corrected because they have spurious values 1049 due to the convolution. 1051 self.log.info(
"Applying brighter fatter correction")
1053 image = exposure.getMaskedImage().getImage()
1056 with self.
gainContext(exposure, image, applyGain):
1058 kLx = numpy.shape(kernel)[0]
1059 kLy = numpy.shape(kernel)[1]
1060 kernelImage = afwImage.ImageD(kLx, kLy)
1061 kernelImage.getArray()[:, :] = kernel
1062 tempImage = image.clone()
1064 nanIndex = numpy.isnan(tempImage.getArray())
1065 tempImage.getArray()[nanIndex] = 0.
1067 outImage = afwImage.ImageF(image.getDimensions())
1068 corr = numpy.zeros_like(image.getArray())
1069 prev_image = numpy.zeros_like(image.getArray())
1070 convCntrl = afwMath.ConvolutionControl(
False,
True, 1)
1071 fixedKernel = afwMath.FixedKernel(kernelImage)
1081 for iteration
in range(maxIter):
1083 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1084 tmpArray = tempImage.getArray()
1085 outArray = outImage.getArray()
1087 with numpy.errstate(invalid=
"ignore", over=
"ignore"):
1089 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1090 gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1091 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1094 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1095 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1096 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1098 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1100 tmpArray[:, :] = image.getArray()[:, :]
1101 tmpArray[nanIndex] = 0.
1102 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1105 diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1107 if diff < threshold:
1109 prev_image[:, :] = tmpArray[:, :]
1111 if iteration == maxIter - 1:
1112 self.log.warn(
"Brighter fatter correction did not converge, final difference %f" % diff)
1114 self.log.info(
"Finished brighter fatter in %d iterations" % (iteration + 1))
1115 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1116 corr[startY + 1:endY - 1, startX + 1:endX - 1]
1119 sensorTransmission=None, atmosphereTransmission=None):
1120 """Attach a TransmissionCurve to an Exposure, given separate curves for 1121 different components. 1125 exposure : `lsst.afw.image.Exposure` 1126 Exposure object to modify by attaching the product of all given 1127 ``TransmissionCurves`` in post-assembly trimmed detector 1128 coordinates. Must have a valid ``Detector`` attached that matches 1129 the detector associated with sensorTransmission. 1130 opticsTransmission : `lsst.afw.image.TransmissionCurve` 1131 A ``TransmissionCurve`` that represents the throughput of the 1132 optics, to be evaluated in focal-plane coordinates. 1133 filterTransmission : `lsst.afw.image.TransmissionCurve` 1134 A ``TransmissionCurve`` that represents the throughput of the 1135 filter itself, to be evaluated in focal-plane coordinates. 1136 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1137 A ``TransmissionCurve`` that represents the throughput of the 1138 sensor itself, to be evaluated in post-assembly trimmed detector 1140 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1141 A ``TransmissionCurve`` that represents the throughput of the 1142 atmosphere, assumed to be spatially constant. 1144 All ``TransmissionCurve`` arguments are optional; if none are provided, 1145 the attached ``TransmissionCurve`` will have unit transmission 1150 combined : ``lsst.afw.image.TransmissionCurve`` 1151 The TransmissionCurve attached to the exposure. 1153 return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1154 filterTransmission=filterTransmission,
1155 sensorTransmission=sensorTransmission,
1156 atmosphereTransmission=atmosphereTransmission)
1160 """Context manager that applies and removes gain 1163 ccd = exp.getDetector()
1165 sim = image.Factory(image, amp.getBBox())
1166 sim *= amp.getGain()
1172 ccd = exp.getDetector()
1174 sim = image.Factory(image, amp.getBBox())
1175 sim /= amp.getGain()
1179 """Context manager that applies and removes flats and darks, 1180 if the task is configured to apply them. 1182 if self.config.doDark
and dark
is not None:
1184 if self.config.doFlat:
1189 if self.config.doFlat:
1191 if self.config.doDark
and dark
is not None:
1196 """A Detector-like object that supports returning gain and saturation level""" 1199 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1201 self.
_gain = config.gain
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
def runDataRef(self, sensorRef)
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
def gainContext(self, exp, image, apply)
def __init__(self, args, kwargs)
Constructor for IsrTask.
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Apply common instrument signature correction algorithms to a raw frame.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Is linearization wanted for this detector?
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
def flatContext(self, exp, flat, dark=None)
def overscanCorrection(self, exposure, amp)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)