1 from __future__
import absolute_import, division, print_function
2 from builtins
import range
3 from builtins
import object
35 from lsstDebug
import getDebugFrame
37 from .
import isrFunctions
38 from .assembleCcdTask
import AssembleCcdTask
39 from .fringe
import FringeTask
43 from contextlib
import contextmanager
44 from .isr
import maskNans
45 from .crosstalk
import CrosstalkTask
49 doBias = pexConfig.Field(
51 doc=
"Apply bias frame correction?",
54 doDark = pexConfig.Field(
56 doc=
"Apply dark frame correction?",
59 doFlat = pexConfig.Field(
61 doc=
"Apply flat field correction?",
64 doFringe = pexConfig.Field(
66 doc=
"Apply fringe correction?",
69 doDefect = pexConfig.Field(
71 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
74 doAddDistortionModel = pexConfig.Field(
76 doc=
"Apply a distortion model based on camera geometry to the WCS?",
79 doWrite = pexConfig.Field(
81 doc=
"Persist postISRCCD?",
84 biasDataProductName = pexConfig.Field(
86 doc=
"Name of the bias data product",
89 darkDataProductName = pexConfig.Field(
91 doc=
"Name of the dark data product",
94 flatDataProductName = pexConfig.Field(
96 doc=
"Name of the flat data product",
99 assembleCcd = pexConfig.ConfigurableField(
100 target=AssembleCcdTask,
101 doc=
"CCD assembly task",
103 gain = pexConfig.Field(
105 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
106 default=float(
"NaN"),
108 readNoise = pexConfig.Field(
110 doc=
"The read noise to use if no Detector is present in the Exposure",
113 saturation = pexConfig.Field(
115 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
116 default=float(
"NaN"),
118 fringeAfterFlat = pexConfig.Field(
120 doc=
"Do fringe subtraction after flat-fielding?",
123 fringe = pexConfig.ConfigurableField(
125 doc=
"Fringe subtraction task",
127 fwhm = pexConfig.Field(
129 doc=
"FWHM of PSF (arcsec)",
132 saturatedMaskName = pexConfig.Field(
134 doc=
"Name of mask plane to use in saturation detection and interpolation",
137 suspectMaskName = pexConfig.Field(
139 doc=
"Name of mask plane to use for suspect pixels",
142 flatScalingType = pexConfig.ChoiceField(
144 doc=
"The method for scaling the flat on the fly.",
147 "USER":
"Scale by flatUserScale",
148 "MEAN":
"Scale by the inverse of the mean",
149 "MEDIAN":
"Scale by the inverse of the median",
152 flatUserScale = pexConfig.Field(
154 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
157 overscanFitType = pexConfig.ChoiceField(
159 doc=
"The method for fitting the overscan bias level.",
162 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
163 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
164 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
165 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
166 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
167 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
168 "MEAN":
"Correct using the mean of the overscan region",
169 "MEDIAN":
"Correct using the median of the overscan region",
172 overscanOrder = pexConfig.Field(
174 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
175 "or number of spline knots if overscan fit type is a spline."),
178 overscanRej = pexConfig.Field(
180 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
183 growSaturationFootprintSize = pexConfig.Field(
185 doc=
"Number of pixels by which to grow the saturation footprints",
188 doSaturationInterpolation = pexConfig.Field(
190 doc=
"Perform interpolation over pixels masked as saturated?",
193 doNanInterpAfterFlat = pexConfig.Field(
195 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 196 "also have to interpolate them before flat-fielding."),
199 fluxMag0T1 = pexConfig.Field(
201 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure",
204 keysToRemoveFromAssembledCcd = pexConfig.ListField(
206 doc=
"fields to remove from the metadata of the assembled ccd.",
209 doAssembleIsrExposures = pexConfig.Field(
212 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 214 doAssembleCcd = pexConfig.Field(
217 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 219 expectWcs = pexConfig.Field(
222 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)" 224 doLinearize = pexConfig.Field(
226 doc=
"Correct for nonlinearity of the detector's response?",
229 doCrosstalk = pexConfig.Field(
231 doc=
"Apply intra-CCD crosstalk correction?",
234 crosstalk = pexConfig.ConfigurableField(
235 target=CrosstalkTask,
236 doc=
"Intra-CCD crosstalk correction",
238 doBrighterFatter = pexConfig.Field(
241 doc=
"Apply the brighter fatter correction" 243 brighterFatterKernelFile = pexConfig.Field(
246 doc=
"Kernel file used for the brighter fatter correction" 248 brighterFatterMaxIter = pexConfig.Field(
251 doc=
"Maximum number of iterations for the brighter fatter correction" 253 brighterFatterThreshold = pexConfig.Field(
256 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 257 " absolute value of the difference between the current corrected image and the one" 258 " from the previous iteration summed over all the pixels." 260 brighterFatterApplyGain = pexConfig.Field(
263 doc=
"Should the gain be applied when applying the brighter fatter correction?" 265 datasetType = pexConfig.Field(
267 doc=
"Dataset type for input data; users will typically leave this alone, " 268 "but camera-specific ISR tasks will override it",
271 fallbackFilterName = pexConfig.Field(dtype=str,
272 doc=
"Fallback default filter name for calibrations", optional=
True)
273 doAttachTransmissionCurve = pexConfig.Field(
276 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 278 doUseOpticsTransmission = pexConfig.Field(
281 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 283 doUseFilterTransmission = pexConfig.Field(
286 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 288 doUseSensorTransmission = pexConfig.Field(
291 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 293 doUseAtmosphereTransmission = pexConfig.Field(
296 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 298 doEmpiricalReadNoise = pexConfig.Field(
301 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 316 @brief Apply common instrument signature correction algorithms to a raw frame. 318 @section ip_isr_isr_Contents Contents 320 - @ref ip_isr_isr_Purpose 321 - @ref ip_isr_isr_Initialize 323 - @ref ip_isr_isr_Config 324 - @ref ip_isr_isr_Debug 327 @section ip_isr_isr_Purpose Description 329 The process for correcting imaging data is very similar from camera to camera. 330 This task provides a vanilla implementation of doing these corrections, including 331 the ability to turn certain corrections off if they are not needed. 332 The inputs to the primary method, run, are a raw exposure to be corrected and the 333 calibration data products. The raw input is a single chip sized mosaic of all amps 334 including overscans and other non-science pixels. 335 The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 336 and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 337 This task may not meet all needs and it is expected that it will be subclassed for 338 specific applications. 340 @section ip_isr_isr_Initialize Task initialization 342 @copydoc \_\_init\_\_ 344 @section ip_isr_isr_IO Inputs/Outputs to the run method 348 @section ip_isr_isr_Config Configuration parameters 350 See @ref IsrTaskConfig 352 @section ip_isr_isr_Debug Debug variables 354 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 355 flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 356 href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 357 Using lsstDebug to control debugging output</a> for more about @b debug.py files. 359 The available variables in IsrTask are: 362 <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 365 <DD> display exposure after ISR has been applied 369 For example, put something like 373 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 374 if name == "lsst.ip.isrFunctions.isrTask": 375 di.display = {'postISRCCD':2} 377 lsstDebug.Info = DebugInfo 379 into your debug.py file and run the commandline task with the @c --debug flag. 383 ConfigClass = IsrTaskConfig
387 '''!Constructor for IsrTask 388 @param[in] *args a list of positional arguments passed on to the Task constructor 389 @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 390 Call the lsst.pipe.base.task.Task.__init__ method 391 Then setup the assembly and fringe correction subtasks 393 pipeBase.Task.__init__(self, *args, **kwargs)
394 self.makeSubtask(
"assembleCcd")
395 self.makeSubtask(
"fringe")
396 self.makeSubtask(
"crosstalk")
399 """!Retrieve necessary frames for instrument signature removal 400 @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 401 of the detector data to be processed 402 @param[in] rawExposure a reference raw exposure that will later be 403 corrected with the retrieved calibration data; 404 should not be modified in this method. 405 @return a pipeBase.Struct with fields containing kwargs expected by run() 406 - bias: exposure of bias frame 407 - dark: exposure of dark frame 408 - flat: exposure of flat field 409 - defects: list of detects 410 - fringeStruct: a pipeBase.Struct with field fringes containing 411 exposure of fringe frame or list of fringe exposure 413 ccd = rawExposure.getDetector()
415 biasExposure = self.
getIsrExposure(dataRef, self.config.biasDataProductName) \
416 if self.config.doBias
else None 418 linearizer = dataRef.get(
"linearizer", immediate=
True)
if self.
doLinearize(ccd)
else None 419 darkExposure = self.
getIsrExposure(dataRef, self.config.darkDataProductName) \
420 if self.config.doDark
else None 421 flatExposure = self.
getIsrExposure(dataRef, self.config.flatDataProductName) \
422 if self.config.doFlat
else None 423 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
if self.config.doBrighterFatter
else None 424 defectList = dataRef.get(
"defects")
if self.config.doDefect
else None 426 if self.config.doCrosstalk:
427 crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
429 crosstalkSources =
None 431 if self.config.doFringe
and self.fringe.checkFilter(rawExposure):
432 fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
433 if self.config.doAssembleIsrExposures
else None)
435 fringeStruct = pipeBase.Struct(fringes=
None)
437 if self.config.doAttachTransmissionCurve:
438 opticsTransmission = (dataRef.get(
"transmission_optics")
439 if self.config.doUseOpticsTransmission
else None)
440 filterTransmission = (dataRef.get(
"transmission_filter")
441 if self.config.doUseFilterTransmission
else None)
442 sensorTransmission = (dataRef.get(
"transmission_sensor")
443 if self.config.doUseSensorTransmission
else None)
444 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
445 if self.config.doUseAtmosphereTransmission
else None)
447 opticsTransmission =
None 448 filterTransmission =
None 449 sensorTransmission =
None 450 atmosphereTransmission =
None 453 return pipeBase.Struct(bias=biasExposure,
454 linearizer=linearizer,
458 fringes=fringeStruct,
459 bfKernel=brighterFatterKernel,
460 opticsTransmission=opticsTransmission,
461 filterTransmission=filterTransmission,
462 sensorTransmission=sensorTransmission,
463 atmosphereTransmission=atmosphereTransmission,
464 crosstalkSources=crosstalkSources,
468 def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
469 fringes=None, bfKernel=None, camera=None,
470 opticsTransmission=None, filterTransmission=None,
471 sensorTransmission=None, atmosphereTransmission=None,
472 crosstalkSources=None):
473 """!Perform instrument signature removal on an exposure 476 - Detect saturation, apply overscan correction, bias, dark and flat 477 - Perform CCD assembly 478 - Interpolate over defects, saturated pixels and all NaNs 480 @param[in] ccdExposure lsst.afw.image.exposure of detector data 481 @param[in] bias exposure of bias frame 482 @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 483 @param[in] dark exposure of dark frame 484 @param[in] flat exposure of flatfield 485 @param[in] defects list of detects 486 @param[in] fringes a pipeBase.Struct with field fringes containing 487 exposure of fringe frame or list of fringe exposure 488 @param[in] bfKernel kernel for brighter-fatter correction 489 @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 490 used by addDistortionModel 491 @param[in] opticsTransmission a TransmissionCurve for the optics 492 @param[in] filterTransmission a TransmissionCurve for the filter 493 @param[in] sensorTransmission a TransmissionCurve for the sensor 494 @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 495 @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk 497 @return a pipeBase.Struct with field: 501 if isinstance(ccdExposure, ButlerDataRef):
504 ccd = ccdExposure.getDetector()
507 if self.config.doBias
and bias
is None:
508 raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
510 raise RuntimeError(
"Must supply a linearizer if config.doBias True")
511 if self.config.doDark
and dark
is None:
512 raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
513 if self.config.doFlat
and flat
is None:
514 raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
515 if self.config.doBrighterFatter
and bfKernel
is None:
516 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter True")
518 fringes = pipeBase.Struct(fringes=
None)
519 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
520 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
521 if self.config.doDefect
and defects
is None:
522 raise RuntimeError(
"Must supply defects if config.doDefect True")
523 if self.config.doAddDistortionModel
and camera
is None:
524 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel True")
529 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 530 ccd = [
FakeAmp(ccdExposure, self.config)]
535 if ccdExposure.getBBox().contains(amp.getBBox()):
539 overscans.append(overscanResults.overscanImage
if overscanResults
is not None else None)
541 overscans.append(
None)
543 if self.config.doCrosstalk:
544 self.crosstalk.
run(ccdExposure, crosstalkSources)
546 if self.config.doAssembleCcd:
547 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
548 if self.config.expectWcs
and not ccdExposure.getWcs():
549 self.log.warn(
"No WCS found in input exposure")
551 if self.config.doBias:
555 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
557 assert len(ccd) == len(overscans)
558 for amp, overscanImage
in zip(ccd, overscans):
560 if ccdExposure.getBBox().contains(amp.getBBox()):
561 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
564 interpolationDone =
False 566 if self.config.doBrighterFatter:
573 if self.config.doDefect:
575 if self.config.doSaturationInterpolation:
578 interpolationDone =
True 581 self.config.brighterFatterMaxIter,
582 self.config.brighterFatterThreshold,
583 self.config.brighterFatterApplyGain,
586 if self.config.doDark:
589 if self.config.doFringe
and not self.config.fringeAfterFlat:
590 self.fringe.
run(ccdExposure, **fringes.getDict())
592 if self.config.doFlat:
595 if not interpolationDone:
596 if self.config.doDefect:
598 if self.config.doSaturationInterpolation:
600 if not interpolationDone
or self.config.doNanInterpAfterFlat:
603 if self.config.doFringe
and self.config.fringeAfterFlat:
604 self.fringe.
run(ccdExposure, **fringes.getDict())
606 exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
607 ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
609 if self.config.doAddDistortionModel:
612 if self.config.doAttachTransmissionCurve:
614 filterTransmission=filterTransmission,
615 sensorTransmission=sensorTransmission,
616 atmosphereTransmission=atmosphereTransmission)
618 frame = getDebugFrame(self._display,
"postISRCCD")
620 getDisplay(frame).mtv(ccdExposure)
622 return pipeBase.Struct(
623 exposure=ccdExposure,
628 """Perform instrument signature removal on a ButlerDataRef of a Sensor 630 - Read in necessary detrending/isr/calibration data 631 - Process raw exposure in run() 632 - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 636 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 637 DataRef of the detector data to be processed 641 result : `pipeBase.Struct` 642 Struct contains field "exposure," which is the exposure after application of ISR 644 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
645 ccdExposure = sensorRef.get(
'raw')
646 camera = sensorRef.get(
"camera")
647 if camera
is None and self.config.doAddDistortionModel:
648 raise RuntimeError(
"config.doAddDistortionModel is True " 649 "but could not get a camera from the butler")
652 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
654 if self.config.doWrite:
655 sensorRef.put(result.exposure,
"postISRCCD")
660 """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 662 if isinstance(exposure, afwImage.ExposureF):
665 if not hasattr(exposure,
"convertF"):
666 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
668 newexposure = exposure.convertF()
669 maskedImage = newexposure.getMaskedImage()
670 varArray = maskedImage.getVariance().getArray()
672 maskArray = maskedImage.getMask().getArray()
677 """!Apply bias correction in place 679 @param[in,out] exposure exposure to process 680 @param[in] biasExposure bias exposure of same size as exposure 682 isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
685 """!Apply dark correction in place 687 @param[in,out] exposure exposure to process 688 @param[in] darkExposure dark exposure of same size as exposure 689 @param[in] invert if True, remove the dark from an already-corrected image 691 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
692 if math.isnan(expScale):
693 raise RuntimeError(
"Exposure darktime is NAN")
694 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
695 if math.isnan(darkScale):
696 raise RuntimeError(
"Dark calib darktime is NAN")
697 isrFunctions.darkCorrection(
698 maskedImage=exposure.getMaskedImage(),
699 darkMaskedImage=darkExposure.getMaskedImage(),
706 """!Is linearization wanted for this detector? 708 Checks config.doLinearize and the linearity type of the first amplifier. 710 @param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 712 return self.config.doLinearize
and \
713 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
716 """Set the variance plane using the amplifier gain and read noise 718 The read noise is calculated from the ``overscanImage`` if the 719 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 720 the value from the amplifier data is used. 724 ampExposure : `lsst.afw.image.Exposure` 726 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 727 Amplifier detector data. 728 overscanImage : `lsst.afw.image.MaskedImage`, optional. 729 Image of overscan, required only for empirical read noise. 731 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
733 if not math.isnan(gain):
736 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
737 (amp.getName(), gain, patchedGain))
740 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
741 stats = afwMath.StatisticsControl()
742 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
743 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
744 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
746 readNoise = amp.getReadNoise()
748 isrFunctions.updateVariance(
749 maskedImage=ampExposure.getMaskedImage(),
755 """!Apply flat correction in place 757 @param[in,out] exposure exposure to process 758 @param[in] flatExposure flatfield exposure same size as exposure 759 @param[in] invert if True, unflatten an already-flattened image instead. 761 isrFunctions.flatCorrection(
762 maskedImage=exposure.getMaskedImage(),
763 flatMaskedImage=flatExposure.getMaskedImage(),
764 scalingType=self.config.flatScalingType,
765 userScale=self.config.flatUserScale,
770 """!Retrieve a calibration dataset for removing instrument signature 772 @param[in] dataRef data reference for exposure 773 @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 774 @param[in] immediate if True, disable butler proxies to enable error 775 handling within this routine 779 exp = dataRef.get(datasetType, immediate=immediate)
780 except Exception
as exc1:
781 if not self.config.fallbackFilterName:
782 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
784 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
785 except Exception
as exc2:
786 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
787 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
788 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
790 if self.config.doAssembleIsrExposures:
791 exp = self.assembleCcd.assembleCcd(exp)
795 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 797 @param[in,out] exposure exposure to process; only the amp DataSec is processed 798 @param[in] amp amplifier device data 800 if not math.isnan(amp.getSaturation()):
801 maskedImage = exposure.getMaskedImage()
802 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
803 isrFunctions.makeThresholdMask(
804 maskedImage=dataView,
805 threshold=amp.getSaturation(),
807 maskName=self.config.saturatedMaskName,
811 """!Interpolate over saturated pixels, in place 813 @param[in,out] ccdExposure exposure to process 816 - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 817 - Call this after CCD assembly, since saturated regions may cross amplifier boundaries 819 isrFunctions.interpolateFromMask(
820 maskedImage=ccdExposure.getMaskedImage(),
821 fwhm=self.config.fwhm,
822 growFootprints=self.config.growSaturationFootprintSize,
823 maskName=self.config.saturatedMaskName,
827 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 829 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 830 This is intended to indicate pixels that may be affected by unknown systematics; 831 for example if non-linearity corrections above a certain level are unstable 832 then that would be a useful value for suspectLevel. A value of `nan` indicates 833 that no such level exists and no pixels are to be masked as suspicious. 835 @param[in,out] exposure exposure to process; only the amp DataSec is processed 836 @param[in] amp amplifier device data 838 suspectLevel = amp.getSuspectLevel()
839 if math.isnan(suspectLevel):
842 maskedImage = exposure.getMaskedImage()
843 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
844 isrFunctions.makeThresholdMask(
845 maskedImage=dataView,
846 threshold=suspectLevel,
848 maskName=self.config.suspectMaskName,
852 """!Mask defects using mask plane "BAD" and interpolate over them, in place 854 @param[in,out] ccdExposure exposure to process 855 @param[in] defectBaseList a list of defects to mask and interpolate 857 @warning: call this after CCD assembly, since defects may cross amplifier boundaries 859 maskedImage = ccdExposure.getMaskedImage()
861 for d
in defectBaseList:
863 nd = measAlg.Defect(bbox)
864 defectList.append(nd)
865 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
866 isrFunctions.interpolateDefectList(
867 maskedImage=maskedImage,
868 defectList=defectList,
869 fwhm=self.config.fwhm,
873 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 875 We mask and interpolate over all NaNs, including those 876 that are masked with other bits (because those may or may 877 not be interpolated over later, and we want to remove all 878 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 879 is used to preserve the historical name. 881 @param[in,out] exposure exposure to process 883 maskedImage = exposure.getMaskedImage()
886 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
887 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
888 numNans =
maskNans(maskedImage, maskVal)
889 self.metadata.set(
"NUMNANS", numNans)
893 self.log.warn(
"There were %i unmasked NaNs", numNans)
894 nanDefectList = isrFunctions.getDefectListFromMask(
895 maskedImage=maskedImage,
896 maskName=
'UNMASKEDNAN',
898 isrFunctions.interpolateDefectList(
899 maskedImage=exposure.getMaskedImage(),
900 defectList=nanDefectList,
901 fwhm=self.config.fwhm,
905 """Apply overscan correction, in-place 909 exposure : `lsst.afw.image.Exposure` 910 Exposure to process; must include both data and bias regions. 911 amp : `lsst.afw.table.AmpInfoRecord` 912 Amplifier device data. 916 result : `lsst.pipe.base.Struct` or `NoneType` 917 `None` if there is no overscan; otherwise, this is a 918 result struct with components: 920 - ``imageFit``: Value(s) removed from image (scalar or 921 `lsst.afw.image.Image`). 922 - ``overscanFit``: Value(s) removed from overscan (scalar or 923 `lsst.afw.image.Image`). 924 - ``overscanImage``: Image of the overscan, post-subtraction 925 (`lsst.afw.image.Image`). 927 if not amp.getHasRawInfo():
928 raise RuntimeError(
"This method must be executed on an amp with raw information.")
930 if amp.getRawHorizontalOverscanBBox().isEmpty():
931 self.log.info(
"No Overscan region. Not performing Overscan Correction.")
934 maskedImage = exposure.getMaskedImage()
935 dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
936 overscanImage = maskedImage.Factory(maskedImage, amp.getRawHorizontalOverscanBBox())
938 results = isrFunctions.overscanCorrection(
939 ampMaskedImage=dataView,
940 overscanImage=overscanImage,
941 fitType=self.config.overscanFitType,
942 order=self.config.overscanOrder,
943 collapseRej=self.config.overscanRej,
945 results.overscanImage = overscanImage
949 """!Update the WCS in exposure with a distortion model based on camera geometry 951 Add a model for optical distortion based on geometry found in `camera` 952 and the `exposure`'s detector. The raw input exposure is assumed 953 have a TAN WCS that has no compensation for optical distortion. 954 Two other possibilities are: 955 - The raw input exposure already has a model for optical distortion, 956 as is the case for raw DECam data. 957 In that case you should set config.doAddDistortionModel False. 958 - The raw input exposure has a model for distortion, but it has known 959 deficiencies severe enough to be worth fixing (e.g. because they 960 cause problems for fitting a better WCS). In that case you should 961 override this method with a version suitable for your raw data. 963 @param[in,out] exposure exposure to process; must include a Detector and a WCS; 964 the WCS of the exposure is modified in place 965 @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 967 self.log.info(
"Adding a distortion model to the WCS")
968 wcs = exposure.getWcs()
970 raise RuntimeError(
"exposure has no WCS")
972 raise RuntimeError(
"camera is None")
973 detector = exposure.getDetector()
975 raise RuntimeError(
"exposure has no Detector")
976 pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
977 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
978 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
979 exposure.setWcs(distortedWcs)
982 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners 984 @param[in,out] ccdExposure exposure to process 985 @param[in] fpPolygon Polygon in focal plane coordinates 988 ccd = ccdExposure.getDetector()
989 fpCorners = ccd.getCorners(FOCAL_PLANE)
990 ccdPolygon = Polygon(fpCorners)
993 intersect = ccdPolygon.intersectionSingle(fpPolygon)
996 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
997 validPolygon = Polygon(ccdPoints)
998 ccdExposure.getInfo().setValidPolygon(validPolygon)
1001 """Apply brighter fatter correction in place for the image 1003 This correction takes a kernel that has been derived from flat field images to 1004 redistribute the charge. The gradient of the kernel is the deflection 1005 field due to the accumulated charge. 1007 Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 1008 using the following equation: 1010 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 1012 To evaluate the derivative term we expand it as follows: 1014 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 1016 Because we use the measured counts instead of the incident counts we apply the correction 1017 iteratively to reconstruct the original counts and the correction. We stop iterating when the 1018 summed difference between the current corrected image and the one from the previous iteration 1019 is below the threshold. We do not require convergence because the number of iterations is 1020 too large a computational cost. How we define the threshold still needs to be evaluated, the 1021 current default was shown to work reasonably well on a small set of images. For more information 1022 on the method see DocuShare Document-19407. 1024 The edges as defined by the kernel are not corrected because they have spurious values 1025 due to the convolution. 1027 self.log.info(
"Applying brighter fatter correction")
1029 image = exposure.getMaskedImage().getImage()
1032 with self.
gainContext(exposure, image, applyGain):
1034 kLx = numpy.shape(kernel)[0]
1035 kLy = numpy.shape(kernel)[1]
1036 kernelImage = afwImage.ImageD(kLx, kLy)
1037 kernelImage.getArray()[:, :] = kernel
1038 tempImage = image.clone()
1040 nanIndex = numpy.isnan(tempImage.getArray())
1041 tempImage.getArray()[nanIndex] = 0.
1043 outImage = afwImage.ImageF(image.getDimensions())
1044 corr = numpy.zeros_like(image.getArray())
1045 prev_image = numpy.zeros_like(image.getArray())
1046 convCntrl = afwMath.ConvolutionControl(
False,
True, 1)
1047 fixedKernel = afwMath.FixedKernel(kernelImage)
1057 for iteration
in range(maxIter):
1059 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1060 tmpArray = tempImage.getArray()
1061 outArray = outImage.getArray()
1063 with numpy.errstate(invalid=
"ignore", over=
"ignore"):
1065 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1066 gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1067 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1070 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1071 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1072 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1074 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1076 tmpArray[:, :] = image.getArray()[:, :]
1077 tmpArray[nanIndex] = 0.
1078 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1081 diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1083 if diff < threshold:
1085 prev_image[:, :] = tmpArray[:, :]
1087 if iteration == maxIter - 1:
1088 self.log.warn(
"Brighter fatter correction did not converge, final difference %f" % diff)
1090 self.log.info(
"Finished brighter fatter in %d iterations" % (iteration + 1))
1091 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1092 corr[startY + 1:endY - 1, startX + 1:endX - 1]
1095 sensorTransmission=None, atmosphereTransmission=None):
1096 """Attach a TransmissionCurve to an Exposure, given separate curves for 1097 different components. 1101 exposure : `lsst.afw.image.Exposure` 1102 Exposure object to modify by attaching the product of all given 1103 ``TransmissionCurves`` in post-assembly trimmed detector 1104 coordinates. Must have a valid ``Detector`` attached that matches 1105 the detector associated with sensorTransmission. 1106 opticsTransmission : `lsst.afw.image.TransmissionCurve` 1107 A ``TransmissionCurve`` that represents the throughput of the 1108 optics, to be evaluated in focal-plane coordinates. 1109 filterTransmission : `lsst.afw.image.TransmissionCurve` 1110 A ``TransmissionCurve`` that represents the throughput of the 1111 filter itself, to be evaluated in focal-plane coordinates. 1112 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1113 A ``TransmissionCurve`` that represents the throughput of the 1114 sensor itself, to be evaluated in post-assembly trimmed detector 1116 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1117 A ``TransmissionCurve`` that represents the throughput of the 1118 atmosphere, assumed to be spatially constant. 1120 All ``TransmissionCurve`` arguments are optional; if none are provided, 1121 the attached ``TransmissionCurve`` will have unit transmission 1126 combined : ``lsst.afw.image.TransmissionCurve`` 1127 The TransmissionCurve attached to the exposure. 1129 return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1130 filterTransmission=filterTransmission,
1131 sensorTransmission=sensorTransmission,
1132 atmosphereTransmission=atmosphereTransmission)
1136 """Context manager that applies and removes gain 1139 ccd = exp.getDetector()
1141 sim = image.Factory(image, amp.getBBox())
1142 sim *= amp.getGain()
1148 ccd = exp.getDetector()
1150 sim = image.Factory(image, amp.getBBox())
1151 sim /= amp.getGain()
1155 """Context manager that applies and removes flats and darks, 1156 if the task is configured to apply them. 1158 if self.config.doDark
and dark
is not None:
1160 if self.config.doFlat:
1165 if self.config.doFlat:
1167 if self.config.doDark
and dark
is not None:
1172 """A Detector-like object that supports returning gain and saturation level""" 1175 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1177 self.
_gain = config.gain
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
def runDataRef(self, sensorRef)
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
def gainContext(self, exp, image, apply)
def __init__(self, args, kwargs)
Constructor for IsrTask.
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Apply common instrument signature correction algorithms to a raw frame.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Is linearization wanted for this detector?
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
def flatContext(self, exp, flat, dark=None)
def overscanCorrection(self, exposure, amp)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)