32 from lsstDebug
import getDebugFrame
34 from .
import isrFunctions
35 from .assembleCcdTask
import AssembleCcdTask
36 from .fringe
import FringeTask
40 from contextlib
import contextmanager
41 from .isr
import maskNans
42 from .crosstalk
import CrosstalkTask
46 doBias = pexConfig.Field(
48 doc=
"Apply bias frame correction?",
51 doDark = pexConfig.Field(
53 doc=
"Apply dark frame correction?",
56 doFlat = pexConfig.Field(
58 doc=
"Apply flat field correction?",
61 doFringe = pexConfig.Field(
63 doc=
"Apply fringe correction?",
66 doDefect = pexConfig.Field(
68 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
71 doAddDistortionModel = pexConfig.Field(
73 doc=
"Apply a distortion model based on camera geometry to the WCS?",
76 doWrite = pexConfig.Field(
78 doc=
"Persist postISRCCD?",
81 biasDataProductName = pexConfig.Field(
83 doc=
"Name of the bias data product",
86 darkDataProductName = pexConfig.Field(
88 doc=
"Name of the dark data product",
91 flatDataProductName = pexConfig.Field(
93 doc=
"Name of the flat data product",
96 assembleCcd = pexConfig.ConfigurableField(
97 target=AssembleCcdTask,
98 doc=
"CCD assembly task",
100 gain = pexConfig.Field(
102 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
103 default=float(
"NaN"),
105 readNoise = pexConfig.Field(
107 doc=
"The read noise to use if no Detector is present in the Exposure",
110 saturation = pexConfig.Field(
112 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
113 default=float(
"NaN"),
115 fringeAfterFlat = pexConfig.Field(
117 doc=
"Do fringe subtraction after flat-fielding?",
120 fringe = pexConfig.ConfigurableField(
122 doc=
"Fringe subtraction task",
124 fwhm = pexConfig.Field(
126 doc=
"FWHM of PSF (arcsec)",
129 saturatedMaskName = pexConfig.Field(
131 doc=
"Name of mask plane to use in saturation detection and interpolation",
134 suspectMaskName = pexConfig.Field(
136 doc=
"Name of mask plane to use for suspect pixels",
139 flatScalingType = pexConfig.ChoiceField(
141 doc=
"The method for scaling the flat on the fly.",
144 "USER":
"Scale by flatUserScale",
145 "MEAN":
"Scale by the inverse of the mean",
146 "MEDIAN":
"Scale by the inverse of the median",
149 flatUserScale = pexConfig.Field(
151 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
154 overscanFitType = pexConfig.ChoiceField(
156 doc=
"The method for fitting the overscan bias level.",
159 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
160 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
161 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
162 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
163 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
164 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
165 "MEAN":
"Correct using the mean of the overscan region",
166 "MEDIAN":
"Correct using the median of the overscan region",
169 overscanOrder = pexConfig.Field(
171 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
172 "or number of spline knots if overscan fit type is a spline."),
175 overscanRej = pexConfig.Field(
177 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
180 growSaturationFootprintSize = pexConfig.Field(
182 doc=
"Number of pixels by which to grow the saturation footprints",
185 doSaturationInterpolation = pexConfig.Field(
187 doc=
"Perform interpolation over pixels masked as saturated?",
190 doNanInterpAfterFlat = pexConfig.Field(
192 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 193 "also have to interpolate them before flat-fielding."),
196 fluxMag0T1 = pexConfig.Field(
198 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure",
201 keysToRemoveFromAssembledCcd = pexConfig.ListField(
203 doc=
"fields to remove from the metadata of the assembled ccd.",
206 doAssembleIsrExposures = pexConfig.Field(
209 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 211 doAssembleCcd = pexConfig.Field(
214 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 216 expectWcs = pexConfig.Field(
219 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)" 221 doLinearize = pexConfig.Field(
223 doc=
"Correct for nonlinearity of the detector's response?",
226 doCrosstalk = pexConfig.Field(
228 doc=
"Apply intra-CCD crosstalk correction?",
231 crosstalk = pexConfig.ConfigurableField(
232 target=CrosstalkTask,
233 doc=
"Intra-CCD crosstalk correction",
235 doBrighterFatter = pexConfig.Field(
238 doc=
"Apply the brighter fatter correction" 240 brighterFatterKernelFile = pexConfig.Field(
243 doc=
"Kernel file used for the brighter fatter correction" 245 brighterFatterMaxIter = pexConfig.Field(
248 doc=
"Maximum number of iterations for the brighter fatter correction" 250 brighterFatterThreshold = pexConfig.Field(
253 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 254 " absolute value of the difference between the current corrected image and the one" 255 " from the previous iteration summed over all the pixels." 257 brighterFatterApplyGain = pexConfig.Field(
260 doc=
"Should the gain be applied when applying the brighter fatter correction?" 262 datasetType = pexConfig.Field(
264 doc=
"Dataset type for input data; users will typically leave this alone, " 265 "but camera-specific ISR tasks will override it",
268 fallbackFilterName = pexConfig.Field(dtype=str,
269 doc=
"Fallback default filter name for calibrations", optional=
True)
270 doAttachTransmissionCurve = pexConfig.Field(
273 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 275 doUseOpticsTransmission = pexConfig.Field(
278 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 280 doUseFilterTransmission = pexConfig.Field(
283 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 285 doUseSensorTransmission = pexConfig.Field(
288 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 290 doUseAtmosphereTransmission = pexConfig.Field(
293 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 295 doEmpiricalReadNoise = pexConfig.Field(
298 doc=
"Calculate empirical read noise instead of value from AmpInfo data?" 313 @brief Apply common instrument signature correction algorithms to a raw frame. 315 @section ip_isr_isr_Contents Contents 317 - @ref ip_isr_isr_Purpose 318 - @ref ip_isr_isr_Initialize 320 - @ref ip_isr_isr_Config 321 - @ref ip_isr_isr_Debug 324 @section ip_isr_isr_Purpose Description 326 The process for correcting imaging data is very similar from camera to camera. 327 This task provides a vanilla implementation of doing these corrections, including 328 the ability to turn certain corrections off if they are not needed. 329 The inputs to the primary method, run, are a raw exposure to be corrected and the 330 calibration data products. The raw input is a single chip sized mosaic of all amps 331 including overscans and other non-science pixels. 332 The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 333 and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 334 This task may not meet all needs and it is expected that it will be subclassed for 335 specific applications. 337 @section ip_isr_isr_Initialize Task initialization 339 @copydoc \_\_init\_\_ 341 @section ip_isr_isr_IO Inputs/Outputs to the run method 345 @section ip_isr_isr_Config Configuration parameters 347 See @ref IsrTaskConfig 349 @section ip_isr_isr_Debug Debug variables 351 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 352 flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 353 href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 354 Using lsstDebug to control debugging output</a> for more about @b debug.py files. 356 The available variables in IsrTask are: 359 <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 362 <DD> display exposure after ISR has been applied 366 For example, put something like 370 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 371 if name == "lsst.ip.isrFunctions.isrTask": 372 di.display = {'postISRCCD':2} 374 lsstDebug.Info = DebugInfo 376 into your debug.py file and run the commandline task with the @c --debug flag. 380 ConfigClass = IsrTaskConfig
384 '''!Constructor for IsrTask 385 @param[in] *args a list of positional arguments passed on to the Task constructor 386 @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 387 Call the lsst.pipe.base.task.Task.__init__ method 388 Then setup the assembly and fringe correction subtasks 390 pipeBase.Task.__init__(self, *args, **kwargs)
391 self.makeSubtask(
"assembleCcd")
392 self.makeSubtask(
"fringe")
393 self.makeSubtask(
"crosstalk")
396 """!Retrieve necessary frames for instrument signature removal 397 @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 398 of the detector data to be processed 399 @param[in] rawExposure a reference raw exposure that will later be 400 corrected with the retrieved calibration data; 401 should not be modified in this method. 402 @return a pipeBase.Struct with fields containing kwargs expected by run() 403 - bias: exposure of bias frame 404 - dark: exposure of dark frame 405 - flat: exposure of flat field 406 - defects: list of detects 407 - fringeStruct: a pipeBase.Struct with field fringes containing 408 exposure of fringe frame or list of fringe exposure 410 ccd = rawExposure.getDetector()
412 biasExposure = self.
getIsrExposure(dataRef, self.config.biasDataProductName) \
413 if self.config.doBias
else None 415 linearizer = dataRef.get(
"linearizer", immediate=
True)
if self.
doLinearize(ccd)
else None 416 darkExposure = self.
getIsrExposure(dataRef, self.config.darkDataProductName) \
417 if self.config.doDark
else None 418 flatExposure = self.
getIsrExposure(dataRef, self.config.flatDataProductName) \
419 if self.config.doFlat
else None 420 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
if self.config.doBrighterFatter
else None 421 defectList = dataRef.get(
"defects")
if self.config.doDefect
else None 423 if self.config.doCrosstalk:
424 crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
426 crosstalkSources =
None 428 if self.config.doFringe
and self.fringe.checkFilter(rawExposure):
429 fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
430 if self.config.doAssembleIsrExposures
else None)
432 fringeStruct = pipeBase.Struct(fringes=
None)
434 if self.config.doAttachTransmissionCurve:
435 opticsTransmission = (dataRef.get(
"transmission_optics")
436 if self.config.doUseOpticsTransmission
else None)
437 filterTransmission = (dataRef.get(
"transmission_filter")
438 if self.config.doUseFilterTransmission
else None)
439 sensorTransmission = (dataRef.get(
"transmission_sensor")
440 if self.config.doUseSensorTransmission
else None)
441 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
442 if self.config.doUseAtmosphereTransmission
else None)
444 opticsTransmission =
None 445 filterTransmission =
None 446 sensorTransmission =
None 447 atmosphereTransmission =
None 450 return pipeBase.Struct(bias=biasExposure,
451 linearizer=linearizer,
455 fringes=fringeStruct,
456 bfKernel=brighterFatterKernel,
457 opticsTransmission=opticsTransmission,
458 filterTransmission=filterTransmission,
459 sensorTransmission=sensorTransmission,
460 atmosphereTransmission=atmosphereTransmission,
461 crosstalkSources=crosstalkSources,
465 def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
466 fringes=None, bfKernel=None, camera=None,
467 opticsTransmission=None, filterTransmission=None,
468 sensorTransmission=None, atmosphereTransmission=None,
469 crosstalkSources=None):
470 """!Perform instrument signature removal on an exposure 473 - Detect saturation, apply overscan correction, bias, dark and flat 474 - Perform CCD assembly 475 - Interpolate over defects, saturated pixels and all NaNs 477 @param[in] ccdExposure lsst.afw.image.exposure of detector data 478 @param[in] bias exposure of bias frame 479 @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 480 @param[in] dark exposure of dark frame 481 @param[in] flat exposure of flatfield 482 @param[in] defects list of detects 483 @param[in] fringes a pipeBase.Struct with field fringes containing 484 exposure of fringe frame or list of fringe exposure 485 @param[in] bfKernel kernel for brighter-fatter correction 486 @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 487 used by addDistortionModel 488 @param[in] opticsTransmission a TransmissionCurve for the optics 489 @param[in] filterTransmission a TransmissionCurve for the filter 490 @param[in] sensorTransmission a TransmissionCurve for the sensor 491 @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 492 @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk 494 @return a pipeBase.Struct with field: 498 if isinstance(ccdExposure, ButlerDataRef):
501 ccd = ccdExposure.getDetector()
504 if self.config.doBias
and bias
is None:
505 raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
507 raise RuntimeError(
"Must supply a linearizer if config.doBias True")
508 if self.config.doDark
and dark
is None:
509 raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
510 if self.config.doFlat
and flat
is None:
511 raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
512 if self.config.doBrighterFatter
and bfKernel
is None:
513 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter True")
515 fringes = pipeBase.Struct(fringes=
None)
516 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
517 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
518 if self.config.doDefect
and defects
is None:
519 raise RuntimeError(
"Must supply defects if config.doDefect True")
520 if self.config.doAddDistortionModel
and camera
is None:
521 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel True")
526 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 527 ccd = [
FakeAmp(ccdExposure, self.config)]
532 if ccdExposure.getBBox().contains(amp.getBBox()):
536 overscans.append(overscanResults.overscanImage
if overscanResults
is not None else None)
538 overscans.append(
None)
540 if self.config.doCrosstalk:
541 self.crosstalk.
run(ccdExposure, crosstalkSources)
543 if self.config.doAssembleCcd:
544 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
545 if self.config.expectWcs
and not ccdExposure.getWcs():
546 self.log.warn(
"No WCS found in input exposure")
548 if self.config.doBias:
552 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
554 assert len(ccd) == len(overscans)
555 for amp, overscanImage
in zip(ccd, overscans):
557 if ccdExposure.getBBox().contains(amp.getBBox()):
558 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
561 interpolationDone =
False 563 if self.config.doBrighterFatter:
570 if self.config.doDefect:
572 if self.config.doSaturationInterpolation:
575 interpolationDone =
True 578 self.config.brighterFatterMaxIter,
579 self.config.brighterFatterThreshold,
580 self.config.brighterFatterApplyGain,
583 if self.config.doDark:
586 if self.config.doFringe
and not self.config.fringeAfterFlat:
587 self.fringe.
run(ccdExposure, **fringes.getDict())
589 if self.config.doFlat:
592 if not interpolationDone:
593 if self.config.doDefect:
595 if self.config.doSaturationInterpolation:
597 if not interpolationDone
or self.config.doNanInterpAfterFlat:
600 if self.config.doFringe
and self.config.fringeAfterFlat:
601 self.fringe.
run(ccdExposure, **fringes.getDict())
603 exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
604 ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
606 if self.config.doAddDistortionModel:
609 if self.config.doAttachTransmissionCurve:
611 filterTransmission=filterTransmission,
612 sensorTransmission=sensorTransmission,
613 atmosphereTransmission=atmosphereTransmission)
615 frame = getDebugFrame(self._display,
"postISRCCD")
617 getDisplay(frame).mtv(ccdExposure)
619 return pipeBase.Struct(
620 exposure=ccdExposure,
625 """Perform instrument signature removal on a ButlerDataRef of a Sensor 627 - Read in necessary detrending/isr/calibration data 628 - Process raw exposure in run() 629 - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 633 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 634 DataRef of the detector data to be processed 638 result : `pipeBase.Struct` 639 Struct contains field "exposure," which is the exposure after application of ISR 641 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
642 ccdExposure = sensorRef.get(
'raw')
643 camera = sensorRef.get(
"camera")
644 if camera
is None and self.config.doAddDistortionModel:
645 raise RuntimeError(
"config.doAddDistortionModel is True " 646 "but could not get a camera from the butler")
649 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
651 if self.config.doWrite:
652 sensorRef.put(result.exposure,
"postISRCCD")
657 """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 659 if isinstance(exposure, afwImage.ExposureF):
662 if not hasattr(exposure,
"convertF"):
663 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
665 newexposure = exposure.convertF()
666 maskedImage = newexposure.getMaskedImage()
667 varArray = maskedImage.getVariance().getArray()
669 maskArray = maskedImage.getMask().getArray()
674 """!Apply bias correction in place 676 @param[in,out] exposure exposure to process 677 @param[in] biasExposure bias exposure of same size as exposure 679 isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
682 """!Apply dark correction in place 684 @param[in,out] exposure exposure to process 685 @param[in] darkExposure dark exposure of same size as exposure 686 @param[in] invert if True, remove the dark from an already-corrected image 688 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
689 if math.isnan(expScale):
690 raise RuntimeError(
"Exposure darktime is NAN")
691 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
692 if math.isnan(darkScale):
693 raise RuntimeError(
"Dark calib darktime is NAN")
694 isrFunctions.darkCorrection(
695 maskedImage=exposure.getMaskedImage(),
696 darkMaskedImage=darkExposure.getMaskedImage(),
703 """!Is linearization wanted for this detector? 705 Checks config.doLinearize and the linearity type of the first amplifier. 707 @param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 709 return self.config.doLinearize
and \
710 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
713 """Set the variance plane using the amplifier gain and read noise 715 The read noise is calculated from the ``overscanImage`` if the 716 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise 717 the value from the amplifier data is used. 721 ampExposure : `lsst.afw.image.Exposure` 723 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 724 Amplifier detector data. 725 overscanImage : `lsst.afw.image.MaskedImage`, optional. 726 Image of overscan, required only for empirical read noise. 728 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
730 if not math.isnan(gain):
733 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
734 (amp.getName(), gain, patchedGain))
737 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
738 stats = afwMath.StatisticsControl()
739 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
740 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
741 self.log.info(
"Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
743 readNoise = amp.getReadNoise()
745 isrFunctions.updateVariance(
746 maskedImage=ampExposure.getMaskedImage(),
752 """!Apply flat correction in place 754 @param[in,out] exposure exposure to process 755 @param[in] flatExposure flatfield exposure same size as exposure 756 @param[in] invert if True, unflatten an already-flattened image instead. 758 isrFunctions.flatCorrection(
759 maskedImage=exposure.getMaskedImage(),
760 flatMaskedImage=flatExposure.getMaskedImage(),
761 scalingType=self.config.flatScalingType,
762 userScale=self.config.flatUserScale,
767 """!Retrieve a calibration dataset for removing instrument signature 769 @param[in] dataRef data reference for exposure 770 @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 771 @param[in] immediate if True, disable butler proxies to enable error 772 handling within this routine 776 exp = dataRef.get(datasetType, immediate=immediate)
777 except Exception
as exc1:
778 if not self.config.fallbackFilterName:
779 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
781 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
782 except Exception
as exc2:
783 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
784 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
785 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
787 if self.config.doAssembleIsrExposures:
788 exp = self.assembleCcd.assembleCcd(exp)
792 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 794 @param[in,out] exposure exposure to process; only the amp DataSec is processed 795 @param[in] amp amplifier device data 797 if not math.isnan(amp.getSaturation()):
798 maskedImage = exposure.getMaskedImage()
799 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
800 isrFunctions.makeThresholdMask(
801 maskedImage=dataView,
802 threshold=amp.getSaturation(),
804 maskName=self.config.saturatedMaskName,
808 """!Interpolate over saturated pixels, in place 810 @param[in,out] ccdExposure exposure to process 813 - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 814 - Call this after CCD assembly, since saturated regions may cross amplifier boundaries 816 isrFunctions.interpolateFromMask(
817 maskedImage=ccdExposure.getMaskedImage(),
818 fwhm=self.config.fwhm,
819 growFootprints=self.config.growSaturationFootprintSize,
820 maskName=self.config.saturatedMaskName,
824 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 826 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 827 This is intended to indicate pixels that may be affected by unknown systematics; 828 for example if non-linearity corrections above a certain level are unstable 829 then that would be a useful value for suspectLevel. A value of `nan` indicates 830 that no such level exists and no pixels are to be masked as suspicious. 832 @param[in,out] exposure exposure to process; only the amp DataSec is processed 833 @param[in] amp amplifier device data 835 suspectLevel = amp.getSuspectLevel()
836 if math.isnan(suspectLevel):
839 maskedImage = exposure.getMaskedImage()
840 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
841 isrFunctions.makeThresholdMask(
842 maskedImage=dataView,
843 threshold=suspectLevel,
845 maskName=self.config.suspectMaskName,
849 """!Mask defects using mask plane "BAD" and interpolate over them, in place 851 @param[in,out] ccdExposure exposure to process 852 @param[in] defectBaseList a list of defects to mask and interpolate 854 @warning: call this after CCD assembly, since defects may cross amplifier boundaries 856 maskedImage = ccdExposure.getMaskedImage()
858 for d
in defectBaseList:
860 nd = measAlg.Defect(bbox)
861 defectList.append(nd)
862 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
863 isrFunctions.interpolateDefectList(
864 maskedImage=maskedImage,
865 defectList=defectList,
866 fwhm=self.config.fwhm,
870 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 872 We mask and interpolate over all NaNs, including those 873 that are masked with other bits (because those may or may 874 not be interpolated over later, and we want to remove all 875 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 876 is used to preserve the historical name. 878 @param[in,out] exposure exposure to process 880 maskedImage = exposure.getMaskedImage()
883 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
884 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
885 numNans =
maskNans(maskedImage, maskVal)
886 self.metadata.set(
"NUMNANS", numNans)
890 self.log.warn(
"There were %i unmasked NaNs", numNans)
891 nanDefectList = isrFunctions.getDefectListFromMask(
892 maskedImage=maskedImage,
893 maskName=
'UNMASKEDNAN',
895 isrFunctions.interpolateDefectList(
896 maskedImage=exposure.getMaskedImage(),
897 defectList=nanDefectList,
898 fwhm=self.config.fwhm,
902 """Apply overscan correction, in-place 906 exposure : `lsst.afw.image.Exposure` 907 Exposure to process; must include both data and bias regions. 908 amp : `lsst.afw.table.AmpInfoRecord` 909 Amplifier device data. 913 result : `lsst.pipe.base.Struct` or `NoneType` 914 `None` if there is no overscan; otherwise, this is a 915 result struct with components: 917 - ``imageFit``: Value(s) removed from image (scalar or 918 `lsst.afw.image.Image`). 919 - ``overscanFit``: Value(s) removed from overscan (scalar or 920 `lsst.afw.image.Image`). 921 - ``overscanImage``: Image of the overscan, post-subtraction 922 (`lsst.afw.image.Image`). 924 if not amp.getHasRawInfo():
925 raise RuntimeError(
"This method must be executed on an amp with raw information.")
927 if amp.getRawHorizontalOverscanBBox().isEmpty():
928 self.log.info(
"No Overscan region. Not performing Overscan Correction.")
931 maskedImage = exposure.getMaskedImage()
932 dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
933 overscanImage = maskedImage.Factory(maskedImage, amp.getRawHorizontalOverscanBBox())
935 results = isrFunctions.overscanCorrection(
936 ampMaskedImage=dataView,
937 overscanImage=overscanImage,
938 fitType=self.config.overscanFitType,
939 order=self.config.overscanOrder,
940 collapseRej=self.config.overscanRej,
942 results.overscanImage = overscanImage
946 """!Update the WCS in exposure with a distortion model based on camera geometry 948 Add a model for optical distortion based on geometry found in `camera` 949 and the `exposure`'s detector. The raw input exposure is assumed 950 have a TAN WCS that has no compensation for optical distortion. 951 Two other possibilities are: 952 - The raw input exposure already has a model for optical distortion, 953 as is the case for raw DECam data. 954 In that case you should set config.doAddDistortionModel False. 955 - The raw input exposure has a model for distortion, but it has known 956 deficiencies severe enough to be worth fixing (e.g. because they 957 cause problems for fitting a better WCS). In that case you should 958 override this method with a version suitable for your raw data. 960 @param[in,out] exposure exposure to process; must include a Detector and a WCS; 961 the WCS of the exposure is modified in place 962 @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 964 self.log.info(
"Adding a distortion model to the WCS")
965 wcs = exposure.getWcs()
967 raise RuntimeError(
"exposure has no WCS")
969 raise RuntimeError(
"camera is None")
970 detector = exposure.getDetector()
972 raise RuntimeError(
"exposure has no Detector")
973 pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
974 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
975 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
976 exposure.setWcs(distortedWcs)
979 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners 981 @param[in,out] ccdExposure exposure to process 982 @param[in] fpPolygon Polygon in focal plane coordinates 985 ccd = ccdExposure.getDetector()
986 fpCorners = ccd.getCorners(FOCAL_PLANE)
987 ccdPolygon = Polygon(fpCorners)
990 intersect = ccdPolygon.intersectionSingle(fpPolygon)
993 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
994 validPolygon = Polygon(ccdPoints)
995 ccdExposure.getInfo().setValidPolygon(validPolygon)
998 """Apply brighter fatter correction in place for the image 1000 This correction takes a kernel that has been derived from flat field images to 1001 redistribute the charge. The gradient of the kernel is the deflection 1002 field due to the accumulated charge. 1004 Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 1005 using the following equation: 1007 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 1009 To evaluate the derivative term we expand it as follows: 1011 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 1013 Because we use the measured counts instead of the incident counts we apply the correction 1014 iteratively to reconstruct the original counts and the correction. We stop iterating when the 1015 summed difference between the current corrected image and the one from the previous iteration 1016 is below the threshold. We do not require convergence because the number of iterations is 1017 too large a computational cost. How we define the threshold still needs to be evaluated, the 1018 current default was shown to work reasonably well on a small set of images. For more information 1019 on the method see DocuShare Document-19407. 1021 The edges as defined by the kernel are not corrected because they have spurious values 1022 due to the convolution. 1024 self.log.info(
"Applying brighter fatter correction")
1026 image = exposure.getMaskedImage().getImage()
1029 with self.
gainContext(exposure, image, applyGain):
1031 kLx = numpy.shape(kernel)[0]
1032 kLy = numpy.shape(kernel)[1]
1033 kernelImage = afwImage.ImageD(kLx, kLy)
1034 kernelImage.getArray()[:, :] = kernel
1035 tempImage = image.clone()
1037 nanIndex = numpy.isnan(tempImage.getArray())
1038 tempImage.getArray()[nanIndex] = 0.
1040 outImage = afwImage.ImageF(image.getDimensions())
1041 corr = numpy.zeros_like(image.getArray())
1042 prev_image = numpy.zeros_like(image.getArray())
1043 convCntrl = afwMath.ConvolutionControl(
False,
True, 1)
1044 fixedKernel = afwMath.FixedKernel(kernelImage)
1054 for iteration
in range(maxIter):
1056 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1057 tmpArray = tempImage.getArray()
1058 outArray = outImage.getArray()
1060 with numpy.errstate(invalid=
"ignore", over=
"ignore"):
1062 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1063 gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1064 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1067 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1068 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1069 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1071 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1073 tmpArray[:, :] = image.getArray()[:, :]
1074 tmpArray[nanIndex] = 0.
1075 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1078 diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1080 if diff < threshold:
1082 prev_image[:, :] = tmpArray[:, :]
1084 if iteration == maxIter - 1:
1085 self.log.warn(
"Brighter fatter correction did not converge, final difference %f" % diff)
1087 self.log.info(
"Finished brighter fatter in %d iterations" % (iteration + 1))
1088 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1089 corr[startY + 1:endY - 1, startX + 1:endX - 1]
1092 sensorTransmission=None, atmosphereTransmission=None):
1093 """Attach a TransmissionCurve to an Exposure, given separate curves for 1094 different components. 1098 exposure : `lsst.afw.image.Exposure` 1099 Exposure object to modify by attaching the product of all given 1100 ``TransmissionCurves`` in post-assembly trimmed detector 1101 coordinates. Must have a valid ``Detector`` attached that matches 1102 the detector associated with sensorTransmission. 1103 opticsTransmission : `lsst.afw.image.TransmissionCurve` 1104 A ``TransmissionCurve`` that represents the throughput of the 1105 optics, to be evaluated in focal-plane coordinates. 1106 filterTransmission : `lsst.afw.image.TransmissionCurve` 1107 A ``TransmissionCurve`` that represents the throughput of the 1108 filter itself, to be evaluated in focal-plane coordinates. 1109 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1110 A ``TransmissionCurve`` that represents the throughput of the 1111 sensor itself, to be evaluated in post-assembly trimmed detector 1113 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1114 A ``TransmissionCurve`` that represents the throughput of the 1115 atmosphere, assumed to be spatially constant. 1117 All ``TransmissionCurve`` arguments are optional; if none are provided, 1118 the attached ``TransmissionCurve`` will have unit transmission 1123 combined : ``lsst.afw.image.TransmissionCurve`` 1124 The TransmissionCurve attached to the exposure. 1126 return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1127 filterTransmission=filterTransmission,
1128 sensorTransmission=sensorTransmission,
1129 atmosphereTransmission=atmosphereTransmission)
1133 """Context manager that applies and removes gain 1136 ccd = exp.getDetector()
1138 sim = image.Factory(image, amp.getBBox())
1139 sim *= amp.getGain()
1145 ccd = exp.getDetector()
1147 sim = image.Factory(image, amp.getBBox())
1148 sim /= amp.getGain()
1152 """Context manager that applies and removes flats and darks, 1153 if the task is configured to apply them. 1155 if self.config.doDark
and dark
is not None:
1157 if self.config.doFlat:
1162 if self.config.doFlat:
1164 if self.config.doDark
and dark
is not None:
1169 """A Detector-like object that supports returning gain and saturation level""" 1172 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1174 self.
_gain = config.gain
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
def runDataRef(self, sensorRef)
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
def gainContext(self, exp, image, apply)
def __init__(self, args, kwargs)
Constructor for IsrTask.
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Apply common instrument signature correction algorithms to a raw frame.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Is linearization wanted for this detector?
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
def flatContext(self, exp, flat, dark=None)
def overscanCorrection(self, exposure, amp)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def updateVariance(self, ampExposure, amp, overscanImage=None)
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)