1 from __future__
import absolute_import, division, print_function
2 from builtins
import range
3 from builtins
import object
35 from lsstDebug
import getDebugFrame
37 from .
import isrFunctions
38 from .assembleCcdTask
import AssembleCcdTask
39 from .fringe
import FringeTask
43 from contextlib
import contextmanager
44 from .isr
import maskNans
45 from .crosstalk
import CrosstalkTask
49 doBias = pexConfig.Field(
51 doc=
"Apply bias frame correction?",
54 doDark = pexConfig.Field(
56 doc=
"Apply dark frame correction?",
59 doFlat = pexConfig.Field(
61 doc=
"Apply flat field correction?",
64 doFringe = pexConfig.Field(
66 doc=
"Apply fringe correction?",
69 doDefect = pexConfig.Field(
71 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
74 doAddDistortionModel = pexConfig.Field(
76 doc=
"Apply a distortion model based on camera geometry to the WCS?",
79 doWrite = pexConfig.Field(
81 doc=
"Persist postISRCCD?",
84 biasDataProductName = pexConfig.Field(
86 doc=
"Name of the bias data product",
89 darkDataProductName = pexConfig.Field(
91 doc=
"Name of the dark data product",
94 flatDataProductName = pexConfig.Field(
96 doc=
"Name of the flat data product",
99 assembleCcd = pexConfig.ConfigurableField(
100 target=AssembleCcdTask,
101 doc=
"CCD assembly task",
103 gain = pexConfig.Field(
105 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
106 default=float(
"NaN"),
108 readNoise = pexConfig.Field(
110 doc=
"The read noise to use if no Detector is present in the Exposure",
113 saturation = pexConfig.Field(
115 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
116 default=float(
"NaN"),
118 fringeAfterFlat = pexConfig.Field(
120 doc=
"Do fringe subtraction after flat-fielding?",
123 fringe = pexConfig.ConfigurableField(
125 doc=
"Fringe subtraction task",
127 fwhm = pexConfig.Field(
129 doc=
"FWHM of PSF (arcsec)",
132 saturatedMaskName = pexConfig.Field(
134 doc=
"Name of mask plane to use in saturation detection and interpolation",
137 suspectMaskName = pexConfig.Field(
139 doc=
"Name of mask plane to use for suspect pixels",
142 flatScalingType = pexConfig.ChoiceField(
144 doc=
"The method for scaling the flat on the fly.",
147 "USER":
"Scale by flatUserScale",
148 "MEAN":
"Scale by the inverse of the mean",
149 "MEDIAN":
"Scale by the inverse of the median",
152 flatUserScale = pexConfig.Field(
154 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
157 overscanFitType = pexConfig.ChoiceField(
159 doc=
"The method for fitting the overscan bias level.",
162 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
163 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
164 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
165 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
166 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
167 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
168 "MEAN":
"Correct using the mean of the overscan region",
169 "MEDIAN":
"Correct using the median of the overscan region",
172 overscanOrder = pexConfig.Field(
174 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
175 "or number of spline knots if overscan fit type is a spline."),
178 overscanRej = pexConfig.Field(
180 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
183 growSaturationFootprintSize = pexConfig.Field(
185 doc=
"Number of pixels by which to grow the saturation footprints",
188 doSaturationInterpolation = pexConfig.Field(
190 doc=
"Perform interpolation over pixels masked as saturated?",
193 doNanInterpAfterFlat = pexConfig.Field(
195 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 196 "also have to interpolate them before flat-fielding."),
199 fluxMag0T1 = pexConfig.Field(
201 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure",
204 keysToRemoveFromAssembledCcd = pexConfig.ListField(
206 doc=
"fields to remove from the metadata of the assembled ccd.",
209 doAssembleIsrExposures = pexConfig.Field(
212 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 214 doAssembleCcd = pexConfig.Field(
217 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 219 expectWcs = pexConfig.Field(
222 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)" 224 doLinearize = pexConfig.Field(
226 doc=
"Correct for nonlinearity of the detector's response?",
229 doCrosstalk = pexConfig.Field(
231 doc=
"Apply intra-CCD crosstalk correction?",
234 crosstalk = pexConfig.ConfigurableField(
235 target=CrosstalkTask,
236 doc=
"Intra-CCD crosstalk correction",
238 doBrighterFatter = pexConfig.Field(
241 doc=
"Apply the brighter fatter correction" 243 brighterFatterKernelFile = pexConfig.Field(
246 doc=
"Kernel file used for the brighter fatter correction" 248 brighterFatterMaxIter = pexConfig.Field(
251 doc=
"Maximum number of iterations for the brighter fatter correction" 253 brighterFatterThreshold = pexConfig.Field(
256 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 257 " absolute value of the difference between the current corrected image and the one" 258 " from the previous iteration summed over all the pixels." 260 brighterFatterApplyGain = pexConfig.Field(
263 doc=
"Should the gain be applied when applying the brighter fatter correction?" 265 datasetType = pexConfig.Field(
267 doc=
"Dataset type for input data; users will typically leave this alone, " 268 "but camera-specific ISR tasks will override it",
271 fallbackFilterName = pexConfig.Field(dtype=str,
272 doc=
"Fallback default filter name for calibrations", optional=
True)
273 doAttachTransmissionCurve = pexConfig.Field(
276 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 278 doUseOpticsTransmission = pexConfig.Field(
281 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 283 doUseFilterTransmission = pexConfig.Field(
286 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 288 doUseSensorTransmission = pexConfig.Field(
291 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 293 doUseAtmosphereTransmission = pexConfig.Field(
296 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 311 @brief Apply common instrument signature correction algorithms to a raw frame. 313 @section ip_isr_isr_Contents Contents 315 - @ref ip_isr_isr_Purpose 316 - @ref ip_isr_isr_Initialize 318 - @ref ip_isr_isr_Config 319 - @ref ip_isr_isr_Debug 322 @section ip_isr_isr_Purpose Description 324 The process for correcting imaging data is very similar from camera to camera. 325 This task provides a vanilla implementation of doing these corrections, including 326 the ability to turn certain corrections off if they are not needed. 327 The inputs to the primary method, run, are a raw exposure to be corrected and the 328 calibration data products. The raw input is a single chip sized mosaic of all amps 329 including overscans and other non-science pixels. 330 The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 331 and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 332 This task may not meet all needs and it is expected that it will be subclassed for 333 specific applications. 335 @section ip_isr_isr_Initialize Task initialization 337 @copydoc \_\_init\_\_ 339 @section ip_isr_isr_IO Inputs/Outputs to the run method 343 @section ip_isr_isr_Config Configuration parameters 345 See @ref IsrTaskConfig 347 @section ip_isr_isr_Debug Debug variables 349 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 350 flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 351 href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 352 Using lsstDebug to control debugging output</a> for more about @b debug.py files. 354 The available variables in IsrTask are: 357 <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 360 <DD> display exposure after ISR has been applied 364 For example, put something like 368 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 369 if name == "lsst.ip.isrFunctions.isrTask": 370 di.display = {'postISRCCD':2} 372 lsstDebug.Info = DebugInfo 374 into your debug.py file and run the commandline task with the @c --debug flag. 378 ConfigClass = IsrTaskConfig
382 '''!Constructor for IsrTask 383 @param[in] *args a list of positional arguments passed on to the Task constructor 384 @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 385 Call the lsst.pipe.base.task.Task.__init__ method 386 Then setup the assembly and fringe correction subtasks 388 pipeBase.Task.__init__(self, *args, **kwargs)
389 self.makeSubtask(
"assembleCcd")
390 self.makeSubtask(
"fringe")
391 self.makeSubtask(
"crosstalk")
394 """!Retrieve necessary frames for instrument signature removal 395 @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 396 of the detector data to be processed 397 @param[in] rawExposure a reference raw exposure that will later be 398 corrected with the retrieved calibration data; 399 should not be modified in this method. 400 @return a pipeBase.Struct with fields containing kwargs expected by run() 401 - bias: exposure of bias frame 402 - dark: exposure of dark frame 403 - flat: exposure of flat field 404 - defects: list of detects 405 - fringeStruct: a pipeBase.Struct with field fringes containing 406 exposure of fringe frame or list of fringe exposure 408 ccd = rawExposure.getDetector()
410 biasExposure = self.
getIsrExposure(dataRef, self.config.biasDataProductName) \
411 if self.config.doBias
else None 413 linearizer = dataRef.get(
"linearizer", immediate=
True)
if self.
doLinearize(ccd)
else None 414 darkExposure = self.
getIsrExposure(dataRef, self.config.darkDataProductName) \
415 if self.config.doDark
else None 416 flatExposure = self.
getIsrExposure(dataRef, self.config.flatDataProductName) \
417 if self.config.doFlat
else None 418 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
if self.config.doBrighterFatter
else None 419 defectList = dataRef.get(
"defects")
if self.config.doDefect
else None 421 if self.config.doCrosstalk:
422 crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
424 crosstalkSources =
None 426 if self.config.doFringe
and self.fringe.checkFilter(rawExposure):
427 fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
428 if self.config.doAssembleIsrExposures
else None)
430 fringeStruct = pipeBase.Struct(fringes=
None)
432 if self.config.doAttachTransmissionCurve:
433 opticsTransmission = (dataRef.get(
"transmission_optics")
434 if self.config.doUseOpticsTransmission
else None)
435 filterTransmission = (dataRef.get(
"transmission_filter")
436 if self.config.doUseFilterTransmission
else None)
437 sensorTransmission = (dataRef.get(
"transmission_sensor")
438 if self.config.doUseSensorTransmission
else None)
439 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
440 if self.config.doUseAtmosphereTransmission
else None)
442 opticsTransmission =
None 443 filterTransmission =
None 444 sensorTransmission =
None 445 atmosphereTransmission =
None 448 return pipeBase.Struct(bias=biasExposure,
449 linearizer=linearizer,
453 fringes=fringeStruct,
454 bfKernel=brighterFatterKernel,
455 opticsTransmission=opticsTransmission,
456 filterTransmission=filterTransmission,
457 sensorTransmission=sensorTransmission,
458 atmosphereTransmission=atmosphereTransmission,
459 crosstalkSources=crosstalkSources,
463 def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
464 fringes=None, bfKernel=None, camera=None,
465 opticsTransmission=None, filterTransmission=None,
466 sensorTransmission=None, atmosphereTransmission=None,
467 crosstalkSources=None):
468 """!Perform instrument signature removal on an exposure 471 - Detect saturation, apply overscan correction, bias, dark and flat 472 - Perform CCD assembly 473 - Interpolate over defects, saturated pixels and all NaNs 475 @param[in] ccdExposure lsst.afw.image.exposure of detector data 476 @param[in] bias exposure of bias frame 477 @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 478 @param[in] dark exposure of dark frame 479 @param[in] flat exposure of flatfield 480 @param[in] defects list of detects 481 @param[in] fringes a pipeBase.Struct with field fringes containing 482 exposure of fringe frame or list of fringe exposure 483 @param[in] bfKernel kernel for brighter-fatter correction 484 @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 485 used by addDistortionModel 486 @param[in] opticsTransmission a TransmissionCurve for the optics 487 @param[in] filterTransmission a TransmissionCurve for the filter 488 @param[in] sensorTransmission a TransmissionCurve for the sensor 489 @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 490 @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk 492 @return a pipeBase.Struct with field: 496 if isinstance(ccdExposure, ButlerDataRef):
499 ccd = ccdExposure.getDetector()
502 if self.config.doBias
and bias
is None:
503 raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
505 raise RuntimeError(
"Must supply a linearizer if config.doBias True")
506 if self.config.doDark
and dark
is None:
507 raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
508 if self.config.doFlat
and flat
is None:
509 raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
510 if self.config.doBrighterFatter
and bfKernel
is None:
511 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter True")
513 fringes = pipeBase.Struct(fringes=
None)
514 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
515 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
516 if self.config.doDefect
and defects
is None:
517 raise RuntimeError(
"Must supply defects if config.doDefect True")
518 if self.config.doAddDistortionModel
and camera
is None:
519 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel True")
524 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 525 ccd = [
FakeAmp(ccdExposure, self.config)]
529 if ccdExposure.getBBox().contains(amp.getBBox()):
534 if self.config.doCrosstalk:
535 self.crosstalk.
run(ccdExposure, crosstalkSources)
537 if self.config.doAssembleCcd:
538 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
539 if self.config.expectWcs
and not ccdExposure.getWcs():
540 self.log.warn(
"No WCS found in input exposure")
542 if self.config.doBias:
546 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
550 if ccdExposure.getBBox().contains(amp.getBBox()):
551 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
554 interpolationDone =
False 556 if self.config.doBrighterFatter:
563 if self.config.doDefect:
565 if self.config.doSaturationInterpolation:
568 interpolationDone =
True 571 self.config.brighterFatterMaxIter,
572 self.config.brighterFatterThreshold,
573 self.config.brighterFatterApplyGain,
576 if self.config.doDark:
579 if self.config.doFringe
and not self.config.fringeAfterFlat:
580 self.fringe.
run(ccdExposure, **fringes.getDict())
582 if self.config.doFlat:
585 if not interpolationDone:
586 if self.config.doDefect:
588 if self.config.doSaturationInterpolation:
590 if not interpolationDone
or self.config.doNanInterpAfterFlat:
593 if self.config.doFringe
and self.config.fringeAfterFlat:
594 self.fringe.
run(ccdExposure, **fringes.getDict())
596 exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
597 ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
599 if self.config.doAddDistortionModel:
602 if self.config.doAttachTransmissionCurve:
604 filterTransmission=filterTransmission,
605 sensorTransmission=sensorTransmission,
606 atmosphereTransmission=atmosphereTransmission)
608 frame = getDebugFrame(self._display,
"postISRCCD")
610 getDisplay(frame).mtv(ccdExposure)
612 return pipeBase.Struct(
613 exposure=ccdExposure,
618 """Perform instrument signature removal on a ButlerDataRef of a Sensor 620 - Read in necessary detrending/isr/calibration data 621 - Process raw exposure in run() 622 - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 626 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 627 DataRef of the detector data to be processed 631 result : `pipeBase.Struct` 632 Struct contains field "exposure," which is the exposure after application of ISR 634 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
635 ccdExposure = sensorRef.get(
'raw')
636 camera = sensorRef.get(
"camera")
637 if camera
is None and self.config.doAddDistortionModel:
638 raise RuntimeError(
"config.doAddDistortionModel is True " 639 "but could not get a camera from the butler")
642 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
644 if self.config.doWrite:
645 sensorRef.put(result.exposure,
"postISRCCD")
650 """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 652 if isinstance(exposure, afwImage.ExposureF):
655 if not hasattr(exposure,
"convertF"):
656 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
658 newexposure = exposure.convertF()
659 maskedImage = newexposure.getMaskedImage()
660 varArray = maskedImage.getVariance().getArray()
662 maskArray = maskedImage.getMask().getArray()
667 """!Apply bias correction in place 669 @param[in,out] exposure exposure to process 670 @param[in] biasExposure bias exposure of same size as exposure 672 isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
675 """!Apply dark correction in place 677 @param[in,out] exposure exposure to process 678 @param[in] darkExposure dark exposure of same size as exposure 679 @param[in] invert if True, remove the dark from an already-corrected image 681 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
682 if math.isnan(expScale):
683 raise RuntimeError(
"Exposure darktime is NAN")
684 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
685 if math.isnan(darkScale):
686 raise RuntimeError(
"Dark calib darktime is NAN")
687 isrFunctions.darkCorrection(
688 maskedImage=exposure.getMaskedImage(),
689 darkMaskedImage=darkExposure.getMaskedImage(),
696 """!Is linearization wanted for this detector? 698 Checks config.doLinearize and the linearity type of the first amplifier. 700 @param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 702 return self.config.doLinearize
and \
703 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
706 """!Set the variance plane based on the image plane, plus amplifier gain and read noise 708 @param[in,out] ampExposure exposure to process 709 @param[in] amp amplifier detector information 712 if not math.isnan(gain):
715 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
716 (amp.getName(), gain, patchedGain))
719 isrFunctions.updateVariance(
720 maskedImage=ampExposure.getMaskedImage(),
722 readNoise=amp.getReadNoise(),
726 """!Apply flat correction in place 728 @param[in,out] exposure exposure to process 729 @param[in] flatExposure flatfield exposure same size as exposure 730 @param[in] invert if True, unflatten an already-flattened image instead. 732 isrFunctions.flatCorrection(
733 maskedImage=exposure.getMaskedImage(),
734 flatMaskedImage=flatExposure.getMaskedImage(),
735 scalingType=self.config.flatScalingType,
736 userScale=self.config.flatUserScale,
741 """!Retrieve a calibration dataset for removing instrument signature 743 @param[in] dataRef data reference for exposure 744 @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 745 @param[in] immediate if True, disable butler proxies to enable error 746 handling within this routine 750 exp = dataRef.get(datasetType, immediate=immediate)
751 except Exception
as exc1:
752 if not self.config.fallbackFilterName:
753 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
755 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
756 except Exception
as exc2:
757 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
758 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
759 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
761 if self.config.doAssembleIsrExposures:
762 exp = self.assembleCcd.assembleCcd(exp)
766 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 768 @param[in,out] exposure exposure to process; only the amp DataSec is processed 769 @param[in] amp amplifier device data 771 if not math.isnan(amp.getSaturation()):
772 maskedImage = exposure.getMaskedImage()
773 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
774 isrFunctions.makeThresholdMask(
775 maskedImage=dataView,
776 threshold=amp.getSaturation(),
778 maskName=self.config.saturatedMaskName,
782 """!Interpolate over saturated pixels, in place 784 @param[in,out] ccdExposure exposure to process 787 - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 788 - Call this after CCD assembly, since saturated regions may cross amplifier boundaries 790 isrFunctions.interpolateFromMask(
791 maskedImage=ccdExposure.getMaskedImage(),
792 fwhm=self.config.fwhm,
793 growFootprints=self.config.growSaturationFootprintSize,
794 maskName=self.config.saturatedMaskName,
798 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 800 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 801 This is intended to indicate pixels that may be affected by unknown systematics; 802 for example if non-linearity corrections above a certain level are unstable 803 then that would be a useful value for suspectLevel. A value of `nan` indicates 804 that no such level exists and no pixels are to be masked as suspicious. 806 @param[in,out] exposure exposure to process; only the amp DataSec is processed 807 @param[in] amp amplifier device data 809 suspectLevel = amp.getSuspectLevel()
810 if math.isnan(suspectLevel):
813 maskedImage = exposure.getMaskedImage()
814 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
815 isrFunctions.makeThresholdMask(
816 maskedImage=dataView,
817 threshold=suspectLevel,
819 maskName=self.config.suspectMaskName,
823 """!Mask defects using mask plane "BAD" and interpolate over them, in place 825 @param[in,out] ccdExposure exposure to process 826 @param[in] defectBaseList a list of defects to mask and interpolate 828 @warning: call this after CCD assembly, since defects may cross amplifier boundaries 830 maskedImage = ccdExposure.getMaskedImage()
832 for d
in defectBaseList:
834 nd = measAlg.Defect(bbox)
835 defectList.append(nd)
836 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
837 isrFunctions.interpolateDefectList(
838 maskedImage=maskedImage,
839 defectList=defectList,
840 fwhm=self.config.fwhm,
844 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 846 We mask and interpolate over all NaNs, including those 847 that are masked with other bits (because those may or may 848 not be interpolated over later, and we want to remove all 849 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 850 is used to preserve the historical name. 852 @param[in,out] exposure exposure to process 854 maskedImage = exposure.getMaskedImage()
857 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
858 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
859 numNans =
maskNans(maskedImage, maskVal)
860 self.metadata.set(
"NUMNANS", numNans)
864 self.log.warn(
"There were %i unmasked NaNs", numNans)
865 nanDefectList = isrFunctions.getDefectListFromMask(
866 maskedImage=maskedImage,
867 maskName=
'UNMASKEDNAN',
869 isrFunctions.interpolateDefectList(
870 maskedImage=exposure.getMaskedImage(),
871 defectList=nanDefectList,
872 fwhm=self.config.fwhm,
876 """!Apply overscan correction, in place 878 @param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels 879 @param[in] amp amplifier device data 881 if not amp.getHasRawInfo():
882 raise RuntimeError(
"This method must be executed on an amp with raw information.")
884 if amp.getRawHorizontalOverscanBBox().isEmpty():
885 self.log.info(
"No Overscan region. Not performing Overscan Correction.")
888 maskedImage = exposure.getMaskedImage()
889 dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
891 expImage = exposure.getMaskedImage().getImage()
892 overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
894 isrFunctions.overscanCorrection(
895 ampMaskedImage=dataView,
896 overscanImage=overscanImage,
897 fitType=self.config.overscanFitType,
898 order=self.config.overscanOrder,
899 collapseRej=self.config.overscanRej,
903 """!Update the WCS in exposure with a distortion model based on camera geometry 905 Add a model for optical distortion based on geometry found in `camera` 906 and the `exposure`'s detector. The raw input exposure is assumed 907 have a TAN WCS that has no compensation for optical distortion. 908 Two other possibilities are: 909 - The raw input exposure already has a model for optical distortion, 910 as is the case for raw DECam data. 911 In that case you should set config.doAddDistortionModel False. 912 - The raw input exposure has a model for distortion, but it has known 913 deficiencies severe enough to be worth fixing (e.g. because they 914 cause problems for fitting a better WCS). In that case you should 915 override this method with a version suitable for your raw data. 917 @param[in,out] exposure exposure to process; must include a Detector and a WCS; 918 the WCS of the exposure is modified in place 919 @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 921 self.log.info(
"Adding a distortion model to the WCS")
922 wcs = exposure.getWcs()
924 raise RuntimeError(
"exposure has no WCS")
926 raise RuntimeError(
"camera is None")
927 detector = exposure.getDetector()
929 raise RuntimeError(
"exposure has no Detector")
930 pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
931 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
932 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
933 exposure.setWcs(distortedWcs)
936 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners 938 @param[in,out] ccdExposure exposure to process 939 @param[in] fpPolygon Polygon in focal plane coordinates 942 ccd = ccdExposure.getDetector()
943 fpCorners = ccd.getCorners(FOCAL_PLANE)
944 ccdPolygon = Polygon(fpCorners)
947 intersect = ccdPolygon.intersectionSingle(fpPolygon)
950 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
951 validPolygon = Polygon(ccdPoints)
952 ccdExposure.getInfo().setValidPolygon(validPolygon)
955 """Apply brighter fatter correction in place for the image 957 This correction takes a kernel that has been derived from flat field images to 958 redistribute the charge. The gradient of the kernel is the deflection 959 field due to the accumulated charge. 961 Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 962 using the following equation: 964 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 966 To evaluate the derivative term we expand it as follows: 968 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 970 Because we use the measured counts instead of the incident counts we apply the correction 971 iteratively to reconstruct the original counts and the correction. We stop iterating when the 972 summed difference between the current corrected image and the one from the previous iteration 973 is below the threshold. We do not require convergence because the number of iterations is 974 too large a computational cost. How we define the threshold still needs to be evaluated, the 975 current default was shown to work reasonably well on a small set of images. For more information 976 on the method see DocuShare Document-19407. 978 The edges as defined by the kernel are not corrected because they have spurious values 979 due to the convolution. 981 self.log.info(
"Applying brighter fatter correction")
983 image = exposure.getMaskedImage().getImage()
988 kLx = numpy.shape(kernel)[0]
989 kLy = numpy.shape(kernel)[1]
990 kernelImage = afwImage.ImageD(kLx, kLy)
991 kernelImage.getArray()[:, :] = kernel
992 tempImage = image.clone()
994 nanIndex = numpy.isnan(tempImage.getArray())
995 tempImage.getArray()[nanIndex] = 0.
997 outImage = afwImage.ImageF(image.getDimensions())
998 corr = numpy.zeros_like(image.getArray())
999 prev_image = numpy.zeros_like(image.getArray())
1000 convCntrl = afwMath.ConvolutionControl(
False,
True, 1)
1001 fixedKernel = afwMath.FixedKernel(kernelImage)
1011 for iteration
in range(maxIter):
1013 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1014 tmpArray = tempImage.getArray()
1015 outArray = outImage.getArray()
1017 with numpy.errstate(invalid=
"ignore", over=
"ignore"):
1019 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1020 gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1021 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1024 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1025 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1026 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1028 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1030 tmpArray[:, :] = image.getArray()[:, :]
1031 tmpArray[nanIndex] = 0.
1032 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1035 diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1037 if diff < threshold:
1039 prev_image[:, :] = tmpArray[:, :]
1041 if iteration == maxIter - 1:
1042 self.log.warn(
"Brighter fatter correction did not converge, final difference %f" % diff)
1044 self.log.info(
"Finished brighter fatter in %d iterations" % (iteration + 1))
1045 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1046 corr[startY + 1:endY - 1, startX + 1:endX - 1]
1049 sensorTransmission=None, atmosphereTransmission=None):
1050 """Attach a TransmissionCurve to an Exposure, given separate curves for 1051 different components. 1055 exposure : `lsst.afw.image.Exposure` 1056 Exposure object to modify by attaching the product of all given 1057 ``TransmissionCurves`` in post-assembly trimmed detector 1058 coordinates. Must have a valid ``Detector`` attached that matches 1059 the detector associated with sensorTransmission. 1060 opticsTransmission : `lsst.afw.image.TransmissionCurve` 1061 A ``TransmissionCurve`` that represents the throughput of the 1062 optics, to be evaluated in focal-plane coordinates. 1063 filterTransmission : `lsst.afw.image.TransmissionCurve` 1064 A ``TransmissionCurve`` that represents the throughput of the 1065 filter itself, to be evaluated in focal-plane coordinates. 1066 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1067 A ``TransmissionCurve`` that represents the throughput of the 1068 sensor itself, to be evaluated in post-assembly trimmed detector 1070 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1071 A ``TransmissionCurve`` that represents the throughput of the 1072 atmosphere, assumed to be spatially constant. 1074 All ``TransmissionCurve`` arguments are optional; if none are provided, 1075 the attached ``TransmissionCurve`` will have unit transmission 1080 combined : ``lsst.afw.image.TransmissionCurve`` 1081 The TransmissionCurve attached to the exposure. 1083 return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1084 filterTransmission=filterTransmission,
1085 sensorTransmission=sensorTransmission,
1086 atmosphereTransmission=atmosphereTransmission)
1090 """Context manager that applies and removes gain 1093 ccd = exp.getDetector()
1095 sim = image.Factory(image, amp.getBBox())
1096 sim *= amp.getGain()
1102 ccd = exp.getDetector()
1104 sim = image.Factory(image, amp.getBBox())
1105 sim /= amp.getGain()
1109 """Context manager that applies and removes flats and darks, 1110 if the task is configured to apply them. 1112 if self.config.doDark
and dark
is not None:
1114 if self.config.doFlat:
1119 if self.config.doFlat:
1121 if self.config.doDark
and dark
is not None:
1126 """A Detector-like object that supports returning gain and saturation level""" 1129 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1131 self.
_gain = config.gain
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
def runDataRef(self, sensorRef)
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
def gainContext(self, exp, image, apply)
def __init__(self, args, kwargs)
Constructor for IsrTask.
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Apply common instrument signature correction algorithms to a raw frame.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Is linearization wanted for this detector?
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
def flatContext(self, exp, flat, dark=None)
def updateVariance(self, ampExposure, amp)
Set the variance plane based on the image plane, plus amplifier gain and read noise.
def overscanCorrection(self, exposure, amp)
Apply overscan correction, in place.
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)