1 from __future__
import absolute_import, division, print_function
2 from builtins
import range
3 from builtins
import object
35 from lsstDebug
import getDebugFrame
37 from .
import isrFunctions
38 from .assembleCcdTask
import AssembleCcdTask
39 from .fringe
import FringeTask
43 from contextlib
import contextmanager
44 from .isr
import maskNans
45 from .crosstalk
import CrosstalkTask
49 doBias = pexConfig.Field(
51 doc=
"Apply bias frame correction?",
54 doDark = pexConfig.Field(
56 doc=
"Apply dark frame correction?",
59 doFlat = pexConfig.Field(
61 doc=
"Apply flat field correction?",
64 doFringe = pexConfig.Field(
66 doc=
"Apply fringe correction?",
69 doDefect = pexConfig.Field(
71 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
74 doAddDistortionModel = pexConfig.Field(
76 doc=
"Apply a distortion model based on camera geometry to the WCS?",
79 doWrite = pexConfig.Field(
81 doc=
"Persist postISRCCD?",
84 assembleCcd = pexConfig.ConfigurableField(
85 target=AssembleCcdTask,
86 doc=
"CCD assembly task",
88 gain = pexConfig.Field(
90 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
93 readNoise = pexConfig.Field(
95 doc=
"The read noise to use if no Detector is present in the Exposure",
98 saturation = pexConfig.Field(
100 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
101 default=float(
"NaN"),
103 fringeAfterFlat = pexConfig.Field(
105 doc=
"Do fringe subtraction after flat-fielding?",
108 fringe = pexConfig.ConfigurableField(
110 doc=
"Fringe subtraction task",
112 fwhm = pexConfig.Field(
114 doc=
"FWHM of PSF (arcsec)",
117 saturatedMaskName = pexConfig.Field(
119 doc=
"Name of mask plane to use in saturation detection and interpolation",
122 suspectMaskName = pexConfig.Field(
124 doc=
"Name of mask plane to use for suspect pixels",
127 flatScalingType = pexConfig.ChoiceField(
129 doc=
"The method for scaling the flat on the fly.",
132 "USER":
"Scale by flatUserScale",
133 "MEAN":
"Scale by the inverse of the mean",
134 "MEDIAN":
"Scale by the inverse of the median",
137 flatUserScale = pexConfig.Field(
139 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
142 overscanFitType = pexConfig.ChoiceField(
144 doc=
"The method for fitting the overscan bias level.",
147 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
148 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
149 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
150 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
151 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
152 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
153 "MEAN":
"Correct using the mean of the overscan region",
154 "MEDIAN":
"Correct using the median of the overscan region",
157 overscanOrder = pexConfig.Field(
159 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, " +
160 "or number of spline knots if overscan fit type is a spline."),
163 overscanRej = pexConfig.Field(
165 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
168 growSaturationFootprintSize = pexConfig.Field(
170 doc=
"Number of pixels by which to grow the saturation footprints",
173 doSaturationInterpolation = pexConfig.Field(
175 doc=
"Perform interpolation over pixels masked as saturated?",
178 doNanInterpAfterFlat = pexConfig.Field(
180 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we " 181 "also have to interpolate them before flat-fielding."),
184 fluxMag0T1 = pexConfig.Field(
186 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure",
189 keysToRemoveFromAssembledCcd = pexConfig.ListField(
191 doc=
"fields to remove from the metadata of the assembled ccd.",
194 doAssembleIsrExposures = pexConfig.Field(
197 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?" 199 doAssembleCcd = pexConfig.Field(
202 doc=
"Assemble amp-level exposures into a ccd-level exposure?" 204 expectWcs = pexConfig.Field(
207 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)" 209 doLinearize = pexConfig.Field(
211 doc=
"Correct for nonlinearity of the detector's response?",
214 doCrosstalk = pexConfig.Field(
216 doc=
"Apply intra-CCD crosstalk correction?",
219 crosstalk = pexConfig.ConfigurableField(
220 target=CrosstalkTask,
221 doc=
"Intra-CCD crosstalk correction",
223 doBrighterFatter = pexConfig.Field(
226 doc=
"Apply the brighter fatter correction" 228 brighterFatterKernelFile = pexConfig.Field(
231 doc=
"Kernel file used for the brighter fatter correction" 233 brighterFatterMaxIter = pexConfig.Field(
236 doc=
"Maximum number of iterations for the brighter fatter correction" 238 brighterFatterThreshold = pexConfig.Field(
241 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the " 242 " absolute value of the difference between the current corrected image and the one" 243 " from the previous iteration summed over all the pixels." 245 brighterFatterApplyGain = pexConfig.Field(
248 doc=
"Should the gain be applied when applying the brighter fatter correction?" 250 datasetType = pexConfig.Field(
252 doc=
"Dataset type for input data; users will typically leave this alone, " 253 "but camera-specific ISR tasks will override it",
256 fallbackFilterName = pexConfig.Field(dtype=str,
257 doc=
"Fallback default filter name for calibrations", optional=
True)
258 doAttachTransmissionCurve = pexConfig.Field(
261 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?" 263 doUseOpticsTransmission = pexConfig.Field(
266 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 268 doUseFilterTransmission = pexConfig.Field(
271 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 273 doUseSensorTransmission = pexConfig.Field(
276 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 278 doUseAtmosphereTransmission = pexConfig.Field(
281 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 296 @brief Apply common instrument signature correction algorithms to a raw frame. 298 @section ip_isr_isr_Contents Contents 300 - @ref ip_isr_isr_Purpose 301 - @ref ip_isr_isr_Initialize 303 - @ref ip_isr_isr_Config 304 - @ref ip_isr_isr_Debug 307 @section ip_isr_isr_Purpose Description 309 The process for correcting imaging data is very similar from camera to camera. 310 This task provides a vanilla implementation of doing these corrections, including 311 the ability to turn certain corrections off if they are not needed. 312 The inputs to the primary method, run, are a raw exposure to be corrected and the 313 calibration data products. The raw input is a single chip sized mosaic of all amps 314 including overscans and other non-science pixels. 315 The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 316 and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 317 This task may not meet all needs and it is expected that it will be subclassed for 318 specific applications. 320 @section ip_isr_isr_Initialize Task initialization 322 @copydoc \_\_init\_\_ 324 @section ip_isr_isr_IO Inputs/Outputs to the run method 328 @section ip_isr_isr_Config Configuration parameters 330 See @ref IsrTaskConfig 332 @section ip_isr_isr_Debug Debug variables 334 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 335 flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 336 href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 337 Using lsstDebug to control debugging output</a> for more about @b debug.py files. 339 The available variables in IsrTask are: 342 <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 345 <DD> display exposure after ISR has been applied 349 For example, put something like 353 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 354 if name == "lsst.ip.isrFunctions.isrTask": 355 di.display = {'postISRCCD':2} 357 lsstDebug.Info = DebugInfo 359 into your debug.py file and run the commandline task with the @c --debug flag. 363 ConfigClass = IsrTaskConfig
367 '''!Constructor for IsrTask 368 @param[in] *args a list of positional arguments passed on to the Task constructor 369 @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 370 Call the lsst.pipe.base.task.Task.__init__ method 371 Then setup the assembly and fringe correction subtasks 373 pipeBase.Task.__init__(self, *args, **kwargs)
374 self.makeSubtask(
"assembleCcd")
375 self.makeSubtask(
"fringe")
376 self.makeSubtask(
"crosstalk")
379 """!Retrieve necessary frames for instrument signature removal 380 @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 381 of the detector data to be processed 382 @param[in] rawExposure a reference raw exposure that will later be 383 corrected with the retrieved calibration data; 384 should not be modified in this method. 385 @return a pipeBase.Struct with fields containing kwargs expected by run() 386 - bias: exposure of bias frame 387 - dark: exposure of dark frame 388 - flat: exposure of flat field 389 - defects: list of detects 390 - fringeStruct: a pipeBase.Struct with field fringes containing 391 exposure of fringe frame or list of fringe exposure 393 ccd = rawExposure.getDetector()
395 biasExposure = self.
getIsrExposure(dataRef,
"bias")
if self.config.doBias
else None 397 linearizer = dataRef.get(
"linearizer", immediate=
True)
if self.
doLinearize(ccd)
else None 398 darkExposure = self.
getIsrExposure(dataRef,
"dark")
if self.config.doDark
else None 399 flatExposure = self.
getIsrExposure(dataRef,
"flat")
if self.config.doFlat
else None 400 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
if self.config.doBrighterFatter
else None 401 defectList = dataRef.get(
"defects")
if self.config.doDefect
else None 403 if self.config.doFringe
and self.fringe.checkFilter(rawExposure):
404 fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
405 if self.config.doAssembleIsrExposures
else None)
407 fringeStruct = pipeBase.Struct(fringes=
None)
409 if self.config.doAttachTransmissionCurve:
410 opticsTransmission = (dataRef.get(
"transmission_optics")
411 if self.config.doUseOpticsTransmission
else None)
412 filterTransmission = (dataRef.get(
"transmission_filter")
413 if self.config.doUseFilterTransmission
else None)
414 sensorTransmission = (dataRef.get(
"transmission_sensor")
415 if self.config.doUseSensorTransmission
else None)
416 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
417 if self.config.doUseAtmosphereTransmission
else None)
419 opticsTransmission =
None 420 filterTransmission =
None 421 sensorTransmission =
None 422 atmosphereTransmission =
None 425 return pipeBase.Struct(bias=biasExposure,
426 linearizer=linearizer,
430 fringes=fringeStruct,
431 bfKernel=brighterFatterKernel,
432 opticsTransmission=opticsTransmission,
433 filterTransmission=filterTransmission,
434 sensorTransmission=sensorTransmission,
435 atmosphereTransmission=atmosphereTransmission,
439 def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
440 fringes=None, bfKernel=None, camera=None,
441 opticsTransmission=None, filterTransmission=None,
442 sensorTransmission=None, atmosphereTransmission=None):
443 """!Perform instrument signature removal on an exposure 446 - Detect saturation, apply overscan correction, bias, dark and flat 447 - Perform CCD assembly 448 - Interpolate over defects, saturated pixels and all NaNs 450 @param[in] ccdExposure lsst.afw.image.exposure of detector data 451 @param[in] bias exposure of bias frame 452 @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 453 @param[in] dark exposure of dark frame 454 @param[in] flat exposure of flatfield 455 @param[in] defects list of detects 456 @param[in] fringes a pipeBase.Struct with field fringes containing 457 exposure of fringe frame or list of fringe exposure 458 @param[in] bfKernel kernel for brighter-fatter correction 459 @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 460 used by addDistortionModel 461 @param[in] opticsTransmission a TransmissionCurve for the optics 462 @param[in] filterTransmission a TransmissionCurve for the filter 463 @param[in] sensorTransmission a TransmissionCurve for the sensor 464 @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 466 @return a pipeBase.Struct with field: 470 if isinstance(ccdExposure, ButlerDataRef):
473 ccd = ccdExposure.getDetector()
476 if self.config.doBias
and bias
is None:
477 raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
479 raise RuntimeError(
"Must supply a linearizer if config.doBias True")
480 if self.config.doDark
and dark
is None:
481 raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
482 if self.config.doFlat
and flat
is None:
483 raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
484 if self.config.doBrighterFatter
and bfKernel
is None:
485 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter True")
487 fringes = pipeBase.Struct(fringes=
None)
488 if self.config.doFringe
and not isinstance(fringes, pipeBase.Struct):
489 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
490 if self.config.doDefect
and defects
is None:
491 raise RuntimeError(
"Must supply defects if config.doDefect True")
492 if self.config.doAddDistortionModel
and camera
is None:
493 raise RuntimeError(
"Must supply camera if config.doAddDistortionModel True")
498 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd" 499 ccd = [
FakeAmp(ccdExposure, self.config)]
503 if ccdExposure.getBBox().contains(amp.getBBox()):
508 if self.config.doAssembleCcd:
509 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
510 if self.config.expectWcs
and not ccdExposure.getWcs():
511 self.log.warn(
"No WCS found in input exposure")
513 if self.config.doBias:
517 linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
519 if self.config.doCrosstalk:
520 self.crosstalk.
run(ccdExposure)
524 if ccdExposure.getBBox().contains(amp.getBBox()):
525 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
528 interpolationDone =
False 530 if self.config.doBrighterFatter:
537 if self.config.doDefect:
539 if self.config.doSaturationInterpolation:
542 interpolationDone =
True 545 self.config.brighterFatterMaxIter,
546 self.config.brighterFatterThreshold,
547 self.config.brighterFatterApplyGain,
550 if self.config.doDark:
553 if self.config.doFringe
and not self.config.fringeAfterFlat:
554 self.fringe.
run(ccdExposure, **fringes.getDict())
556 if self.config.doFlat:
559 if not interpolationDone:
560 if self.config.doDefect:
562 if self.config.doSaturationInterpolation:
564 if not interpolationDone
or self.config.doNanInterpAfterFlat:
567 if self.config.doFringe
and self.config.fringeAfterFlat:
568 self.fringe.
run(ccdExposure, **fringes.getDict())
570 exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
571 ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
573 if self.config.doAddDistortionModel:
576 if self.config.doAttachTransmissionCurve:
578 filterTransmission=filterTransmission,
579 sensorTransmission=sensorTransmission,
580 atmosphereTransmission=atmosphereTransmission)
582 frame = getDebugFrame(self._display,
"postISRCCD")
584 getDisplay(frame).mtv(ccdExposure)
586 return pipeBase.Struct(
587 exposure=ccdExposure,
592 """!Perform instrument signature removal on a ButlerDataRef of a Sensor 594 - Read in necessary detrending/isr/calibration data 595 - Process raw exposure in run() 596 - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 598 @param[in] sensorRef daf.persistence.butlerSubset.ButlerDataRef of the 599 detector data to be processed 600 @return a pipeBase.Struct with fields: 601 - exposure: the exposure after application of ISR 603 self.log.info(
"Performing ISR on sensor %s" % (sensorRef.dataId))
604 ccdExposure = sensorRef.get(
'raw')
605 camera = sensorRef.get(
"camera")
606 if camera
is None and self.config.doAddDistortionModel:
607 raise RuntimeError(
"config.doAddDistortionModel is True " 608 "but could not get a camera from the butler")
611 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
613 if self.config.doWrite:
614 sensorRef.put(result.exposure,
"postISRCCD")
619 """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 621 if isinstance(exposure, afwImage.ExposureF):
624 if not hasattr(exposure,
"convertF"):
625 raise RuntimeError(
"Unable to convert exposure (%s) to float" % type(exposure))
627 newexposure = exposure.convertF()
628 maskedImage = newexposure.getMaskedImage()
629 varArray = maskedImage.getVariance().getArray()
631 maskArray = maskedImage.getMask().getArray()
636 """!Apply bias correction in place 638 @param[in,out] exposure exposure to process 639 @param[in] biasExposure bias exposure of same size as exposure 641 isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
644 """!Apply dark correction in place 646 @param[in,out] exposure exposure to process 647 @param[in] darkExposure dark exposure of same size as exposure 648 @param[in] invert if True, remove the dark from an already-corrected image 650 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
651 if math.isnan(expScale):
652 raise RuntimeError(
"Exposure darktime is NAN")
653 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
654 if math.isnan(darkScale):
655 raise RuntimeError(
"Dark calib darktime is NAN")
656 isrFunctions.darkCorrection(
657 maskedImage=exposure.getMaskedImage(),
658 darkMaskedImage=darkExposure.getMaskedImage(),
665 """!Is linearization wanted for this detector? 667 Checks config.doLinearize and the linearity type of the first amplifier. 669 @param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 671 return self.config.doLinearize
and \
672 detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
675 """!Set the variance plane based on the image plane, plus amplifier gain and read noise 677 @param[in,out] ampExposure exposure to process 678 @param[in] amp amplifier detector information 681 if not math.isnan(gain):
684 self.log.warn(
"Gain for amp %s == %g <= 0; setting to %f" %
685 (amp.getName(), gain, patchedGain))
688 isrFunctions.updateVariance(
689 maskedImage=ampExposure.getMaskedImage(),
691 readNoise=amp.getReadNoise(),
695 """!Apply flat correction in place 697 @param[in,out] exposure exposure to process 698 @param[in] flatExposure flatfield exposure same size as exposure 699 @param[in] invert if True, unflatten an already-flattened image instead. 701 isrFunctions.flatCorrection(
702 maskedImage=exposure.getMaskedImage(),
703 flatMaskedImage=flatExposure.getMaskedImage(),
704 scalingType=self.config.flatScalingType,
705 userScale=self.config.flatUserScale,
710 """!Retrieve a calibration dataset for removing instrument signature 712 @param[in] dataRef data reference for exposure 713 @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 714 @param[in] immediate if True, disable butler proxies to enable error 715 handling within this routine 719 exp = dataRef.get(datasetType, immediate=immediate)
720 except Exception
as exc1:
721 if not self.config.fallbackFilterName:
722 raise RuntimeError(
"Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
724 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
725 except Exception
as exc2:
726 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
727 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
728 self.log.warn(
"Using fallback calibration from filter %s" % self.config.fallbackFilterName)
730 if self.config.doAssembleIsrExposures:
731 exp = self.assembleCcd.assembleCcd(exp)
735 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 737 @param[in,out] exposure exposure to process; only the amp DataSec is processed 738 @param[in] amp amplifier device data 740 if not math.isnan(amp.getSaturation()):
741 maskedImage = exposure.getMaskedImage()
742 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
743 isrFunctions.makeThresholdMask(
744 maskedImage=dataView,
745 threshold=amp.getSaturation(),
747 maskName=self.config.saturatedMaskName,
751 """!Interpolate over saturated pixels, in place 753 @param[in,out] ccdExposure exposure to process 756 - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 757 - Call this after CCD assembly, since saturated regions may cross amplifier boundaries 759 isrFunctions.interpolateFromMask(
760 maskedImage=ccdExposure.getMaskedImage(),
761 fwhm=self.config.fwhm,
762 growFootprints=self.config.growSaturationFootprintSize,
763 maskName=self.config.saturatedMaskName,
767 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 769 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 770 This is intended to indicate pixels that may be affected by unknown systematics; 771 for example if non-linearity corrections above a certain level are unstable 772 then that would be a useful value for suspectLevel. A value of `nan` indicates 773 that no such level exists and no pixels are to be masked as suspicious. 775 @param[in,out] exposure exposure to process; only the amp DataSec is processed 776 @param[in] amp amplifier device data 778 suspectLevel = amp.getSuspectLevel()
779 if math.isnan(suspectLevel):
782 maskedImage = exposure.getMaskedImage()
783 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
784 isrFunctions.makeThresholdMask(
785 maskedImage=dataView,
786 threshold=suspectLevel,
788 maskName=self.config.suspectMaskName,
792 """!Mask defects using mask plane "BAD" and interpolate over them, in place 794 @param[in,out] ccdExposure exposure to process 795 @param[in] defectBaseList a list of defects to mask and interpolate 797 @warning: call this after CCD assembly, since defects may cross amplifier boundaries 799 maskedImage = ccdExposure.getMaskedImage()
801 for d
in defectBaseList:
803 nd = measAlg.Defect(bbox)
804 defectList.append(nd)
805 isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName=
'BAD')
806 isrFunctions.interpolateDefectList(
807 maskedImage=maskedImage,
808 defectList=defectList,
809 fwhm=self.config.fwhm,
813 """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 815 We mask and interpolate over all NaNs, including those 816 that are masked with other bits (because those may or may 817 not be interpolated over later, and we want to remove all 818 NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 819 is used to preserve the historical name. 821 @param[in,out] exposure exposure to process 823 maskedImage = exposure.getMaskedImage()
826 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
827 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
828 numNans =
maskNans(maskedImage, maskVal)
829 self.metadata.set(
"NUMNANS", numNans)
833 self.log.warn(
"There were %i unmasked NaNs", numNans)
834 nanDefectList = isrFunctions.getDefectListFromMask(
835 maskedImage=maskedImage,
836 maskName=
'UNMASKEDNAN',
838 isrFunctions.interpolateDefectList(
839 maskedImage=exposure.getMaskedImage(),
840 defectList=nanDefectList,
841 fwhm=self.config.fwhm,
845 """!Apply overscan correction, in place 847 @param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels 848 @param[in] amp amplifier device data 850 if not amp.getHasRawInfo():
851 raise RuntimeError(
"This method must be executed on an amp with raw information.")
853 if amp.getRawHorizontalOverscanBBox().isEmpty():
854 self.log.info(
"No Overscan region. Not performing Overscan Correction.")
857 maskedImage = exposure.getMaskedImage()
858 dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
860 expImage = exposure.getMaskedImage().getImage()
861 overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
863 isrFunctions.overscanCorrection(
864 ampMaskedImage=dataView,
865 overscanImage=overscanImage,
866 fitType=self.config.overscanFitType,
867 order=self.config.overscanOrder,
868 collapseRej=self.config.overscanRej,
872 """!Update the WCS in exposure with a distortion model based on camera geometry 874 Add a model for optical distortion based on geometry found in `camera` 875 and the `exposure`'s detector. The raw input exposure is assumed 876 have a TAN WCS that has no compensation for optical distortion. 877 Two other possibilities are: 878 - The raw input exposure already has a model for optical distortion, 879 as is the case for raw DECam data. 880 In that case you should set config.doAddDistortionModel False. 881 - The raw input exposure has a model for distortion, but it has known 882 deficiencies severe enough to be worth fixing (e.g. because they 883 cause problems for fitting a better WCS). In that case you should 884 override this method with a version suitable for your raw data. 886 @param[in,out] exposure exposure to process; must include a Detector and a WCS; 887 the WCS of the exposure is modified in place 888 @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 890 self.log.info(
"Adding a distortion model to the WCS")
891 wcs = exposure.getWcs()
893 raise RuntimeError(
"exposure has no WCS")
895 raise RuntimeError(
"camera is None")
896 detector = exposure.getDetector()
898 raise RuntimeError(
"exposure has no Detector")
899 pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
900 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
901 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
902 exposure.setWcs(distortedWcs)
905 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners 907 @param[in,out] ccdExposure exposure to process 908 @param[in] fpPolygon Polygon in focal plane coordinates 911 ccd = ccdExposure.getDetector()
912 fpCorners = ccd.getCorners(FOCAL_PLANE)
913 ccdPolygon = Polygon(fpCorners)
916 intersect = ccdPolygon.intersectionSingle(fpPolygon)
919 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
920 validPolygon = Polygon(ccdPoints)
921 ccdExposure.getInfo().setValidPolygon(validPolygon)
924 """Apply brighter fatter correction in place for the image 926 This correction takes a kernel that has been derived from flat field images to 927 redistribute the charge. The gradient of the kernel is the deflection 928 field due to the accumulated charge. 930 Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 931 using the following equation: 933 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 935 To evaluate the derivative term we expand it as follows: 937 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 939 Because we use the measured counts instead of the incident counts we apply the correction 940 iteratively to reconstruct the original counts and the correction. We stop iterating when the 941 summed difference between the current corrected image and the one from the previous iteration 942 is below the threshold. We do not require convergence because the number of iterations is 943 too large a computational cost. How we define the threshold still needs to be evaluated, the 944 current default was shown to work reasonably well on a small set of images. For more information 945 on the method see DocuShare Document-19407. 947 The edges as defined by the kernel are not corrected because they have spurious values 948 due to the convolution. 950 self.log.info(
"Applying brighter fatter correction")
952 image = exposure.getMaskedImage().getImage()
957 kLx = numpy.shape(kernel)[0]
958 kLy = numpy.shape(kernel)[1]
959 kernelImage = afwImage.ImageD(kLx, kLy)
960 kernelImage.getArray()[:, :] = kernel
961 tempImage = image.clone()
963 nanIndex = numpy.isnan(tempImage.getArray())
964 tempImage.getArray()[nanIndex] = 0.
966 outImage = afwImage.ImageF(image.getDimensions())
967 corr = numpy.zeros_like(image.getArray())
968 prev_image = numpy.zeros_like(image.getArray())
969 convCntrl = afwMath.ConvolutionControl(
False,
True, 1)
970 fixedKernel = afwMath.FixedKernel(kernelImage)
980 for iteration
in range(maxIter):
982 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
983 tmpArray = tempImage.getArray()
984 outArray = outImage.getArray()
986 with numpy.errstate(invalid=
"ignore", over=
"ignore"):
988 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
989 gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
990 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
993 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
994 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
995 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
997 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
999 tmpArray[:, :] = image.getArray()[:, :]
1000 tmpArray[nanIndex] = 0.
1001 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1004 diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1006 if diff < threshold:
1008 prev_image[:, :] = tmpArray[:, :]
1010 if iteration == maxIter - 1:
1011 self.log.warn(
"Brighter fatter correction did not converge, final difference %f" % diff)
1013 self.log.info(
"Finished brighter fatter in %d iterations" % (iteration + 1))
1014 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1015 corr[startY + 1:endY - 1, startX + 1:endX - 1]
1018 sensorTransmission=None, atmosphereTransmission=None):
1019 """Attach a TransmissionCurve to an Exposure, given separate curves for 1020 different components. 1024 exposure : `lsst.afw.image.Exposure` 1025 Exposure object to modify by attaching the product of all given 1026 ``TransmissionCurves`` in post-assembly trimmed detector 1027 coordinates. Must have a valid ``Detector`` attached that matches 1028 the detector associated with sensorTransmission. 1029 opticsTransmission : `lsst.afw.image.TransmissionCurve` 1030 A ``TransmissionCurve`` that represents the throughput of the 1031 optics, to be evaluated in focal-plane coordinates. 1032 filterTransmission : `lsst.afw.image.TransmissionCurve` 1033 A ``TransmissionCurve`` that represents the throughput of the 1034 filter itself, to be evaluated in focal-plane coordinates. 1035 sensorTransmission : `lsst.afw.image.TransmissionCurve` 1036 A ``TransmissionCurve`` that represents the throughput of the 1037 sensor itself, to be evaluated in post-assembly trimmed detector 1039 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 1040 A ``TransmissionCurve`` that represents the throughput of the 1041 atmosphere, assumed to be spatially constant. 1043 All ``TransmissionCurve`` arguments are optional; if none are provided, 1044 the attached ``TransmissionCurve`` will have unit transmission 1049 combined : ``lsst.afw.image.TransmissionCurve`` 1050 The TransmissionCurve attached to the exposure. 1052 return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1053 filterTransmission=filterTransmission,
1054 sensorTransmission=sensorTransmission,
1055 atmosphereTransmission=atmosphereTransmission)
1059 """Context manager that applies and removes gain 1062 ccd = exp.getDetector()
1064 sim = image.Factory(image, amp.getBBox())
1065 sim *= amp.getGain()
1071 ccd = exp.getDetector()
1073 sim = image.Factory(image, amp.getBBox())
1074 sim /= amp.getGain()
1078 """Context manager that applies and removes flats and darks, 1079 if the task is configured to apply them. 1081 if self.config.doDark
and dark
is not None:
1083 if self.config.doFlat:
1088 if self.config.doFlat:
1090 if self.config.doDark
and dark
is not None:
1095 """A Detector-like object that supports returning gain and saturation level""" 1098 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
1100 self.
_gain = config.gain
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
def runDataRef(self, sensorRef)
Perform instrument signature removal on a ButlerDataRef of a Sensor.
def gainContext(self, exp, image, apply)
def __init__(self, args, kwargs)
Constructor for IsrTask.
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Apply common instrument signature correction algorithms to a raw frame.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
def convertIntToFloat(self, exposure)
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Perform instrument signature removal on an exposure.
_RawHorizontalOverscanBBox
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
def doLinearize(self, detector)
Is linearization wanted for this detector?
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
def flatContext(self, exp, flat, dark=None)
def updateVariance(self, ampExposure, amp)
Set the variance plane based on the image plane, plus amplifier gain and read noise.
def overscanCorrection(self, exposure, amp)
Apply overscan correction, in place.
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
def __init__(self, exposure, config)