lsst.ip.isr  13.0-13-gd4b2922+10
 All Classes Namespaces Files Functions Variables Typedefs Macros Groups
isrTask.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 from builtins import object
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 import numpy
27 
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.math as afwMath
34 from lsst.daf.persistence import ButlerDataRef
35 from lsstDebug import getDebugFrame
36 from lsst.afw.display import getDisplay
37 from . import isrFunctions
38 from .assembleCcdTask import AssembleCcdTask
39 from .fringe import FringeTask
40 from lsst.afw.geom.polygon import Polygon
41 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
42 from contextlib import contextmanager
43 from .isr import maskNans
44 
45 
46 class IsrTaskConfig(pexConfig.Config):
47  doBias = pexConfig.Field(
48  dtype=bool,
49  doc="Apply bias frame correction?",
50  default=True,
51  )
52  doDark = pexConfig.Field(
53  dtype=bool,
54  doc="Apply dark frame correction?",
55  default=True,
56  )
57  doFlat = pexConfig.Field(
58  dtype=bool,
59  doc="Apply flat field correction?",
60  default=True,
61  )
62  doFringe = pexConfig.Field(
63  dtype=bool,
64  doc="Apply fringe correction?",
65  default=True,
66  )
67  doDefect = pexConfig.Field(
68  dtype=bool,
69  doc="Apply correction for CCD defects, e.g. hot pixels?",
70  default=True,
71  )
72  doWrite = pexConfig.Field(
73  dtype=bool,
74  doc="Persist postISRCCD?",
75  default=True,
76  )
77  assembleCcd = pexConfig.ConfigurableField(
78  target=AssembleCcdTask,
79  doc="CCD assembly task",
80  )
81  gain = pexConfig.Field(
82  dtype=float,
83  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
84  default=float("NaN"),
85  )
86  readNoise = pexConfig.Field(
87  dtype=float,
88  doc="The read noise to use if no Detector is present in the Exposure",
89  default=0.0,
90  )
91  saturation = pexConfig.Field(
92  dtype=float,
93  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
94  default=float("NaN"),
95  )
96  fringeAfterFlat = pexConfig.Field(
97  dtype=bool,
98  doc="Do fringe subtraction after flat-fielding?",
99  default=True,
100  )
101  fringe = pexConfig.ConfigurableField(
102  target=FringeTask,
103  doc="Fringe subtraction task",
104  )
105  fwhm = pexConfig.Field(
106  dtype=float,
107  doc="FWHM of PSF (arcsec)",
108  default=1.0,
109  )
110  saturatedMaskName = pexConfig.Field(
111  dtype=str,
112  doc="Name of mask plane to use in saturation detection and interpolation",
113  default="SAT",
114  )
115  suspectMaskName = pexConfig.Field(
116  dtype=str,
117  doc="Name of mask plane to use for suspect pixels",
118  default="SUSPECT",
119  )
120  flatScalingType = pexConfig.ChoiceField(
121  dtype=str,
122  doc="The method for scaling the flat on the fly.",
123  default='USER',
124  allowed={
125  "USER": "Scale by flatUserScale",
126  "MEAN": "Scale by the inverse of the mean",
127  "MEDIAN": "Scale by the inverse of the median",
128  },
129  )
130  flatUserScale = pexConfig.Field(
131  dtype=float,
132  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
133  default=1.0,
134  )
135  overscanFitType = pexConfig.ChoiceField(
136  dtype=str,
137  doc="The method for fitting the overscan bias level.",
138  default='MEDIAN',
139  allowed={
140  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
141  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
142  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
143  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
144  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
145  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
146  "MEAN": "Correct using the mean of the overscan region",
147  "MEDIAN": "Correct using the median of the overscan region",
148  },
149  )
150  overscanOrder = pexConfig.Field(
151  dtype=int,
152  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
153  "or number of spline knots if overscan fit type is a spline."),
154  default=1,
155  )
156  overscanRej = pexConfig.Field(
157  dtype=float,
158  doc="Rejection threshold (sigma) for collapsing overscan before fit",
159  default=3.0,
160  )
161  growSaturationFootprintSize = pexConfig.Field(
162  dtype=int,
163  doc="Number of pixels by which to grow the saturation footprints",
164  default=1,
165  )
166  doSaturationInterpolation = pexConfig.Field(
167  dtype=bool,
168  doc="Perform interpolation over pixels masked as saturated?",
169  default=True,
170  )
171  fluxMag0T1 = pexConfig.Field(
172  dtype=float,
173  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
174  default=1e10,
175  )
176  keysToRemoveFromAssembledCcd = pexConfig.ListField(
177  dtype=str,
178  doc="fields to remove from the metadata of the assembled ccd.",
179  default=[],
180  )
181  doAssembleIsrExposures = pexConfig.Field(
182  dtype=bool,
183  default=False,
184  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
185  )
186  doAssembleCcd = pexConfig.Field(
187  dtype=bool,
188  default=True,
189  doc="Assemble amp-level exposures into a ccd-level exposure?"
190  )
191  expectWcs = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)"
195  )
196  doLinearize = pexConfig.Field(
197  dtype=bool,
198  doc="Correct for nonlinearity of the detector's response?",
199  default=True,
200  )
201  doBrighterFatter = pexConfig.Field(
202  dtype=bool,
203  default=False,
204  doc="Apply the brighter fatter correction"
205  )
206  brighterFatterKernelFile = pexConfig.Field(
207  dtype=str,
208  default='',
209  doc="Kernel file used for the brighter fatter correction"
210  )
211  brighterFatterMaxIter = pexConfig.Field(
212  dtype=int,
213  default=10,
214  doc="Maximum number of iterations for the brighter fatter correction"
215  )
216  brighterFatterThreshold = pexConfig.Field(
217  dtype=float,
218  default=1000,
219  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
220  " absolute value of the difference between the current corrected image and the one"
221  " from the previous iteration summed over all the pixels."
222  )
223  brighterFatterApplyGain = pexConfig.Field(
224  dtype=bool,
225  default=True,
226  doc="Should the gain be applied when applying the brighter fatter correction?"
227  )
228  datasetType = pexConfig.Field(
229  dtype=str,
230  doc="Dataset type for input data; users will typically leave this alone, "
231  "but camera-specific ISR tasks will override it",
232  default="raw",
233  )
234  fallbackFilterName = pexConfig.Field(dtype=str,
235  doc="Fallback default filter name for calibrations", optional=True)
236 
237 ## \addtogroup LSST_task_documentation
238 ## \{
239 ## \page IsrTask
240 ## \ref IsrTask_ "IsrTask"
241 ## \copybrief IsrTask
242 ## \}
243 
244 
245 class IsrTask(pipeBase.CmdLineTask):
246  """!
247  \anchor IsrTask_
248 
249  \brief Apply common instrument signature correction algorithms to a raw frame.
250 
251  \section ip_isr_isr_Contents Contents
252 
253  - \ref ip_isr_isr_Purpose
254  - \ref ip_isr_isr_Initialize
255  - \ref ip_isr_isr_IO
256  - \ref ip_isr_isr_Config
257  - \ref ip_isr_isr_Debug
258 
259 
260  \section ip_isr_isr_Purpose Description
261 
262  The process for correcting imaging data is very similar from camera to camera.
263  This task provides a vanilla implementation of doing these corrections, including
264  the ability to turn certain corrections off if they are not needed.
265  The inputs to the primary method, run, are a raw exposure to be corrected and the
266  calibration data products. The raw input is a single chip sized mosaic of all amps
267  including overscans and other non-science pixels.
268  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
269  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
270  This task may not meet all needs and it is expected that it will be subclassed for
271  specific applications.
272 
273  \section ip_isr_isr_Initialize Task initialization
274 
275  \copydoc \_\_init\_\_
276 
277  \section ip_isr_isr_IO Inputs/Outputs to the run method
278 
279  \copydoc run
280 
281  \section ip_isr_isr_Config Configuration parameters
282 
283  See \ref IsrTaskConfig
284 
285  \section ip_isr_isr_Debug Debug variables
286 
287  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
288  flag \c --debug, \c -d to import \b debug.py from your \c PYTHONPATH; see <a
289  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
290  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
291 
292  The available variables in IsrTask are:
293  <DL>
294  <DT> \c display
295  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
296  <DL>
297  <DT> postISRCCD
298  <DD> display exposure after ISR has been applied
299  </DL>
300  </DL>
301 
302  For example, put something like
303  \code{.py}
304  import lsstDebug
305  def DebugInfo(name):
306  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
307  if name == "lsst.ip.isrFunctions.isrTask":
308  di.display = {'postISRCCD':2}
309  return di
310  lsstDebug.Info = DebugInfo
311  \endcode
312  into your debug.py file and run the commandline task with the \c --debug flag.
313 
314  <HR>
315  """
316  ConfigClass = IsrTaskConfig
317  _DefaultName = "isr"
318 
319  def __init__(self, *args, **kwargs):
320  '''!Constructor for IsrTask
321  \param[in] *args -- a list of positional arguments passed on to the Task constructor
322  \param[in] **kwargs -- a dictionary of keyword arguments passed on to the Task constructor
323  Call the lsst.pipe.base.task.Task.__init__ method
324  Then setup the assembly and fringe correction subtasks
325  '''
326  pipeBase.Task.__init__(self, *args, **kwargs)
327  self.makeSubtask("assembleCcd")
328  self.makeSubtask("fringe")
329 
330  def readIsrData(self, dataRef, rawExposure):
331  """!Retrieve necessary frames for instrument signature removal
332  \param[in] dataRef -- a daf.persistence.butlerSubset.ButlerDataRef
333  of the detector data to be processed
334  \param[in] rawExposure -- a reference raw exposure that will later be
335  corrected with the retrieved calibration data;
336  should not be modified in this method.
337  \return a pipeBase.Struct with fields containing kwargs expected by run()
338  - bias: exposure of bias frame
339  - dark: exposure of dark frame
340  - flat: exposure of flat field
341  - defects: list of detects
342  - fringeStruct: a pipeBase.Struct with field fringes containing
343  exposure of fringe frame or list of fringe exposure
344  """
345  ccd = rawExposure.getDetector()
346 
347  biasExposure = self.getIsrExposure(dataRef, "bias") if self.config.doBias else None
348  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
349  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
350  darkExposure = self.getIsrExposure(dataRef, "dark") if self.config.doDark else None
351  flatExposure = self.getIsrExposure(dataRef, "flat") if self.config.doFlat else None
352  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
353  defectList = dataRef.get("defects") if self.config.doDefect else None
354 
355  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
356  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
357  if self.config.doAssembleIsrExposures else None)
358  else:
359  fringeStruct = pipeBase.Struct(fringes=None)
360 
361  # Struct should include only kwargs to run()
362  return pipeBase.Struct(bias=biasExposure,
363  linearizer=linearizer,
364  dark=darkExposure,
365  flat=flatExposure,
366  defects=defectList,
367  fringes=fringeStruct,
368  bfKernel=brighterFatterKernel
369  )
370 
371  @pipeBase.timeMethod
372  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
373  fringes=None, bfKernel=None):
374  """!Perform instrument signature removal on an exposure
375 
376  Steps include:
377  - Detect saturation, apply overscan correction, bias, dark and flat
378  - Perform CCD assembly
379  - Interpolate over defects, saturated pixels and all NaNs
380 
381  \param[in] ccdExposure -- lsst.afw.image.exposure of detector data
382  \param[in] bias -- exposure of bias frame
383  \param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
384  \param[in] dark -- exposure of dark frame
385  \param[in] flat -- exposure of flatfield
386  \param[in] defects -- list of detects
387  \param[in] fringes -- a pipeBase.Struct with field fringes containing
388  exposure of fringe frame or list of fringe exposure
389  \param[in] bfKernel -- kernel for brighter-fatter correction
390 
391  \return a pipeBase.Struct with field:
392  - exposure
393  """
394 
395  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
396  if isinstance(ccdExposure, ButlerDataRef):
397  return self.runDataRef(ccdExposure)
398 
399  ccd = ccdExposure.getDetector()
400 
401  # Validate Input
402  if self.config.doBias and bias is None:
403  raise RuntimeError("Must supply a bias exposure if config.doBias True")
404  if self.doLinearize(ccd) and linearizer is None:
405  raise RuntimeError("Must supply a linearizer if config.doBias True")
406  if self.config.doDark and dark is None:
407  raise RuntimeError("Must supply a dark exposure if config.doDark True")
408  if self.config.doFlat and flat is None:
409  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
410  if self.config.doBrighterFatter and bfKernel is None:
411  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
412  if fringes is None:
413  fringes = pipeBase.Struct(fringes=None)
414  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
415  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
416  if self.config.doDefect and defects is None:
417  raise RuntimeError("Must supply defects if config.doDefect True")
418 
419  ccdExposure = self.convertIntToFloat(ccdExposure)
420 
421  if not ccd:
422  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
423  ccd = [FakeAmp(ccdExposure, self.config)]
424 
425  for amp in ccd:
426  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
427  if ccdExposure.getBBox().contains(amp.getBBox()):
428  self.saturationDetection(ccdExposure, amp)
429  self.suspectDetection(ccdExposure, amp)
430  self.overscanCorrection(ccdExposure, amp)
431 
432  if self.config.doAssembleCcd:
433  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
434  if self.config.expectWcs and not ccdExposure.getWcs():
435  self.log.warn("No WCS found in input exposure")
436 
437  if self.config.doBias:
438  self.biasCorrection(ccdExposure, bias)
439 
440  if self.doLinearize(ccd):
441  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
442 
443  for amp in ccd:
444  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
445  if ccdExposure.getBBox().contains(amp.getBBox()):
446  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
447  self.updateVariance(ampExposure, amp)
448 
449  if self.config.doBrighterFatter:
450  self.brighterFatterCorrection(ccdExposure, bfKernel,
451  self.config.brighterFatterMaxIter,
452  self.config.brighterFatterThreshold,
453  self.config.brighterFatterApplyGain,
454  )
455 
456  if self.config.doDark:
457  self.darkCorrection(ccdExposure, dark)
458 
459  if self.config.doFringe and not self.config.fringeAfterFlat:
460  self.fringe.run(ccdExposure, **fringes.getDict())
461 
462  if self.config.doFlat:
463  self.flatCorrection(ccdExposure, flat)
464 
465  if self.config.doDefect:
466  self.maskAndInterpDefect(ccdExposure, defects)
467 
468  if self.config.doSaturationInterpolation:
469  self.saturationInterpolation(ccdExposure)
470 
471  self.maskAndInterpNan(ccdExposure)
472 
473  if self.config.doFringe and self.config.fringeAfterFlat:
474  self.fringe.run(ccdExposure, **fringes.getDict())
475 
476  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
477  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
478 
479  frame = getDebugFrame(self._display, "postISRCCD")
480  if frame:
481  getDisplay(frame).mtv(ccdExposure)
482 
483  return pipeBase.Struct(
484  exposure=ccdExposure,
485  )
486 
487  @pipeBase.timeMethod
488  def runDataRef(self, sensorRef):
489  """!Perform instrument signature removal on a ButlerDataRef of a Sensor
490 
491  - Read in necessary detrending/isr/calibration data
492  - Process raw exposure in run()
493  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
494 
495  \param[in] sensorRef -- daf.persistence.butlerSubset.ButlerDataRef of the
496  detector data to be processed
497  \return a pipeBase.Struct with fields:
498  - exposure: the exposure after application of ISR
499  """
500  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
501  ccdExposure = sensorRef.get('raw')
502  isrData = self.readIsrData(sensorRef, ccdExposure)
503 
504  result = self.run(ccdExposure, **isrData.getDict())
505 
506  if self.config.doWrite:
507  sensorRef.put(result.exposure, "postISRCCD")
508 
509  return result
510 
511  def convertIntToFloat(self, exposure):
512  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
513  """
514  if isinstance(exposure, afwImage.ExposureF):
515  # Nothing to be done
516  return exposure
517  if not hasattr(exposure, "convertF"):
518  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
519 
520  newexposure = exposure.convertF()
521  maskedImage = newexposure.getMaskedImage()
522  varArray = maskedImage.getVariance().getArray()
523  varArray[:, :] = 1
524  maskArray = maskedImage.getMask().getArray()
525  maskArray[:, :] = 0
526  return newexposure
527 
528  def biasCorrection(self, exposure, biasExposure):
529  """!Apply bias correction in place
530 
531  \param[in,out] exposure exposure to process
532  \param[in] biasExposure bias exposure of same size as exposure
533  """
534  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
535 
536  def darkCorrection(self, exposure, darkExposure):
537  """!Apply dark correction in place
538 
539  \param[in,out] exposure exposure to process
540  \param[in] darkExposure dark exposure of same size as exposure
541  """
542  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
543  if math.isnan(expScale):
544  raise RuntimeError("Exposure darktime is NAN")
545  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
546  if math.isnan(darkScale):
547  raise RuntimeError("Dark calib darktime is NAN")
548  isrFunctions.darkCorrection(
549  maskedImage=exposure.getMaskedImage(),
550  darkMaskedImage=darkExposure.getMaskedImage(),
551  expScale=expScale,
552  darkScale=darkScale,
553  )
554 
555  def doLinearize(self, detector):
556  """!Is linearization wanted for this detector?
557 
558  Checks config.doLinearize and the linearity type of the first amplifier.
559 
560  \param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
561  """
562  return self.config.doLinearize and \
563  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
564 
565  def updateVariance(self, ampExposure, amp):
566  """!Set the variance plane based on the image plane, plus amplifier gain and read noise
567 
568  \param[in,out] ampExposure exposure to process
569  \param[in] amp amplifier detector information
570  """
571  if not math.isnan(amp.getGain()):
572  isrFunctions.updateVariance(
573  maskedImage=ampExposure.getMaskedImage(),
574  gain=amp.getGain(),
575  readNoise=amp.getReadNoise(),
576  )
577 
578  def flatCorrection(self, exposure, flatExposure):
579  """!Apply flat correction in place
580 
581  \param[in,out] exposure exposure to process
582  \param[in] flatExposure flatfield exposure same size as exposure
583  """
584  isrFunctions.flatCorrection(
585  maskedImage=exposure.getMaskedImage(),
586  flatMaskedImage=flatExposure.getMaskedImage(),
587  scalingType=self.config.flatScalingType,
588  userScale=self.config.flatUserScale,
589  )
590 
591  def getIsrExposure(self, dataRef, datasetType, immediate=True):
592  """!Retrieve a calibration dataset for removing instrument signature
593 
594  \param[in] dataRef data reference for exposure
595  \param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
596  \param[in] immediate if True, disable butler proxies to enable error
597  handling within this routine
598  \return exposure
599  """
600  try:
601  exp = dataRef.get(datasetType, immediate=immediate)
602  except Exception as exc1:
603  if not self.config.fallbackFilterName:
604  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
605  try:
606  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
607  except Exception as exc2:
608  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
609  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
610  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
611 
612  if self.config.doAssembleIsrExposures:
613  exp = self.assembleCcd.assembleCcd(exp)
614  return exp
615 
616  def saturationDetection(self, exposure, amp):
617  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
618 
619  \param[in,out] exposure exposure to process; only the amp DataSec is processed
620  \param[in] amp amplifier device data
621  """
622  if not math.isnan(amp.getSaturation()):
623  maskedImage = exposure.getMaskedImage()
624  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
625  isrFunctions.makeThresholdMask(
626  maskedImage=dataView,
627  threshold=amp.getSaturation(),
628  growFootprints=0,
629  maskName=self.config.saturatedMaskName,
630  )
631 
632  def saturationInterpolation(self, ccdExposure):
633  """!Interpolate over saturated pixels, in place
634 
635  \param[in,out] ccdExposure exposure to process
636 
637  \warning:
638  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
639  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
640  """
641  isrFunctions.interpolateFromMask(
642  maskedImage=ccdExposure.getMaskedImage(),
643  fwhm=self.config.fwhm,
644  growFootprints=self.config.growSaturationFootprintSize,
645  maskName=self.config.saturatedMaskName,
646  )
647 
648  def suspectDetection(self, exposure, amp):
649  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
650 
651  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
652  This is intended to indicate pixels that may be affected by unknown systematics;
653  for example if non-linearity corrections above a certain level are unstable
654  then that would be a useful value for suspectLevel. A value of `nan` indicates
655  that no such level exists and no pixels are to be masked as suspicious.
656 
657  \param[in,out] exposure exposure to process; only the amp DataSec is processed
658  \param[in] amp amplifier device data
659  """
660  suspectLevel = amp.getSuspectLevel()
661  if math.isnan(suspectLevel):
662  return
663 
664  maskedImage = exposure.getMaskedImage()
665  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
666  isrFunctions.makeThresholdMask(
667  maskedImage=dataView,
668  threshold=suspectLevel,
669  growFootprints=0,
670  maskName=self.config.suspectMaskName,
671  )
672 
673  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
674  """!Mask defects using mask plane "BAD" and interpolate over them, in place
675 
676  \param[in,out] ccdExposure exposure to process
677  \param[in] defectBaseList a list of defects to mask and interpolate
678 
679  \warning: call this after CCD assembly, since defects may cross amplifier boundaries
680  """
681  maskedImage = ccdExposure.getMaskedImage()
682  defectList = []
683  for d in defectBaseList:
684  bbox = d.getBBox()
685  nd = measAlg.Defect(bbox)
686  defectList.append(nd)
687  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
688  isrFunctions.interpolateDefectList(
689  maskedImage=maskedImage,
690  defectList=defectList,
691  fwhm=self.config.fwhm,
692  )
693 
694  def maskAndInterpNan(self, exposure):
695  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
696 
697  We mask and interpolate over all NaNs, including those
698  that are masked with other bits (because those may or may
699  not be interpolated over later, and we want to remove all
700  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
701  is used to preserve the historical name.
702 
703  \param[in,out] exposure exposure to process
704  """
705  maskedImage = exposure.getMaskedImage()
706 
707  # Find and mask NaNs
708  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
709  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
710  numNans = maskNans(maskedImage, maskVal)
711  self.metadata.set("NUMNANS", numNans)
712 
713  # Interpolate over these previously-unmasked NaNs
714  if numNans > 0:
715  self.log.warn("There were %i unmasked NaNs", numNans)
716  nanDefectList = isrFunctions.getDefectListFromMask(
717  maskedImage=maskedImage,
718  maskName='UNMASKEDNAN',
719  growFootprints=0,
720  )
721  isrFunctions.interpolateDefectList(
722  maskedImage=exposure.getMaskedImage(),
723  defectList=nanDefectList,
724  fwhm=self.config.fwhm,
725  )
726 
727  def overscanCorrection(self, exposure, amp):
728  """!Apply overscan correction, in place
729 
730  \param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
731  \param[in] amp amplifier device data
732  """
733  if not amp.getHasRawInfo():
734  raise RuntimeError("This method must be executed on an amp with raw information.")
735 
736  if amp.getRawHorizontalOverscanBBox().isEmpty():
737  self.log.info("No Overscan region. Not performing Overscan Correction.")
738  return None
739 
740  maskedImage = exposure.getMaskedImage()
741  dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
742 
743  expImage = exposure.getMaskedImage().getImage()
744  overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
745 
746  isrFunctions.overscanCorrection(
747  ampMaskedImage=dataView,
748  overscanImage=overscanImage,
749  fitType=self.config.overscanFitType,
750  order=self.config.overscanOrder,
751  collapseRej=self.config.overscanRej,
752  )
753 
754  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
755  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
756 
757  \param[in,out] ccdExposure exposure to process
758  \param[in] fpPolygon Polygon in focal plane coordinates
759  """
760  # Get ccd corners in focal plane coordinates
761  ccd = ccdExposure.getDetector()
762  fpCorners = ccd.getCorners(FOCAL_PLANE)
763  ccdPolygon = Polygon(fpCorners)
764 
765  # Get intersection of ccd corners with fpPolygon
766  intersect = ccdPolygon.intersectionSingle(fpPolygon)
767 
768  # Transform back to pixel positions and build new polygon
769  ccdPoints = [ccd.transform(ccd.makeCameraPoint(x, FOCAL_PLANE), PIXELS).getPoint() for x in intersect]
770  validPolygon = Polygon(ccdPoints)
771  ccdExposure.getInfo().setValidPolygon(validPolygon)
772 
773  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
774  """Apply brighter fatter correction in place for the image
775 
776  This correction takes a kernel that has been derived from flat field images to
777  redistribute the charge. The gradient of the kernel is the deflection
778  field due to the accumulated charge.
779 
780  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
781  using the following equation:
782 
783  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
784 
785  To evaluate the derivative term we expand it as follows:
786 
787  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
788 
789  Because we use the measured counts instead of the incident counts we apply the correction
790  iteratively to reconstruct the original counts and the correction. We stop iterating when the
791  summed difference between the current corrected image and the one from the previous iteration
792  is below the threshold. We do not require convergence because the number of iterations is
793  too large a computational cost. How we define the threshold still needs to be evaluated, the
794  current default was shown to work reasonably well on a small set of images. For more information
795  on the method see DocuShare Document-19407.
796 
797  The edges as defined by the kernel are not corrected because they have spurious values
798  due to the convolution.
799  """
800  self.log.info("Applying brighter fatter correction")
801 
802  image = exposure.getMaskedImage().getImage()
803 
804  # The image needs to be units of electrons/holes
805  with self.gainContext(exposure, image, applyGain):
806 
807  kLx = numpy.shape(kernel)[0]
808  kLy = numpy.shape(kernel)[1]
809  kernelImage = afwImage.ImageD(kLx, kLy)
810  kernelImage.getArray()[:, :] = kernel
811  tempImage = image.clone()
812 
813  nanIndex = numpy.isnan(tempImage.getArray())
814  tempImage.getArray()[nanIndex] = 0.
815 
816  outImage = afwImage.ImageF(image.getDimensions())
817  corr = numpy.zeros_like(image.getArray())
818  prev_image = numpy.zeros_like(image.getArray())
819  convCntrl = afwMath.ConvolutionControl(False, True, 1)
820  fixedKernel = afwMath.FixedKernel(kernelImage)
821 
822  # Define boundary by convolution region. The region that the correction will be
823  # calculated for is one fewer in each dimension because of the second derivative terms.
824  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
825  startX = kLx//2
826  endX = -kLx//2
827  startY = kLy//2
828  endY = -kLy//2
829 
830  for iteration in range(maxIter):
831 
832  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
833  tmpArray = tempImage.getArray()
834  outArray = outImage.getArray()
835 
836  # First derivative term
837  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
838  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
839  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
840 
841  # Second derivative term
842  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
843  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
844  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
845 
846  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
847 
848  tmpArray[:, :] = image.getArray()[:, :]
849  tmpArray[nanIndex] = 0.
850  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
851 
852  if iteration > 0:
853  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
854 
855  if diff < threshold:
856  break
857  prev_image[:, :] = tmpArray[:, :]
858 
859  if iteration == maxIter - 1:
860  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
861 
862  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
863  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
864  corr[startY + 1:endY - 1, startX + 1:endX - 1]
865 
866 
867  @contextmanager
868  def gainContext(self, exp, image, apply):
869  """Context manager that applies and removes gain
870  """
871  if apply:
872  ccd = exp.getDetector()
873  for amp in ccd:
874  sim = image.Factory(image, amp.getBBox())
875  sim *= amp.getGain()
876 
877  try:
878  yield exp
879  finally:
880  if apply:
881  ccd = exp.getDetector()
882  for amp in ccd:
883  sim = image.Factory(image, amp.getBBox())
884  sim /= amp.getGain()
885 
886 
887 class FakeAmp(object):
888  """A Detector-like object that supports returning gain and saturation level"""
889 
890  def __init__(self, exposure, config):
891  self._bbox = exposure.getBBox(afwImage.LOCAL)
892  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
893  self._gain = config.gain
894  self._readNoise = config.readNoise
895  self._saturation = config.saturation
896 
897  def getBBox(self):
898  return self._bbox
899 
900  def getRawBBox(self):
901  return self._bbox
902 
903  def getHasRawInfo(self):
904  return True # but see getRawHorizontalOverscanBBox()
905 
907  return self._RawHorizontalOverscanBBox
908 
909  def getGain(self):
910  return self._gain
911 
912  def getReadNoise(self):
913  return self._readNoise
914 
915  def getSaturation(self):
916  return self._saturation
917 
918  def getSuspectLevel(self):
919  return float("NaN")
def doLinearize
Is linearization wanted for this detector?
Definition: isrTask.py:555
def getIsrExposure
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:591
def maskAndInterpNan
Mask NaNs using mask plane &quot;UNMASKEDNAN&quot; and interpolate over them, in place.
Definition: isrTask.py:694
def flatCorrection
Apply flat correction in place.
Definition: isrTask.py:578
def run
Perform instrument signature removal on an exposure.
Definition: isrTask.py:373
def __init__
Constructor for IsrTask.
Definition: isrTask.py:319
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:245
def runDataRef
Perform instrument signature removal on a ButlerDataRef of a Sensor.
Definition: isrTask.py:488
def suspectDetection
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:648
def updateVariance
Set the variance plane based on the image plane, plus amplifier gain and read noise.
Definition: isrTask.py:565
def maskAndInterpDefect
Mask defects using mask plane &quot;BAD&quot; and interpolate over them, in place.
Definition: isrTask.py:673
def setValidPolygonIntersect
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:754
def overscanCorrection
Apply overscan correction, in place.
Definition: isrTask.py:727
def saturationDetection
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:616
def darkCorrection
Apply dark correction in place.
Definition: isrTask.py:536
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def saturationInterpolation
Interpolate over saturated pixels, in place.
Definition: isrTask.py:632
def biasCorrection
Apply bias correction in place.
Definition: isrTask.py:528
def readIsrData
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:330