lsst.ip.isr  13.0-12-g1758e1b
 All Classes Namespaces Files Functions Variables Typedefs Macros Groups
isrTask.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 from builtins import object
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 import numpy
27 
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.math as afwMath
34 from lsst.daf.persistence import ButlerDataRef
35 from lsstDebug import getDebugFrame
36 from lsst.afw.display import getDisplay
37 from . import isrFunctions
38 from .assembleCcdTask import AssembleCcdTask
39 from .fringe import FringeTask
40 from lsst.afw.geom.polygon import Polygon
41 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
42 from contextlib import contextmanager
43 from .isr import maskNans
44 
45 
46 class IsrTaskConfig(pexConfig.Config):
47  doBias = pexConfig.Field(
48  dtype=bool,
49  doc="Apply bias frame correction?",
50  default=True,
51  )
52  doDark = pexConfig.Field(
53  dtype=bool,
54  doc="Apply dark frame correction?",
55  default=True,
56  )
57  doFlat = pexConfig.Field(
58  dtype=bool,
59  doc="Apply flat field correction?",
60  default=True,
61  )
62  doFringe = pexConfig.Field(
63  dtype=bool,
64  doc="Apply fringe correction?",
65  default=True,
66  )
67  doDefect = pexConfig.Field(
68  dtype=bool,
69  doc="Apply correction for CCD defects, e.g. hot pixels?",
70  default=True,
71  )
72  doWrite = pexConfig.Field(
73  dtype=bool,
74  doc="Persist postISRCCD?",
75  default=True,
76  )
77  assembleCcd = pexConfig.ConfigurableField(
78  target=AssembleCcdTask,
79  doc="CCD assembly task",
80  )
81  gain = pexConfig.Field(
82  dtype=float,
83  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
84  default=float("NaN"),
85  )
86  readNoise = pexConfig.Field(
87  dtype=float,
88  doc="The read noise to use if no Detector is present in the Exposure",
89  default=0.0,
90  )
91  saturation = pexConfig.Field(
92  dtype=float,
93  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
94  default=float("NaN"),
95  )
96  fringeAfterFlat = pexConfig.Field(
97  dtype=bool,
98  doc="Do fringe subtraction after flat-fielding?",
99  default=True,
100  )
101  fringe = pexConfig.ConfigurableField(
102  target=FringeTask,
103  doc="Fringe subtraction task",
104  )
105  fwhm = pexConfig.Field(
106  dtype=float,
107  doc="FWHM of PSF (arcsec)",
108  default=1.0,
109  )
110  saturatedMaskName = pexConfig.Field(
111  dtype=str,
112  doc="Name of mask plane to use in saturation detection and interpolation",
113  default="SAT",
114  )
115  suspectMaskName = pexConfig.Field(
116  dtype=str,
117  doc="Name of mask plane to use for suspect pixels",
118  default="SUSPECT",
119  )
120  flatScalingType = pexConfig.ChoiceField(
121  dtype=str,
122  doc="The method for scaling the flat on the fly.",
123  default='USER',
124  allowed={
125  "USER": "Scale by flatUserScale",
126  "MEAN": "Scale by the inverse of the mean",
127  "MEDIAN": "Scale by the inverse of the median",
128  },
129  )
130  flatUserScale = pexConfig.Field(
131  dtype=float,
132  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
133  default=1.0,
134  )
135  overscanFitType = pexConfig.ChoiceField(
136  dtype=str,
137  doc="The method for fitting the overscan bias level.",
138  default='MEDIAN',
139  allowed={
140  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
141  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
142  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
143  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
144  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
145  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
146  "MEAN": "Correct using the mean of the overscan region",
147  "MEDIAN": "Correct using the median of the overscan region",
148  },
149  )
150  overscanOrder = pexConfig.Field(
151  dtype=int,
152  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
153  "or number of spline knots if overscan fit type is a spline."),
154  default=1,
155  )
156  overscanRej = pexConfig.Field(
157  dtype=float,
158  doc="Rejection threshold (sigma) for collapsing overscan before fit",
159  default=3.0,
160  )
161  growSaturationFootprintSize = pexConfig.Field(
162  dtype=int,
163  doc="Number of pixels by which to grow the saturation footprints",
164  default=1,
165  )
166  doSaturationInterpolation = pexConfig.Field(
167  dtype=bool,
168  doc="Perform interpolation over pixels masked as saturated?",
169  default=True,
170  )
171  fluxMag0T1 = pexConfig.Field(
172  dtype=float,
173  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
174  default=1e10,
175  )
176  keysToRemoveFromAssembledCcd = pexConfig.ListField(
177  dtype=str,
178  doc="fields to remove from the metadata of the assembled ccd.",
179  default=[],
180  )
181  doAssembleIsrExposures = pexConfig.Field(
182  dtype=bool,
183  default=False,
184  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
185  )
186  doAssembleCcd = pexConfig.Field(
187  dtype=bool,
188  default=True,
189  doc="Assemble amp-level exposures into a ccd-level exposure?"
190  )
191  doLinearize = pexConfig.Field(
192  dtype=bool,
193  doc="Correct for nonlinearity of the detector's response?",
194  default=True,
195  )
196  doBrighterFatter = pexConfig.Field(
197  dtype=bool,
198  default=False,
199  doc="Apply the brighter fatter correction"
200  )
201  brighterFatterKernelFile = pexConfig.Field(
202  dtype=str,
203  default='',
204  doc="Kernel file used for the brighter fatter correction"
205  )
206  brighterFatterMaxIter = pexConfig.Field(
207  dtype=int,
208  default=10,
209  doc="Maximum number of iterations for the brighter fatter correction"
210  )
211  brighterFatterThreshold = pexConfig.Field(
212  dtype=float,
213  default=1000,
214  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
215  " absolute value of the difference between the current corrected image and the one"
216  " from the previous iteration summed over all the pixels."
217  )
218  brighterFatterApplyGain = pexConfig.Field(
219  dtype=bool,
220  default=True,
221  doc="Should the gain be applied when applying the brighter fatter correction?"
222  )
223  datasetType = pexConfig.Field(
224  dtype=str,
225  doc="Dataset type for input data; users will typically leave this alone, "
226  "but camera-specific ISR tasks will override it",
227  default="raw",
228  )
229  fallbackFilterName = pexConfig.Field(dtype=str,
230  doc="Fallback default filter name for calibrations", optional=True)
231 
232 ## \addtogroup LSST_task_documentation
233 ## \{
234 ## \page IsrTask
235 ## \ref IsrTask_ "IsrTask"
236 ## \copybrief IsrTask
237 ## \}
238 
239 
240 class IsrTask(pipeBase.CmdLineTask):
241  """!
242  \anchor IsrTask_
243 
244  \brief Apply common instrument signature correction algorithms to a raw frame.
245 
246  \section ip_isr_isr_Contents Contents
247 
248  - \ref ip_isr_isr_Purpose
249  - \ref ip_isr_isr_Initialize
250  - \ref ip_isr_isr_IO
251  - \ref ip_isr_isr_Config
252  - \ref ip_isr_isr_Debug
253 
254 
255  \section ip_isr_isr_Purpose Description
256 
257  The process for correcting imaging data is very similar from camera to camera.
258  This task provides a vanilla implementation of doing these corrections, including
259  the ability to turn certain corrections off if they are not needed.
260  The inputs to the primary method, run, are a raw exposure to be corrected and the
261  calibration data products. The raw input is a single chip sized mosaic of all amps
262  including overscans and other non-science pixels.
263  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
264  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
265  This task may not meet all needs and it is expected that it will be subclassed for
266  specific applications.
267 
268  \section ip_isr_isr_Initialize Task initialization
269 
270  \copydoc \_\_init\_\_
271 
272  \section ip_isr_isr_IO Inputs/Outputs to the run method
273 
274  \copydoc run
275 
276  \section ip_isr_isr_Config Configuration parameters
277 
278  See \ref IsrTaskConfig
279 
280  \section ip_isr_isr_Debug Debug variables
281 
282  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
283  flag \c --debug, \c -d to import \b debug.py from your \c PYTHONPATH; see <a
284  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
285  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
286 
287  The available variables in IsrTask are:
288  <DL>
289  <DT> \c display
290  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
291  <DL>
292  <DT> postISRCCD
293  <DD> display exposure after ISR has been applied
294  </DL>
295  </DL>
296 
297  For example, put something like
298  \code{.py}
299  import lsstDebug
300  def DebugInfo(name):
301  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
302  if name == "lsst.ip.isrFunctions.isrTask":
303  di.display = {'postISRCCD':2}
304  return di
305  lsstDebug.Info = DebugInfo
306  \endcode
307  into your debug.py file and run the commandline task with the \c --debug flag.
308 
309  <HR>
310  """
311  ConfigClass = IsrTaskConfig
312  _DefaultName = "isr"
313 
314  def __init__(self, *args, **kwargs):
315  '''!Constructor for IsrTask
316  \param[in] *args -- a list of positional arguments passed on to the Task constructor
317  \param[in] **kwargs -- a dictionary of keyword arguments passed on to the Task constructor
318  Call the lsst.pipe.base.task.Task.__init__ method
319  Then setup the assembly and fringe correction subtasks
320  '''
321  pipeBase.Task.__init__(self, *args, **kwargs)
322  self.makeSubtask("assembleCcd")
323  self.makeSubtask("fringe")
324 
325  def readIsrData(self, dataRef, rawExposure):
326  """!Retrieve necessary frames for instrument signature removal
327  \param[in] dataRef -- a daf.persistence.butlerSubset.ButlerDataRef
328  of the detector data to be processed
329  \param[in] rawExposure -- a reference raw exposure that will later be
330  corrected with the retrieved calibration data;
331  should not be modified in this method.
332  \return a pipeBase.Struct with fields containing kwargs expected by run()
333  - bias: exposure of bias frame
334  - dark: exposure of dark frame
335  - flat: exposure of flat field
336  - defects: list of detects
337  - fringeStruct: a pipeBase.Struct with field fringes containing
338  exposure of fringe frame or list of fringe exposure
339  """
340  ccd = rawExposure.getDetector()
341 
342  biasExposure = self.getIsrExposure(dataRef, "bias") if self.config.doBias else None
343  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
344  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
345  darkExposure = self.getIsrExposure(dataRef, "dark") if self.config.doDark else None
346  flatExposure = self.getIsrExposure(dataRef, "flat") if self.config.doFlat else None
347  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
348  defectList = dataRef.get("defects") if self.config.doDefect else None
349 
350  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
351  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
352  if self.config.doAssembleIsrExposures else None)
353  else:
354  fringeStruct = pipeBase.Struct(fringes=None)
355 
356  # Struct should include only kwargs to run()
357  return pipeBase.Struct(bias=biasExposure,
358  linearizer=linearizer,
359  dark=darkExposure,
360  flat=flatExposure,
361  defects=defectList,
362  fringes=fringeStruct,
363  bfKernel=brighterFatterKernel
364  )
365 
366  @pipeBase.timeMethod
367  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
368  fringes=None, bfKernel=None):
369  """!Perform instrument signature removal on an exposure
370 
371  Steps include:
372  - Detect saturation, apply overscan correction, bias, dark and flat
373  - Perform CCD assembly
374  - Interpolate over defects, saturated pixels and all NaNs
375 
376  \param[in] ccdExposure -- lsst.afw.image.exposure of detector data
377  \param[in] bias -- exposure of bias frame
378  \param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
379  \param[in] dark -- exposure of dark frame
380  \param[in] flat -- exposure of flatfield
381  \param[in] defects -- list of detects
382  \param[in] fringes -- a pipeBase.Struct with field fringes containing
383  exposure of fringe frame or list of fringe exposure
384  \param[in] bfKernel -- kernel for brighter-fatter correction
385 
386  \return a pipeBase.Struct with field:
387  - exposure
388  """
389 
390  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
391  if isinstance(ccdExposure, ButlerDataRef):
392  return self.runDataRef(ccdExposure)
393 
394  ccd = ccdExposure.getDetector()
395 
396  # Validate Input
397  if self.config.doBias and bias is None:
398  raise RuntimeError("Must supply a bias exposure if config.doBias True")
399  if self.doLinearize(ccd) and linearizer is None:
400  raise RuntimeError("Must supply a linearizer if config.doBias True")
401  if self.config.doDark and dark is None:
402  raise RuntimeError("Must supply a dark exposure if config.doDark True")
403  if self.config.doFlat and flat is None:
404  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
405  if self.config.doBrighterFatter and bfKernel is None:
406  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
407  if fringes is None:
408  fringes = pipeBase.Struct(fringes=None)
409  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
410  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
411  if self.config.doDefect and defects is None:
412  raise RuntimeError("Must supply defects if config.doDefect True")
413 
414  ccdExposure = self.convertIntToFloat(ccdExposure)
415 
416  if not ccd:
417  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
418  ccd = [FakeAmp(ccdExposure, self.config)]
419 
420  for amp in ccd:
421  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
422  if ccdExposure.getBBox().contains(amp.getBBox()):
423  self.saturationDetection(ccdExposure, amp)
424  self.suspectDetection(ccdExposure, amp)
425  self.overscanCorrection(ccdExposure, amp)
426 
427  if self.config.doAssembleCcd:
428  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
429 
430  if self.config.doBias:
431  self.biasCorrection(ccdExposure, bias)
432 
433  if self.doLinearize(ccd):
434  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
435 
436  for amp in ccd:
437  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
438  if ccdExposure.getBBox().contains(amp.getBBox()):
439  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
440  self.updateVariance(ampExposure, amp)
441 
442  if self.config.doBrighterFatter:
443  self.brighterFatterCorrection(ccdExposure, bfKernel,
444  self.config.brighterFatterMaxIter,
445  self.config.brighterFatterThreshold,
446  self.config.brighterFatterApplyGain,
447  )
448 
449  if self.config.doDark:
450  self.darkCorrection(ccdExposure, dark)
451 
452  if self.config.doFringe and not self.config.fringeAfterFlat:
453  self.fringe.run(ccdExposure, **fringes.getDict())
454 
455  if self.config.doFlat:
456  self.flatCorrection(ccdExposure, flat)
457 
458  if self.config.doDefect:
459  self.maskAndInterpDefect(ccdExposure, defects)
460 
461  if self.config.doSaturationInterpolation:
462  self.saturationInterpolation(ccdExposure)
463 
464  self.maskAndInterpNan(ccdExposure)
465 
466  if self.config.doFringe and self.config.fringeAfterFlat:
467  self.fringe.run(ccdExposure, **fringes.getDict())
468 
469  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
470  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
471 
472  frame = getDebugFrame(self._display, "postISRCCD")
473  if frame:
474  getDisplay(frame).mtv(ccdExposure)
475 
476  return pipeBase.Struct(
477  exposure=ccdExposure,
478  )
479 
480  @pipeBase.timeMethod
481  def runDataRef(self, sensorRef):
482  """!Perform instrument signature removal on a ButlerDataRef of a Sensor
483 
484  - Read in necessary detrending/isr/calibration data
485  - Process raw exposure in run()
486  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
487 
488  \param[in] sensorRef -- daf.persistence.butlerSubset.ButlerDataRef of the
489  detector data to be processed
490  \return a pipeBase.Struct with fields:
491  - exposure: the exposure after application of ISR
492  """
493  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
494  ccdExposure = sensorRef.get('raw')
495  isrData = self.readIsrData(sensorRef, ccdExposure)
496 
497  result = self.run(ccdExposure, **isrData.getDict())
498 
499  if self.config.doWrite:
500  sensorRef.put(result.exposure, "postISRCCD")
501 
502  return result
503 
504  def convertIntToFloat(self, exposure):
505  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
506  """
507  if isinstance(exposure, afwImage.ExposureF):
508  # Nothing to be done
509  return exposure
510  if not hasattr(exposure, "convertF"):
511  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
512 
513  newexposure = exposure.convertF()
514  maskedImage = newexposure.getMaskedImage()
515  varArray = maskedImage.getVariance().getArray()
516  varArray[:, :] = 1
517  maskArray = maskedImage.getMask().getArray()
518  maskArray[:, :] = 0
519  return newexposure
520 
521  def biasCorrection(self, exposure, biasExposure):
522  """!Apply bias correction in place
523 
524  \param[in,out] exposure exposure to process
525  \param[in] biasExposure bias exposure of same size as exposure
526  """
527  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
528 
529  def darkCorrection(self, exposure, darkExposure):
530  """!Apply dark correction in place
531 
532  \param[in,out] exposure exposure to process
533  \param[in] darkExposure dark exposure of same size as exposure
534  """
535  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
536  if math.isnan(expScale):
537  raise RuntimeError("Exposure darktime is NAN")
538  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
539  if math.isnan(darkScale):
540  raise RuntimeError("Dark calib darktime is NAN")
541  isrFunctions.darkCorrection(
542  maskedImage=exposure.getMaskedImage(),
543  darkMaskedImage=darkExposure.getMaskedImage(),
544  expScale=expScale,
545  darkScale=darkScale,
546  )
547 
548  def doLinearize(self, detector):
549  """!Is linearization wanted for this detector?
550 
551  Checks config.doLinearize and the linearity type of the first amplifier.
552 
553  \param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
554  """
555  return self.config.doLinearize and \
556  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
557 
558  def updateVariance(self, ampExposure, amp):
559  """!Set the variance plane based on the image plane, plus amplifier gain and read noise
560 
561  \param[in,out] ampExposure exposure to process
562  \param[in] amp amplifier detector information
563  """
564  if not math.isnan(amp.getGain()):
565  isrFunctions.updateVariance(
566  maskedImage=ampExposure.getMaskedImage(),
567  gain=amp.getGain(),
568  readNoise=amp.getReadNoise(),
569  )
570 
571  def flatCorrection(self, exposure, flatExposure):
572  """!Apply flat correction in place
573 
574  \param[in,out] exposure exposure to process
575  \param[in] flatExposure flatfield exposure same size as exposure
576  """
577  isrFunctions.flatCorrection(
578  maskedImage=exposure.getMaskedImage(),
579  flatMaskedImage=flatExposure.getMaskedImage(),
580  scalingType=self.config.flatScalingType,
581  userScale=self.config.flatUserScale,
582  )
583 
584  def getIsrExposure(self, dataRef, datasetType, immediate=True):
585  """!Retrieve a calibration dataset for removing instrument signature
586 
587  \param[in] dataRef data reference for exposure
588  \param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
589  \param[in] immediate if True, disable butler proxies to enable error
590  handling within this routine
591  \return exposure
592  """
593  try:
594  exp = dataRef.get(datasetType, immediate=immediate)
595  except Exception as exc1:
596  if not self.config.fallbackFilterName:
597  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
598  try:
599  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
600  except Exception as exc2:
601  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
602  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
603  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
604 
605  if self.config.doAssembleIsrExposures:
606  exp = self.assembleCcd.assembleCcd(exp)
607  return exp
608 
609  def saturationDetection(self, exposure, amp):
610  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
611 
612  \param[in,out] exposure exposure to process; only the amp DataSec is processed
613  \param[in] amp amplifier device data
614  """
615  if not math.isnan(amp.getSaturation()):
616  maskedImage = exposure.getMaskedImage()
617  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
618  isrFunctions.makeThresholdMask(
619  maskedImage=dataView,
620  threshold=amp.getSaturation(),
621  growFootprints=0,
622  maskName=self.config.saturatedMaskName,
623  )
624 
625  def saturationInterpolation(self, ccdExposure):
626  """!Interpolate over saturated pixels, in place
627 
628  \param[in,out] ccdExposure exposure to process
629 
630  \warning:
631  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
632  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
633  """
634  isrFunctions.interpolateFromMask(
635  maskedImage=ccdExposure.getMaskedImage(),
636  fwhm=self.config.fwhm,
637  growFootprints=self.config.growSaturationFootprintSize,
638  maskName=self.config.saturatedMaskName,
639  )
640 
641  def suspectDetection(self, exposure, amp):
642  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
643 
644  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
645  This is intended to indicate pixels that may be affected by unknown systematics;
646  for example if non-linearity corrections above a certain level are unstable
647  then that would be a useful value for suspectLevel. A value of `nan` indicates
648  that no such level exists and no pixels are to be masked as suspicious.
649 
650  \param[in,out] exposure exposure to process; only the amp DataSec is processed
651  \param[in] amp amplifier device data
652  """
653  suspectLevel = amp.getSuspectLevel()
654  if math.isnan(suspectLevel):
655  return
656 
657  maskedImage = exposure.getMaskedImage()
658  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
659  isrFunctions.makeThresholdMask(
660  maskedImage=dataView,
661  threshold=suspectLevel,
662  growFootprints=0,
663  maskName=self.config.suspectMaskName,
664  )
665 
666  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
667  """!Mask defects using mask plane "BAD" and interpolate over them, in place
668 
669  \param[in,out] ccdExposure exposure to process
670  \param[in] defectBaseList a list of defects to mask and interpolate
671 
672  \warning: call this after CCD assembly, since defects may cross amplifier boundaries
673  """
674  maskedImage = ccdExposure.getMaskedImage()
675  defectList = []
676  for d in defectBaseList:
677  bbox = d.getBBox()
678  nd = measAlg.Defect(bbox)
679  defectList.append(nd)
680  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
681  isrFunctions.interpolateDefectList(
682  maskedImage=maskedImage,
683  defectList=defectList,
684  fwhm=self.config.fwhm,
685  )
686 
687  def maskAndInterpNan(self, exposure):
688  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
689 
690  We mask and interpolate over all NaNs, including those
691  that are masked with other bits (because those may or may
692  not be interpolated over later, and we want to remove all
693  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
694  is used to preserve the historical name.
695 
696  \param[in,out] exposure exposure to process
697  """
698  maskedImage = exposure.getMaskedImage()
699 
700  # Find and mask NaNs
701  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
702  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
703  numNans = maskNans(maskedImage, maskVal)
704  self.metadata.set("NUMNANS", numNans)
705 
706  # Interpolate over these previously-unmasked NaNs
707  if numNans > 0:
708  self.log.warn("There were %i unmasked NaNs", numNans)
709  nanDefectList = isrFunctions.getDefectListFromMask(
710  maskedImage=maskedImage,
711  maskName='UNMASKEDNAN',
712  growFootprints=0,
713  )
714  isrFunctions.interpolateDefectList(
715  maskedImage=exposure.getMaskedImage(),
716  defectList=nanDefectList,
717  fwhm=self.config.fwhm,
718  )
719 
720  def overscanCorrection(self, exposure, amp):
721  """!Apply overscan correction, in place
722 
723  \param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
724  \param[in] amp amplifier device data
725  """
726  if not amp.getHasRawInfo():
727  raise RuntimeError("This method must be executed on an amp with raw information.")
728 
729  if amp.getRawHorizontalOverscanBBox().isEmpty():
730  self.log.info("No Overscan region. Not performing Overscan Correction.")
731  return None
732 
733  maskedImage = exposure.getMaskedImage()
734  dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
735 
736  expImage = exposure.getMaskedImage().getImage()
737  overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
738 
739  isrFunctions.overscanCorrection(
740  ampMaskedImage=dataView,
741  overscanImage=overscanImage,
742  fitType=self.config.overscanFitType,
743  order=self.config.overscanOrder,
744  collapseRej=self.config.overscanRej,
745  )
746 
747  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
748  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
749 
750  \param[in,out] ccdExposure exposure to process
751  \param[in] fpPolygon Polygon in focal plane coordinates
752  """
753  # Get ccd corners in focal plane coordinates
754  ccd = ccdExposure.getDetector()
755  fpCorners = ccd.getCorners(FOCAL_PLANE)
756  ccdPolygon = Polygon(fpCorners)
757 
758  # Get intersection of ccd corners with fpPolygon
759  intersect = ccdPolygon.intersectionSingle(fpPolygon)
760 
761  # Transform back to pixel positions and build new polygon
762  ccdPoints = [ccd.transform(ccd.makeCameraPoint(x, FOCAL_PLANE), PIXELS).getPoint() for x in intersect]
763  validPolygon = Polygon(ccdPoints)
764  ccdExposure.getInfo().setValidPolygon(validPolygon)
765 
766  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
767  """Apply brighter fatter correction in place for the image
768 
769  This correction takes a kernel that has been derived from flat field images to
770  redistribute the charge. The gradient of the kernel is the deflection
771  field due to the accumulated charge.
772 
773  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
774  using the following equation:
775 
776  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
777 
778  To evaluate the derivative term we expand it as follows:
779 
780  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
781 
782  Because we use the measured counts instead of the incident counts we apply the correction
783  iteratively to reconstruct the original counts and the correction. We stop iterating when the
784  summed difference between the current corrected image and the one from the previous iteration
785  is below the threshold. We do not require convergence because the number of iterations is
786  too large a computational cost. How we define the threshold still needs to be evaluated, the
787  current default was shown to work reasonably well on a small set of images. For more information
788  on the method see DocuShare Document-19407.
789 
790  The edges as defined by the kernel are not corrected because they have spurious values
791  due to the convolution.
792  """
793  self.log.info("Applying brighter fatter correction")
794 
795  image = exposure.getMaskedImage().getImage()
796 
797  # The image needs to be units of electrons/holes
798  with self.gainContext(exposure, image, applyGain):
799 
800  kLx = numpy.shape(kernel)[0]
801  kLy = numpy.shape(kernel)[1]
802  kernelImage = afwImage.ImageD(kLx, kLy)
803  kernelImage.getArray()[:, :] = kernel
804  tempImage = image.clone()
805 
806  nanIndex = numpy.isnan(tempImage.getArray())
807  tempImage.getArray()[nanIndex] = 0.
808 
809  outImage = afwImage.ImageF(image.getDimensions())
810  corr = numpy.zeros_like(image.getArray())
811  prev_image = numpy.zeros_like(image.getArray())
812  convCntrl = afwMath.ConvolutionControl(False, True, 1)
813  fixedKernel = afwMath.FixedKernel(kernelImage)
814 
815  # Define boundary by convolution region. The region that the correction will be
816  # calculated for is one fewer in each dimension because of the second derivative terms.
817  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
818  startX = kLx//2
819  endX = -kLx//2
820  startY = kLy//2
821  endY = -kLy//2
822 
823  for iteration in range(maxIter):
824 
825  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
826  tmpArray = tempImage.getArray()
827  outArray = outImage.getArray()
828 
829  # First derivative term
830  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
831  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
832  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
833 
834  # Second derivative term
835  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
836  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
837  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
838 
839  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
840 
841  tmpArray[:, :] = image.getArray()[:, :]
842  tmpArray[nanIndex] = 0.
843  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
844 
845  if iteration > 0:
846  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
847 
848  if diff < threshold:
849  break
850  prev_image[:, :] = tmpArray[:, :]
851 
852  if iteration == maxIter - 1:
853  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
854 
855  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
856  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
857  corr[startY + 1:endY - 1, startX + 1:endX - 1]
858 
859 
860  @contextmanager
861  def gainContext(self, exp, image, apply):
862  """Context manager that applies and removes gain
863  """
864  if apply:
865  ccd = exp.getDetector()
866  for amp in ccd:
867  sim = image.Factory(image, amp.getBBox())
868  sim *= amp.getGain()
869 
870  try:
871  yield exp
872  finally:
873  if apply:
874  ccd = exp.getDetector()
875  for amp in ccd:
876  sim = image.Factory(image, amp.getBBox())
877  sim /= amp.getGain()
878 
879 
880 class FakeAmp(object):
881  """A Detector-like object that supports returning gain and saturation level"""
882 
883  def __init__(self, exposure, config):
884  self._bbox = exposure.getBBox(afwImage.LOCAL)
885  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
886  self._gain = config.gain
887  self._readNoise = config.readNoise
888  self._saturation = config.saturation
889 
890  def getBBox(self):
891  return self._bbox
892 
893  def getRawBBox(self):
894  return self._bbox
895 
896  def getHasRawInfo(self):
897  return True # but see getRawHorizontalOverscanBBox()
898 
900  return self._RawHorizontalOverscanBBox
901 
902  def getGain(self):
903  return self._gain
904 
905  def getReadNoise(self):
906  return self._readNoise
907 
908  def getSaturation(self):
909  return self._saturation
910 
911  def getSuspectLevel(self):
912  return float("NaN")
def doLinearize
Is linearization wanted for this detector?
Definition: isrTask.py:548
def getIsrExposure
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:584
def maskAndInterpNan
Mask NaNs using mask plane &quot;UNMASKEDNAN&quot; and interpolate over them, in place.
Definition: isrTask.py:687
def flatCorrection
Apply flat correction in place.
Definition: isrTask.py:571
def run
Perform instrument signature removal on an exposure.
Definition: isrTask.py:368
def __init__
Constructor for IsrTask.
Definition: isrTask.py:314
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:240
def runDataRef
Perform instrument signature removal on a ButlerDataRef of a Sensor.
Definition: isrTask.py:481
def suspectDetection
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:641
def updateVariance
Set the variance plane based on the image plane, plus amplifier gain and read noise.
Definition: isrTask.py:558
def maskAndInterpDefect
Mask defects using mask plane &quot;BAD&quot; and interpolate over them, in place.
Definition: isrTask.py:666
def setValidPolygonIntersect
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:747
def overscanCorrection
Apply overscan correction, in place.
Definition: isrTask.py:720
def saturationDetection
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:609
def darkCorrection
Apply dark correction in place.
Definition: isrTask.py:529
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def saturationInterpolation
Interpolate over saturated pixels, in place.
Definition: isrTask.py:625
def biasCorrection
Apply bias correction in place.
Definition: isrTask.py:521
def readIsrData
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:325