lsst.ip.isr  13.0-15-g0af5a6c+16
isrTask.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 from builtins import object
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 import numpy
27 
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.math as afwMath
34 from lsst.daf.persistence import ButlerDataRef
35 from lsstDebug import getDebugFrame
36 from lsst.afw.display import getDisplay
37 from . import isrFunctions
38 from .assembleCcdTask import AssembleCcdTask
39 from .fringe import FringeTask
40 from lsst.afw.geom.polygon import Polygon
41 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
42 from contextlib import contextmanager
43 from .isr import maskNans
44 from .crosstalk import CrosstalkTask
45 
46 
47 class IsrTaskConfig(pexConfig.Config):
48  doBias = pexConfig.Field(
49  dtype=bool,
50  doc="Apply bias frame correction?",
51  default=True,
52  )
53  doDark = pexConfig.Field(
54  dtype=bool,
55  doc="Apply dark frame correction?",
56  default=True,
57  )
58  doFlat = pexConfig.Field(
59  dtype=bool,
60  doc="Apply flat field correction?",
61  default=True,
62  )
63  doFringe = pexConfig.Field(
64  dtype=bool,
65  doc="Apply fringe correction?",
66  default=True,
67  )
68  doDefect = pexConfig.Field(
69  dtype=bool,
70  doc="Apply correction for CCD defects, e.g. hot pixels?",
71  default=True,
72  )
73  doWrite = pexConfig.Field(
74  dtype=bool,
75  doc="Persist postISRCCD?",
76  default=True,
77  )
78  assembleCcd = pexConfig.ConfigurableField(
79  target=AssembleCcdTask,
80  doc="CCD assembly task",
81  )
82  gain = pexConfig.Field(
83  dtype=float,
84  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
85  default=float("NaN"),
86  )
87  readNoise = pexConfig.Field(
88  dtype=float,
89  doc="The read noise to use if no Detector is present in the Exposure",
90  default=0.0,
91  )
92  saturation = pexConfig.Field(
93  dtype=float,
94  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
95  default=float("NaN"),
96  )
97  fringeAfterFlat = pexConfig.Field(
98  dtype=bool,
99  doc="Do fringe subtraction after flat-fielding?",
100  default=True,
101  )
102  fringe = pexConfig.ConfigurableField(
103  target=FringeTask,
104  doc="Fringe subtraction task",
105  )
106  fwhm = pexConfig.Field(
107  dtype=float,
108  doc="FWHM of PSF (arcsec)",
109  default=1.0,
110  )
111  saturatedMaskName = pexConfig.Field(
112  dtype=str,
113  doc="Name of mask plane to use in saturation detection and interpolation",
114  default="SAT",
115  )
116  suspectMaskName = pexConfig.Field(
117  dtype=str,
118  doc="Name of mask plane to use for suspect pixels",
119  default="SUSPECT",
120  )
121  flatScalingType = pexConfig.ChoiceField(
122  dtype=str,
123  doc="The method for scaling the flat on the fly.",
124  default='USER',
125  allowed={
126  "USER": "Scale by flatUserScale",
127  "MEAN": "Scale by the inverse of the mean",
128  "MEDIAN": "Scale by the inverse of the median",
129  },
130  )
131  flatUserScale = pexConfig.Field(
132  dtype=float,
133  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
134  default=1.0,
135  )
136  overscanFitType = pexConfig.ChoiceField(
137  dtype=str,
138  doc="The method for fitting the overscan bias level.",
139  default='MEDIAN',
140  allowed={
141  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
142  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
143  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
144  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
145  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
146  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
147  "MEAN": "Correct using the mean of the overscan region",
148  "MEDIAN": "Correct using the median of the overscan region",
149  },
150  )
151  overscanOrder = pexConfig.Field(
152  dtype=int,
153  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
154  "or number of spline knots if overscan fit type is a spline."),
155  default=1,
156  )
157  overscanRej = pexConfig.Field(
158  dtype=float,
159  doc="Rejection threshold (sigma) for collapsing overscan before fit",
160  default=3.0,
161  )
162  growSaturationFootprintSize = pexConfig.Field(
163  dtype=int,
164  doc="Number of pixels by which to grow the saturation footprints",
165  default=1,
166  )
167  doSaturationInterpolation = pexConfig.Field(
168  dtype=bool,
169  doc="Perform interpolation over pixels masked as saturated?",
170  default=True,
171  )
172  fluxMag0T1 = pexConfig.Field(
173  dtype=float,
174  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
175  default=1e10,
176  )
177  keysToRemoveFromAssembledCcd = pexConfig.ListField(
178  dtype=str,
179  doc="fields to remove from the metadata of the assembled ccd.",
180  default=[],
181  )
182  doAssembleIsrExposures = pexConfig.Field(
183  dtype=bool,
184  default=False,
185  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
186  )
187  doAssembleCcd = pexConfig.Field(
188  dtype=bool,
189  default=True,
190  doc="Assemble amp-level exposures into a ccd-level exposure?"
191  )
192  expectWcs = pexConfig.Field(
193  dtype=bool,
194  default=True,
195  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)"
196  )
197  doLinearize = pexConfig.Field(
198  dtype=bool,
199  doc="Correct for nonlinearity of the detector's response?",
200  default=True,
201  )
202  doCrosstalk = pexConfig.Field(
203  dtype=bool,
204  doc="Apply intra-CCD crosstalk correction?",
205  default=False,
206  )
207  crosstalk = pexConfig.ConfigurableField(
208  target=CrosstalkTask,
209  doc="Intra-CCD crosstalk correction",
210  )
211  doBrighterFatter = pexConfig.Field(
212  dtype=bool,
213  default=False,
214  doc="Apply the brighter fatter correction"
215  )
216  brighterFatterKernelFile = pexConfig.Field(
217  dtype=str,
218  default='',
219  doc="Kernel file used for the brighter fatter correction"
220  )
221  brighterFatterMaxIter = pexConfig.Field(
222  dtype=int,
223  default=10,
224  doc="Maximum number of iterations for the brighter fatter correction"
225  )
226  brighterFatterThreshold = pexConfig.Field(
227  dtype=float,
228  default=1000,
229  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
230  " absolute value of the difference between the current corrected image and the one"
231  " from the previous iteration summed over all the pixels."
232  )
233  brighterFatterApplyGain = pexConfig.Field(
234  dtype=bool,
235  default=True,
236  doc="Should the gain be applied when applying the brighter fatter correction?"
237  )
238  datasetType = pexConfig.Field(
239  dtype=str,
240  doc="Dataset type for input data; users will typically leave this alone, "
241  "but camera-specific ISR tasks will override it",
242  default="raw",
243  )
244  fallbackFilterName = pexConfig.Field(dtype=str,
245  doc="Fallback default filter name for calibrations", optional=True)
246 
247 
253 
254 
255 class IsrTask(pipeBase.CmdLineTask):
256  """!
257  \anchor IsrTask_
258 
259  \brief Apply common instrument signature correction algorithms to a raw frame.
260 
261  \section ip_isr_isr_Contents Contents
262 
263  - \ref ip_isr_isr_Purpose
264  - \ref ip_isr_isr_Initialize
265  - \ref ip_isr_isr_IO
266  - \ref ip_isr_isr_Config
267  - \ref ip_isr_isr_Debug
268 
269 
270  \section ip_isr_isr_Purpose Description
271 
272  The process for correcting imaging data is very similar from camera to camera.
273  This task provides a vanilla implementation of doing these corrections, including
274  the ability to turn certain corrections off if they are not needed.
275  The inputs to the primary method, run, are a raw exposure to be corrected and the
276  calibration data products. The raw input is a single chip sized mosaic of all amps
277  including overscans and other non-science pixels.
278  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
279  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
280  This task may not meet all needs and it is expected that it will be subclassed for
281  specific applications.
282 
283  \section ip_isr_isr_Initialize Task initialization
284 
285  \copydoc \_\_init\_\_
286 
287  \section ip_isr_isr_IO Inputs/Outputs to the run method
288 
289  \copydoc run
290 
291  \section ip_isr_isr_Config Configuration parameters
292 
293  See \ref IsrTaskConfig
294 
295  \section ip_isr_isr_Debug Debug variables
296 
297  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
298  flag \c --debug, \c -d to import \b debug.py from your \c PYTHONPATH; see <a
299  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
300  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
301 
302  The available variables in IsrTask are:
303  <DL>
304  <DT> \c display
305  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
306  <DL>
307  <DT> postISRCCD
308  <DD> display exposure after ISR has been applied
309  </DL>
310  </DL>
311 
312  For example, put something like
313  \code{.py}
314  import lsstDebug
315  def DebugInfo(name):
316  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
317  if name == "lsst.ip.isrFunctions.isrTask":
318  di.display = {'postISRCCD':2}
319  return di
320  lsstDebug.Info = DebugInfo
321  \endcode
322  into your debug.py file and run the commandline task with the \c --debug flag.
323 
324  <HR>
325  """
326  ConfigClass = IsrTaskConfig
327  _DefaultName = "isr"
328 
329  def __init__(self, *args, **kwargs):
330  '''!Constructor for IsrTask
331  \param[in] *args -- a list of positional arguments passed on to the Task constructor
332  \param[in] **kwargs -- a dictionary of keyword arguments passed on to the Task constructor
333  Call the lsst.pipe.base.task.Task.__init__ method
334  Then setup the assembly and fringe correction subtasks
335  '''
336  pipeBase.Task.__init__(self, *args, **kwargs)
337  self.makeSubtask("assembleCcd")
338  self.makeSubtask("fringe")
339  self.makeSubtask("crosstalk")
340 
341  def readIsrData(self, dataRef, rawExposure):
342  """!Retrieve necessary frames for instrument signature removal
343  \param[in] dataRef -- a daf.persistence.butlerSubset.ButlerDataRef
344  of the detector data to be processed
345  \param[in] rawExposure -- a reference raw exposure that will later be
346  corrected with the retrieved calibration data;
347  should not be modified in this method.
348  \return a pipeBase.Struct with fields containing kwargs expected by run()
349  - bias: exposure of bias frame
350  - dark: exposure of dark frame
351  - flat: exposure of flat field
352  - defects: list of detects
353  - fringeStruct: a pipeBase.Struct with field fringes containing
354  exposure of fringe frame or list of fringe exposure
355  """
356  ccd = rawExposure.getDetector()
357 
358  biasExposure = self.getIsrExposure(dataRef, "bias") if self.config.doBias else None
359  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
360  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
361  darkExposure = self.getIsrExposure(dataRef, "dark") if self.config.doDark else None
362  flatExposure = self.getIsrExposure(dataRef, "flat") if self.config.doFlat else None
363  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
364  defectList = dataRef.get("defects") if self.config.doDefect else None
365 
366  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
367  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
368  if self.config.doAssembleIsrExposures else None)
369  else:
370  fringeStruct = pipeBase.Struct(fringes=None)
371 
372  # Struct should include only kwargs to run()
373  return pipeBase.Struct(bias=biasExposure,
374  linearizer=linearizer,
375  dark=darkExposure,
376  flat=flatExposure,
377  defects=defectList,
378  fringes=fringeStruct,
379  bfKernel=brighterFatterKernel
380  )
381 
382  @pipeBase.timeMethod
383  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
384  fringes=None, bfKernel=None):
385  """!Perform instrument signature removal on an exposure
386 
387  Steps include:
388  - Detect saturation, apply overscan correction, bias, dark and flat
389  - Perform CCD assembly
390  - Interpolate over defects, saturated pixels and all NaNs
391 
392  \param[in] ccdExposure -- lsst.afw.image.exposure of detector data
393  \param[in] bias -- exposure of bias frame
394  \param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
395  \param[in] dark -- exposure of dark frame
396  \param[in] flat -- exposure of flatfield
397  \param[in] defects -- list of detects
398  \param[in] fringes -- a pipeBase.Struct with field fringes containing
399  exposure of fringe frame or list of fringe exposure
400  \param[in] bfKernel -- kernel for brighter-fatter correction
401 
402  \return a pipeBase.Struct with field:
403  - exposure
404  """
405 
406  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
407  if isinstance(ccdExposure, ButlerDataRef):
408  return self.runDataRef(ccdExposure)
409 
410  ccd = ccdExposure.getDetector()
411 
412  # Validate Input
413  if self.config.doBias and bias is None:
414  raise RuntimeError("Must supply a bias exposure if config.doBias True")
415  if self.doLinearize(ccd) and linearizer is None:
416  raise RuntimeError("Must supply a linearizer if config.doBias True")
417  if self.config.doDark and dark is None:
418  raise RuntimeError("Must supply a dark exposure if config.doDark True")
419  if self.config.doFlat and flat is None:
420  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
421  if self.config.doBrighterFatter and bfKernel is None:
422  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
423  if fringes is None:
424  fringes = pipeBase.Struct(fringes=None)
425  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
426  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
427  if self.config.doDefect and defects is None:
428  raise RuntimeError("Must supply defects if config.doDefect True")
429 
430  ccdExposure = self.convertIntToFloat(ccdExposure)
431 
432  if not ccd:
433  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
434  ccd = [FakeAmp(ccdExposure, self.config)]
435 
436  for amp in ccd:
437  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
438  if ccdExposure.getBBox().contains(amp.getBBox()):
439  self.saturationDetection(ccdExposure, amp)
440  self.suspectDetection(ccdExposure, amp)
441  self.overscanCorrection(ccdExposure, amp)
442 
443  if self.config.doAssembleCcd:
444  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
445  if self.config.expectWcs and not ccdExposure.getWcs():
446  self.log.warn("No WCS found in input exposure")
447 
448  if self.config.doBias:
449  self.biasCorrection(ccdExposure, bias)
450 
451  if self.doLinearize(ccd):
452  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
453 
454  if self.config.doCrosstalk:
455  self.crosstalk.run(ccdExposure)
456 
457  for amp in ccd:
458  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
459  if ccdExposure.getBBox().contains(amp.getBBox()):
460  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
461  self.updateVariance(ampExposure, amp)
462 
463  if self.config.doBrighterFatter:
464  self.brighterFatterCorrection(ccdExposure, bfKernel,
465  self.config.brighterFatterMaxIter,
466  self.config.brighterFatterThreshold,
467  self.config.brighterFatterApplyGain,
468  )
469 
470  if self.config.doDark:
471  self.darkCorrection(ccdExposure, dark)
472 
473  if self.config.doFringe and not self.config.fringeAfterFlat:
474  self.fringe.run(ccdExposure, **fringes.getDict())
475 
476  if self.config.doFlat:
477  self.flatCorrection(ccdExposure, flat)
478 
479  if self.config.doDefect:
480  self.maskAndInterpDefect(ccdExposure, defects)
481 
482  if self.config.doSaturationInterpolation:
483  self.saturationInterpolation(ccdExposure)
484 
485  self.maskAndInterpNan(ccdExposure)
486 
487  if self.config.doFringe and self.config.fringeAfterFlat:
488  self.fringe.run(ccdExposure, **fringes.getDict())
489 
490  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
491  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
492 
493  frame = getDebugFrame(self._display, "postISRCCD")
494  if frame:
495  getDisplay(frame).mtv(ccdExposure)
496 
497  return pipeBase.Struct(
498  exposure=ccdExposure,
499  )
500 
501  @pipeBase.timeMethod
502  def runDataRef(self, sensorRef):
503  """!Perform instrument signature removal on a ButlerDataRef of a Sensor
504 
505  - Read in necessary detrending/isr/calibration data
506  - Process raw exposure in run()
507  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
508 
509  \param[in] sensorRef -- daf.persistence.butlerSubset.ButlerDataRef of the
510  detector data to be processed
511  \return a pipeBase.Struct with fields:
512  - exposure: the exposure after application of ISR
513  """
514  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
515  ccdExposure = sensorRef.get('raw')
516  isrData = self.readIsrData(sensorRef, ccdExposure)
517 
518  result = self.run(ccdExposure, **isrData.getDict())
519 
520  if self.config.doWrite:
521  sensorRef.put(result.exposure, "postISRCCD")
522 
523  return result
524 
525  def convertIntToFloat(self, exposure):
526  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
527  """
528  if isinstance(exposure, afwImage.ExposureF):
529  # Nothing to be done
530  return exposure
531  if not hasattr(exposure, "convertF"):
532  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
533 
534  newexposure = exposure.convertF()
535  maskedImage = newexposure.getMaskedImage()
536  varArray = maskedImage.getVariance().getArray()
537  varArray[:, :] = 1
538  maskArray = maskedImage.getMask().getArray()
539  maskArray[:, :] = 0
540  return newexposure
541 
542  def biasCorrection(self, exposure, biasExposure):
543  """!Apply bias correction in place
544 
545  \param[in,out] exposure exposure to process
546  \param[in] biasExposure bias exposure of same size as exposure
547  """
548  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
549 
550  def darkCorrection(self, exposure, darkExposure):
551  """!Apply dark correction in place
552 
553  \param[in,out] exposure exposure to process
554  \param[in] darkExposure dark exposure of same size as exposure
555  """
556  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
557  if math.isnan(expScale):
558  raise RuntimeError("Exposure darktime is NAN")
559  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
560  if math.isnan(darkScale):
561  raise RuntimeError("Dark calib darktime is NAN")
562  isrFunctions.darkCorrection(
563  maskedImage=exposure.getMaskedImage(),
564  darkMaskedImage=darkExposure.getMaskedImage(),
565  expScale=expScale,
566  darkScale=darkScale,
567  )
568 
569  def doLinearize(self, detector):
570  """!Is linearization wanted for this detector?
571 
572  Checks config.doLinearize and the linearity type of the first amplifier.
573 
574  \param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
575  """
576  return self.config.doLinearize and \
577  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
578 
579  def updateVariance(self, ampExposure, amp):
580  """!Set the variance plane based on the image plane, plus amplifier gain and read noise
581 
582  \param[in,out] ampExposure exposure to process
583  \param[in] amp amplifier detector information
584  """
585  if not math.isnan(amp.getGain()):
586  isrFunctions.updateVariance(
587  maskedImage=ampExposure.getMaskedImage(),
588  gain=amp.getGain(),
589  readNoise=amp.getReadNoise(),
590  )
591 
592  def flatCorrection(self, exposure, flatExposure):
593  """!Apply flat correction in place
594 
595  \param[in,out] exposure exposure to process
596  \param[in] flatExposure flatfield exposure same size as exposure
597  """
598  isrFunctions.flatCorrection(
599  maskedImage=exposure.getMaskedImage(),
600  flatMaskedImage=flatExposure.getMaskedImage(),
601  scalingType=self.config.flatScalingType,
602  userScale=self.config.flatUserScale,
603  )
604 
605  def getIsrExposure(self, dataRef, datasetType, immediate=True):
606  """!Retrieve a calibration dataset for removing instrument signature
607 
608  \param[in] dataRef data reference for exposure
609  \param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
610  \param[in] immediate if True, disable butler proxies to enable error
611  handling within this routine
612  \return exposure
613  """
614  try:
615  exp = dataRef.get(datasetType, immediate=immediate)
616  except Exception as exc1:
617  if not self.config.fallbackFilterName:
618  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
619  try:
620  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
621  except Exception as exc2:
622  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
623  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
624  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
625 
626  if self.config.doAssembleIsrExposures:
627  exp = self.assembleCcd.assembleCcd(exp)
628  return exp
629 
630  def saturationDetection(self, exposure, amp):
631  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
632 
633  \param[in,out] exposure exposure to process; only the amp DataSec is processed
634  \param[in] amp amplifier device data
635  """
636  if not math.isnan(amp.getSaturation()):
637  maskedImage = exposure.getMaskedImage()
638  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
639  isrFunctions.makeThresholdMask(
640  maskedImage=dataView,
641  threshold=amp.getSaturation(),
642  growFootprints=0,
643  maskName=self.config.saturatedMaskName,
644  )
645 
646  def saturationInterpolation(self, ccdExposure):
647  """!Interpolate over saturated pixels, in place
648 
649  \param[in,out] ccdExposure exposure to process
650 
651  \warning:
652  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
653  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
654  """
655  isrFunctions.interpolateFromMask(
656  maskedImage=ccdExposure.getMaskedImage(),
657  fwhm=self.config.fwhm,
658  growFootprints=self.config.growSaturationFootprintSize,
659  maskName=self.config.saturatedMaskName,
660  )
661 
662  def suspectDetection(self, exposure, amp):
663  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
664 
665  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
666  This is intended to indicate pixels that may be affected by unknown systematics;
667  for example if non-linearity corrections above a certain level are unstable
668  then that would be a useful value for suspectLevel. A value of `nan` indicates
669  that no such level exists and no pixels are to be masked as suspicious.
670 
671  \param[in,out] exposure exposure to process; only the amp DataSec is processed
672  \param[in] amp amplifier device data
673  """
674  suspectLevel = amp.getSuspectLevel()
675  if math.isnan(suspectLevel):
676  return
677 
678  maskedImage = exposure.getMaskedImage()
679  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
680  isrFunctions.makeThresholdMask(
681  maskedImage=dataView,
682  threshold=suspectLevel,
683  growFootprints=0,
684  maskName=self.config.suspectMaskName,
685  )
686 
687  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
688  """!Mask defects using mask plane "BAD" and interpolate over them, in place
689 
690  \param[in,out] ccdExposure exposure to process
691  \param[in] defectBaseList a list of defects to mask and interpolate
692 
693  \warning: call this after CCD assembly, since defects may cross amplifier boundaries
694  """
695  maskedImage = ccdExposure.getMaskedImage()
696  defectList = []
697  for d in defectBaseList:
698  bbox = d.getBBox()
699  nd = measAlg.Defect(bbox)
700  defectList.append(nd)
701  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
702  isrFunctions.interpolateDefectList(
703  maskedImage=maskedImage,
704  defectList=defectList,
705  fwhm=self.config.fwhm,
706  )
707 
708  def maskAndInterpNan(self, exposure):
709  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
710 
711  We mask and interpolate over all NaNs, including those
712  that are masked with other bits (because those may or may
713  not be interpolated over later, and we want to remove all
714  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
715  is used to preserve the historical name.
716 
717  \param[in,out] exposure exposure to process
718  """
719  maskedImage = exposure.getMaskedImage()
720 
721  # Find and mask NaNs
722  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
723  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
724  numNans = maskNans(maskedImage, maskVal)
725  self.metadata.set("NUMNANS", numNans)
726 
727  # Interpolate over these previously-unmasked NaNs
728  if numNans > 0:
729  self.log.warn("There were %i unmasked NaNs", numNans)
730  nanDefectList = isrFunctions.getDefectListFromMask(
731  maskedImage=maskedImage,
732  maskName='UNMASKEDNAN',
733  growFootprints=0,
734  )
735  isrFunctions.interpolateDefectList(
736  maskedImage=exposure.getMaskedImage(),
737  defectList=nanDefectList,
738  fwhm=self.config.fwhm,
739  )
740 
741  def overscanCorrection(self, exposure, amp):
742  """!Apply overscan correction, in place
743 
744  \param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
745  \param[in] amp amplifier device data
746  """
747  if not amp.getHasRawInfo():
748  raise RuntimeError("This method must be executed on an amp with raw information.")
749 
750  if amp.getRawHorizontalOverscanBBox().isEmpty():
751  self.log.info("No Overscan region. Not performing Overscan Correction.")
752  return None
753 
754  maskedImage = exposure.getMaskedImage()
755  dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
756 
757  expImage = exposure.getMaskedImage().getImage()
758  overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
759 
760  isrFunctions.overscanCorrection(
761  ampMaskedImage=dataView,
762  overscanImage=overscanImage,
763  fitType=self.config.overscanFitType,
764  order=self.config.overscanOrder,
765  collapseRej=self.config.overscanRej,
766  )
767 
768  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
769  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
770 
771  \param[in,out] ccdExposure exposure to process
772  \param[in] fpPolygon Polygon in focal plane coordinates
773  """
774  # Get ccd corners in focal plane coordinates
775  ccd = ccdExposure.getDetector()
776  fpCorners = ccd.getCorners(FOCAL_PLANE)
777  ccdPolygon = Polygon(fpCorners)
778 
779  # Get intersection of ccd corners with fpPolygon
780  intersect = ccdPolygon.intersectionSingle(fpPolygon)
781 
782  # Transform back to pixel positions and build new polygon
783  ccdPoints = [ccd.transform(ccd.makeCameraPoint(x, FOCAL_PLANE), PIXELS).getPoint() for x in intersect]
784  validPolygon = Polygon(ccdPoints)
785  ccdExposure.getInfo().setValidPolygon(validPolygon)
786 
787  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
788  """Apply brighter fatter correction in place for the image
789 
790  This correction takes a kernel that has been derived from flat field images to
791  redistribute the charge. The gradient of the kernel is the deflection
792  field due to the accumulated charge.
793 
794  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
795  using the following equation:
796 
797  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
798 
799  To evaluate the derivative term we expand it as follows:
800 
801  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
802 
803  Because we use the measured counts instead of the incident counts we apply the correction
804  iteratively to reconstruct the original counts and the correction. We stop iterating when the
805  summed difference between the current corrected image and the one from the previous iteration
806  is below the threshold. We do not require convergence because the number of iterations is
807  too large a computational cost. How we define the threshold still needs to be evaluated, the
808  current default was shown to work reasonably well on a small set of images. For more information
809  on the method see DocuShare Document-19407.
810 
811  The edges as defined by the kernel are not corrected because they have spurious values
812  due to the convolution.
813  """
814  self.log.info("Applying brighter fatter correction")
815 
816  image = exposure.getMaskedImage().getImage()
817 
818  # The image needs to be units of electrons/holes
819  with self.gainContext(exposure, image, applyGain):
820 
821  kLx = numpy.shape(kernel)[0]
822  kLy = numpy.shape(kernel)[1]
823  kernelImage = afwImage.ImageD(kLx, kLy)
824  kernelImage.getArray()[:, :] = kernel
825  tempImage = image.clone()
826 
827  nanIndex = numpy.isnan(tempImage.getArray())
828  tempImage.getArray()[nanIndex] = 0.
829 
830  outImage = afwImage.ImageF(image.getDimensions())
831  corr = numpy.zeros_like(image.getArray())
832  prev_image = numpy.zeros_like(image.getArray())
833  convCntrl = afwMath.ConvolutionControl(False, True, 1)
834  fixedKernel = afwMath.FixedKernel(kernelImage)
835 
836  # Define boundary by convolution region. The region that the correction will be
837  # calculated for is one fewer in each dimension because of the second derivative terms.
838  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
839  startX = kLx//2
840  endX = -kLx//2
841  startY = kLy//2
842  endY = -kLy//2
843 
844  for iteration in range(maxIter):
845 
846  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
847  tmpArray = tempImage.getArray()
848  outArray = outImage.getArray()
849 
850  # First derivative term
851  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
852  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
853  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
854 
855  # Second derivative term
856  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
857  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
858  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
859 
860  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
861 
862  tmpArray[:, :] = image.getArray()[:, :]
863  tmpArray[nanIndex] = 0.
864  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
865 
866  if iteration > 0:
867  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
868 
869  if diff < threshold:
870  break
871  prev_image[:, :] = tmpArray[:, :]
872 
873  if iteration == maxIter - 1:
874  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
875 
876  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
877  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
878  corr[startY + 1:endY - 1, startX + 1:endX - 1]
879 
880 
881  @contextmanager
882  def gainContext(self, exp, image, apply):
883  """Context manager that applies and removes gain
884  """
885  if apply:
886  ccd = exp.getDetector()
887  for amp in ccd:
888  sim = image.Factory(image, amp.getBBox())
889  sim *= amp.getGain()
890 
891  try:
892  yield exp
893  finally:
894  if apply:
895  ccd = exp.getDetector()
896  for amp in ccd:
897  sim = image.Factory(image, amp.getBBox())
898  sim /= amp.getGain()
899 
900 
901 class FakeAmp(object):
902  """A Detector-like object that supports returning gain and saturation level"""
903 
904  def __init__(self, exposure, config):
905  self._bbox = exposure.getBBox(afwImage.LOCAL)
906  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
907  self._gain = config.gain
908  self._readNoise = config.readNoise
909  self._saturation = config.saturation
910 
911  def getBBox(self):
912  return self._bbox
913 
914  def getRawBBox(self):
915  return self._bbox
916 
917  def getHasRawInfo(self):
918  return True # but see getRawHorizontalOverscanBBox()
919 
921  return self._RawHorizontalOverscanBBox
922 
923  def getGain(self):
924  return self._gain
925 
926  def getReadNoise(self):
927  return self._readNoise
928 
929  def getSaturation(self):
930  return self._saturation
931 
932  def getSuspectLevel(self):
933  return float("NaN")
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
Definition: isrTask.py:787
def runDataRef(self, sensorRef)
Perform instrument signature removal on a ButlerDataRef of a Sensor.
Definition: isrTask.py:502
def gainContext(self, exp, image, apply)
Definition: isrTask.py:882
def __init__(self, args, kwargs)
Constructor for IsrTask.
Definition: isrTask.py:329
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:341
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:708
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:646
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:384
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:255
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:920
def darkCorrection(self, exposure, darkExposure)
Apply dark correction in place.
Definition: isrTask.py:550
def convertIntToFloat(self, exposure)
Definition: isrTask.py:525
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:605
def doLinearize(self, detector)
Is linearization wanted for this detector?
Definition: isrTask.py:569
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:768
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
Definition: isrTask.py:542
def updateVariance(self, ampExposure, amp)
Set the variance plane based on the image plane, plus amplifier gain and read noise.
Definition: isrTask.py:579
def overscanCorrection(self, exposure, amp)
Apply overscan correction, in place.
Definition: isrTask.py:741
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:662
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:687
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:630
def flatCorrection(self, exposure, flatExposure)
Apply flat correction in place.
Definition: isrTask.py:592
def __init__(self, exposure, config)
Definition: isrTask.py:904