lsst.ip.isr  14.0-3-g27b2198+1
isrTask.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 from builtins import object
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 import numpy
27 
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.math as afwMath
34 from lsst.daf.persistence import ButlerDataRef
35 from lsstDebug import getDebugFrame
36 from lsst.afw.display import getDisplay
37 from . import isrFunctions
38 from .assembleCcdTask import AssembleCcdTask
39 from .fringe import FringeTask
40 from lsst.afw.geom.polygon import Polygon
41 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
42 from contextlib import contextmanager
43 from .isr import maskNans
44 from .crosstalk import CrosstalkTask
45 
46 
47 class IsrTaskConfig(pexConfig.Config):
48  doBias = pexConfig.Field(
49  dtype=bool,
50  doc="Apply bias frame correction?",
51  default=True,
52  )
53  doDark = pexConfig.Field(
54  dtype=bool,
55  doc="Apply dark frame correction?",
56  default=True,
57  )
58  doFlat = pexConfig.Field(
59  dtype=bool,
60  doc="Apply flat field correction?",
61  default=True,
62  )
63  doFringe = pexConfig.Field(
64  dtype=bool,
65  doc="Apply fringe correction?",
66  default=True,
67  )
68  doDefect = pexConfig.Field(
69  dtype=bool,
70  doc="Apply correction for CCD defects, e.g. hot pixels?",
71  default=True,
72  )
73  doWrite = pexConfig.Field(
74  dtype=bool,
75  doc="Persist postISRCCD?",
76  default=True,
77  )
78  assembleCcd = pexConfig.ConfigurableField(
79  target=AssembleCcdTask,
80  doc="CCD assembly task",
81  )
82  gain = pexConfig.Field(
83  dtype=float,
84  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
85  default=float("NaN"),
86  )
87  readNoise = pexConfig.Field(
88  dtype=float,
89  doc="The read noise to use if no Detector is present in the Exposure",
90  default=0.0,
91  )
92  saturation = pexConfig.Field(
93  dtype=float,
94  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
95  default=float("NaN"),
96  )
97  fringeAfterFlat = pexConfig.Field(
98  dtype=bool,
99  doc="Do fringe subtraction after flat-fielding?",
100  default=True,
101  )
102  fringe = pexConfig.ConfigurableField(
103  target=FringeTask,
104  doc="Fringe subtraction task",
105  )
106  fwhm = pexConfig.Field(
107  dtype=float,
108  doc="FWHM of PSF (arcsec)",
109  default=1.0,
110  )
111  saturatedMaskName = pexConfig.Field(
112  dtype=str,
113  doc="Name of mask plane to use in saturation detection and interpolation",
114  default="SAT",
115  )
116  suspectMaskName = pexConfig.Field(
117  dtype=str,
118  doc="Name of mask plane to use for suspect pixels",
119  default="SUSPECT",
120  )
121  flatScalingType = pexConfig.ChoiceField(
122  dtype=str,
123  doc="The method for scaling the flat on the fly.",
124  default='USER',
125  allowed={
126  "USER": "Scale by flatUserScale",
127  "MEAN": "Scale by the inverse of the mean",
128  "MEDIAN": "Scale by the inverse of the median",
129  },
130  )
131  flatUserScale = pexConfig.Field(
132  dtype=float,
133  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
134  default=1.0,
135  )
136  overscanFitType = pexConfig.ChoiceField(
137  dtype=str,
138  doc="The method for fitting the overscan bias level.",
139  default='MEDIAN',
140  allowed={
141  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
142  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
143  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
144  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
145  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
146  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
147  "MEAN": "Correct using the mean of the overscan region",
148  "MEDIAN": "Correct using the median of the overscan region",
149  },
150  )
151  overscanOrder = pexConfig.Field(
152  dtype=int,
153  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
154  "or number of spline knots if overscan fit type is a spline."),
155  default=1,
156  )
157  overscanRej = pexConfig.Field(
158  dtype=float,
159  doc="Rejection threshold (sigma) for collapsing overscan before fit",
160  default=3.0,
161  )
162  growSaturationFootprintSize = pexConfig.Field(
163  dtype=int,
164  doc="Number of pixels by which to grow the saturation footprints",
165  default=1,
166  )
167  doSaturationInterpolation = pexConfig.Field(
168  dtype=bool,
169  doc="Perform interpolation over pixels masked as saturated?",
170  default=True,
171  )
172  doNanInterpAfterFlat = pexConfig.Field(
173  dtype=bool,
174  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
175  "also have to interpolate them before flat-fielding."),
176  default=False,
177  )
178  fluxMag0T1 = pexConfig.Field(
179  dtype=float,
180  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
181  default=1e10,
182  )
183  keysToRemoveFromAssembledCcd = pexConfig.ListField(
184  dtype=str,
185  doc="fields to remove from the metadata of the assembled ccd.",
186  default=[],
187  )
188  doAssembleIsrExposures = pexConfig.Field(
189  dtype=bool,
190  default=False,
191  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
192  )
193  doAssembleCcd = pexConfig.Field(
194  dtype=bool,
195  default=True,
196  doc="Assemble amp-level exposures into a ccd-level exposure?"
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)"
202  )
203  doLinearize = pexConfig.Field(
204  dtype=bool,
205  doc="Correct for nonlinearity of the detector's response?",
206  default=True,
207  )
208  doCrosstalk = pexConfig.Field(
209  dtype=bool,
210  doc="Apply intra-CCD crosstalk correction?",
211  default=False,
212  )
213  crosstalk = pexConfig.ConfigurableField(
214  target=CrosstalkTask,
215  doc="Intra-CCD crosstalk correction",
216  )
217  doBrighterFatter = pexConfig.Field(
218  dtype=bool,
219  default=False,
220  doc="Apply the brighter fatter correction"
221  )
222  brighterFatterKernelFile = pexConfig.Field(
223  dtype=str,
224  default='',
225  doc="Kernel file used for the brighter fatter correction"
226  )
227  brighterFatterMaxIter = pexConfig.Field(
228  dtype=int,
229  default=10,
230  doc="Maximum number of iterations for the brighter fatter correction"
231  )
232  brighterFatterThreshold = pexConfig.Field(
233  dtype=float,
234  default=1000,
235  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
236  " absolute value of the difference between the current corrected image and the one"
237  " from the previous iteration summed over all the pixels."
238  )
239  brighterFatterApplyGain = pexConfig.Field(
240  dtype=bool,
241  default=True,
242  doc="Should the gain be applied when applying the brighter fatter correction?"
243  )
244  datasetType = pexConfig.Field(
245  dtype=str,
246  doc="Dataset type for input data; users will typically leave this alone, "
247  "but camera-specific ISR tasks will override it",
248  default="raw",
249  )
250  fallbackFilterName = pexConfig.Field(dtype=str,
251  doc="Fallback default filter name for calibrations", optional=True)
252 
253 
259 
260 
261 class IsrTask(pipeBase.CmdLineTask):
262  """!
263  \anchor IsrTask_
264 
265  \brief Apply common instrument signature correction algorithms to a raw frame.
266 
267  \section ip_isr_isr_Contents Contents
268 
269  - \ref ip_isr_isr_Purpose
270  - \ref ip_isr_isr_Initialize
271  - \ref ip_isr_isr_IO
272  - \ref ip_isr_isr_Config
273  - \ref ip_isr_isr_Debug
274 
275 
276  \section ip_isr_isr_Purpose Description
277 
278  The process for correcting imaging data is very similar from camera to camera.
279  This task provides a vanilla implementation of doing these corrections, including
280  the ability to turn certain corrections off if they are not needed.
281  The inputs to the primary method, run, are a raw exposure to be corrected and the
282  calibration data products. The raw input is a single chip sized mosaic of all amps
283  including overscans and other non-science pixels.
284  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
285  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
286  This task may not meet all needs and it is expected that it will be subclassed for
287  specific applications.
288 
289  \section ip_isr_isr_Initialize Task initialization
290 
291  \copydoc \_\_init\_\_
292 
293  \section ip_isr_isr_IO Inputs/Outputs to the run method
294 
295  \copydoc run
296 
297  \section ip_isr_isr_Config Configuration parameters
298 
299  See \ref IsrTaskConfig
300 
301  \section ip_isr_isr_Debug Debug variables
302 
303  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
304  flag \c --debug, \c -d to import \b debug.py from your \c PYTHONPATH; see <a
305  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
306  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
307 
308  The available variables in IsrTask are:
309  <DL>
310  <DT> \c display
311  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
312  <DL>
313  <DT> postISRCCD
314  <DD> display exposure after ISR has been applied
315  </DL>
316  </DL>
317 
318  For example, put something like
319  \code{.py}
320  import lsstDebug
321  def DebugInfo(name):
322  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
323  if name == "lsst.ip.isrFunctions.isrTask":
324  di.display = {'postISRCCD':2}
325  return di
326  lsstDebug.Info = DebugInfo
327  \endcode
328  into your debug.py file and run the commandline task with the \c --debug flag.
329 
330  <HR>
331  """
332  ConfigClass = IsrTaskConfig
333  _DefaultName = "isr"
334 
335  def __init__(self, *args, **kwargs):
336  '''!Constructor for IsrTask
337  \param[in] *args -- a list of positional arguments passed on to the Task constructor
338  \param[in] **kwargs -- a dictionary of keyword arguments passed on to the Task constructor
339  Call the lsst.pipe.base.task.Task.__init__ method
340  Then setup the assembly and fringe correction subtasks
341  '''
342  pipeBase.Task.__init__(self, *args, **kwargs)
343  self.makeSubtask("assembleCcd")
344  self.makeSubtask("fringe")
345  self.makeSubtask("crosstalk")
346 
347  def readIsrData(self, dataRef, rawExposure):
348  """!Retrieve necessary frames for instrument signature removal
349  \param[in] dataRef -- a daf.persistence.butlerSubset.ButlerDataRef
350  of the detector data to be processed
351  \param[in] rawExposure -- a reference raw exposure that will later be
352  corrected with the retrieved calibration data;
353  should not be modified in this method.
354  \return a pipeBase.Struct with fields containing kwargs expected by run()
355  - bias: exposure of bias frame
356  - dark: exposure of dark frame
357  - flat: exposure of flat field
358  - defects: list of detects
359  - fringeStruct: a pipeBase.Struct with field fringes containing
360  exposure of fringe frame or list of fringe exposure
361  """
362  ccd = rawExposure.getDetector()
363 
364  biasExposure = self.getIsrExposure(dataRef, "bias") if self.config.doBias else None
365  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
366  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
367  darkExposure = self.getIsrExposure(dataRef, "dark") if self.config.doDark else None
368  flatExposure = self.getIsrExposure(dataRef, "flat") if self.config.doFlat else None
369  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
370  defectList = dataRef.get("defects") if self.config.doDefect else None
371 
372  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
373  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
374  if self.config.doAssembleIsrExposures else None)
375  else:
376  fringeStruct = pipeBase.Struct(fringes=None)
377 
378  # Struct should include only kwargs to run()
379  return pipeBase.Struct(bias=biasExposure,
380  linearizer=linearizer,
381  dark=darkExposure,
382  flat=flatExposure,
383  defects=defectList,
384  fringes=fringeStruct,
385  bfKernel=brighterFatterKernel
386  )
387 
388  @pipeBase.timeMethod
389  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
390  fringes=None, bfKernel=None):
391  """!Perform instrument signature removal on an exposure
392 
393  Steps include:
394  - Detect saturation, apply overscan correction, bias, dark and flat
395  - Perform CCD assembly
396  - Interpolate over defects, saturated pixels and all NaNs
397 
398  \param[in] ccdExposure -- lsst.afw.image.exposure of detector data
399  \param[in] bias -- exposure of bias frame
400  \param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
401  \param[in] dark -- exposure of dark frame
402  \param[in] flat -- exposure of flatfield
403  \param[in] defects -- list of detects
404  \param[in] fringes -- a pipeBase.Struct with field fringes containing
405  exposure of fringe frame or list of fringe exposure
406  \param[in] bfKernel -- kernel for brighter-fatter correction
407 
408  \return a pipeBase.Struct with field:
409  - exposure
410  """
411 
412  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
413  if isinstance(ccdExposure, ButlerDataRef):
414  return self.runDataRef(ccdExposure)
415 
416  ccd = ccdExposure.getDetector()
417 
418  # Validate Input
419  if self.config.doBias and bias is None:
420  raise RuntimeError("Must supply a bias exposure if config.doBias True")
421  if self.doLinearize(ccd) and linearizer is None:
422  raise RuntimeError("Must supply a linearizer if config.doBias True")
423  if self.config.doDark and dark is None:
424  raise RuntimeError("Must supply a dark exposure if config.doDark True")
425  if self.config.doFlat and flat is None:
426  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
427  if self.config.doBrighterFatter and bfKernel is None:
428  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
429  if fringes is None:
430  fringes = pipeBase.Struct(fringes=None)
431  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
432  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
433  if self.config.doDefect and defects is None:
434  raise RuntimeError("Must supply defects if config.doDefect True")
435 
436  ccdExposure = self.convertIntToFloat(ccdExposure)
437 
438  if not ccd:
439  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
440  ccd = [FakeAmp(ccdExposure, self.config)]
441 
442  for amp in ccd:
443  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
444  if ccdExposure.getBBox().contains(amp.getBBox()):
445  self.saturationDetection(ccdExposure, amp)
446  self.suspectDetection(ccdExposure, amp)
447  self.overscanCorrection(ccdExposure, amp)
448 
449  if self.config.doAssembleCcd:
450  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
451  if self.config.expectWcs and not ccdExposure.getWcs():
452  self.log.warn("No WCS found in input exposure")
453 
454  if self.config.doBias:
455  self.biasCorrection(ccdExposure, bias)
456 
457  if self.doLinearize(ccd):
458  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
459 
460  if self.config.doCrosstalk:
461  self.crosstalk.run(ccdExposure)
462 
463  for amp in ccd:
464  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
465  if ccdExposure.getBBox().contains(amp.getBBox()):
466  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
467  self.updateVariance(ampExposure, amp)
468 
469  interpolationDone = False
470 
471  if self.config.doBrighterFatter:
472 
473  # We need to apply flats and darks before we can interpolate, and we
474  # need to interpolate before we do B-F, but we do B-F without the
475  # flats and darks applied so we can work in units of electrons or holes.
476  # This context manager applies and then removes the darks and flats.
477  with self.flatContext(ccdExposure, flat, dark):
478  if self.config.doDefect:
479  self.maskAndInterpDefect(ccdExposure, defects)
480  if self.config.doSaturationInterpolation:
481  self.saturationInterpolation(ccdExposure)
482  self.maskAndInterpNan(ccdExposure)
483  interpolationDone = True
484 
485  self.brighterFatterCorrection(ccdExposure, bfKernel,
486  self.config.brighterFatterMaxIter,
487  self.config.brighterFatterThreshold,
488  self.config.brighterFatterApplyGain,
489  )
490 
491  if self.config.doDark:
492  self.darkCorrection(ccdExposure, dark)
493 
494  if self.config.doFringe and not self.config.fringeAfterFlat:
495  self.fringe.run(ccdExposure, **fringes.getDict())
496 
497  if self.config.doFlat:
498  self.flatCorrection(ccdExposure, flat)
499 
500  if not interpolationDone:
501  if self.config.doDefect:
502  self.maskAndInterpDefect(ccdExposure, defects)
503  if self.config.doSaturationInterpolation:
504  self.saturationInterpolation(ccdExposure)
505  if not interpolationDone or self.config.doNanInterpAfterFlat:
506  self.maskAndInterpNan(ccdExposure)
507 
508  if self.config.doFringe and self.config.fringeAfterFlat:
509  self.fringe.run(ccdExposure, **fringes.getDict())
510 
511  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
512  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
513 
514  frame = getDebugFrame(self._display, "postISRCCD")
515  if frame:
516  getDisplay(frame).mtv(ccdExposure)
517 
518  return pipeBase.Struct(
519  exposure=ccdExposure,
520  )
521 
522  @pipeBase.timeMethod
523  def runDataRef(self, sensorRef):
524  """!Perform instrument signature removal on a ButlerDataRef of a Sensor
525 
526  - Read in necessary detrending/isr/calibration data
527  - Process raw exposure in run()
528  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
529 
530  \param[in] sensorRef -- daf.persistence.butlerSubset.ButlerDataRef of the
531  detector data to be processed
532  \return a pipeBase.Struct with fields:
533  - exposure: the exposure after application of ISR
534  """
535  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
536  ccdExposure = sensorRef.get('raw')
537  isrData = self.readIsrData(sensorRef, ccdExposure)
538 
539  result = self.run(ccdExposure, **isrData.getDict())
540 
541  if self.config.doWrite:
542  sensorRef.put(result.exposure, "postISRCCD")
543 
544  return result
545 
546  def convertIntToFloat(self, exposure):
547  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
548  """
549  if isinstance(exposure, afwImage.ExposureF):
550  # Nothing to be done
551  return exposure
552  if not hasattr(exposure, "convertF"):
553  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
554 
555  newexposure = exposure.convertF()
556  maskedImage = newexposure.getMaskedImage()
557  varArray = maskedImage.getVariance().getArray()
558  varArray[:, :] = 1
559  maskArray = maskedImage.getMask().getArray()
560  maskArray[:, :] = 0
561  return newexposure
562 
563  def biasCorrection(self, exposure, biasExposure):
564  """!Apply bias correction in place
565 
566  \param[in,out] exposure exposure to process
567  \param[in] biasExposure bias exposure of same size as exposure
568  """
569  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
570 
571  def darkCorrection(self, exposure, darkExposure, invert=False):
572  """!Apply dark correction in place
573 
574  \param[in,out] exposure exposure to process
575  \param[in] darkExposure dark exposure of same size as exposure
576  \param[in] invert if True, remove the dark from an already-corrected image
577  """
578  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
579  if math.isnan(expScale):
580  raise RuntimeError("Exposure darktime is NAN")
581  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
582  if math.isnan(darkScale):
583  raise RuntimeError("Dark calib darktime is NAN")
584  isrFunctions.darkCorrection(
585  maskedImage=exposure.getMaskedImage(),
586  darkMaskedImage=darkExposure.getMaskedImage(),
587  expScale=expScale,
588  darkScale=darkScale,
589  invert=invert
590  )
591 
592  def doLinearize(self, detector):
593  """!Is linearization wanted for this detector?
594 
595  Checks config.doLinearize and the linearity type of the first amplifier.
596 
597  \param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
598  """
599  return self.config.doLinearize and \
600  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
601 
602  def updateVariance(self, ampExposure, amp):
603  """!Set the variance plane based on the image plane, plus amplifier gain and read noise
604 
605  \param[in,out] ampExposure exposure to process
606  \param[in] amp amplifier detector information
607  """
608  if not math.isnan(amp.getGain()):
609  isrFunctions.updateVariance(
610  maskedImage=ampExposure.getMaskedImage(),
611  gain=amp.getGain(),
612  readNoise=amp.getReadNoise(),
613  )
614 
615  def flatCorrection(self, exposure, flatExposure, invert=False):
616  """!Apply flat correction in place
617 
618  \param[in,out] exposure exposure to process
619  \param[in] flatExposure flatfield exposure same size as exposure
620  \param[in] invert if True, unflatten an already-flattened image instead.
621  """
622  isrFunctions.flatCorrection(
623  maskedImage=exposure.getMaskedImage(),
624  flatMaskedImage=flatExposure.getMaskedImage(),
625  scalingType=self.config.flatScalingType,
626  userScale=self.config.flatUserScale,
627  invert=invert
628  )
629 
630  def getIsrExposure(self, dataRef, datasetType, immediate=True):
631  """!Retrieve a calibration dataset for removing instrument signature
632 
633  \param[in] dataRef data reference for exposure
634  \param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
635  \param[in] immediate if True, disable butler proxies to enable error
636  handling within this routine
637  \return exposure
638  """
639  try:
640  exp = dataRef.get(datasetType, immediate=immediate)
641  except Exception as exc1:
642  if not self.config.fallbackFilterName:
643  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
644  try:
645  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
646  except Exception as exc2:
647  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
648  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
649  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
650 
651  if self.config.doAssembleIsrExposures:
652  exp = self.assembleCcd.assembleCcd(exp)
653  return exp
654 
655  def saturationDetection(self, exposure, amp):
656  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
657 
658  \param[in,out] exposure exposure to process; only the amp DataSec is processed
659  \param[in] amp amplifier device data
660  """
661  if not math.isnan(amp.getSaturation()):
662  maskedImage = exposure.getMaskedImage()
663  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
664  isrFunctions.makeThresholdMask(
665  maskedImage=dataView,
666  threshold=amp.getSaturation(),
667  growFootprints=0,
668  maskName=self.config.saturatedMaskName,
669  )
670 
671  def saturationInterpolation(self, ccdExposure):
672  """!Interpolate over saturated pixels, in place
673 
674  \param[in,out] ccdExposure exposure to process
675 
676  \warning:
677  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
678  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
679  """
680  isrFunctions.interpolateFromMask(
681  maskedImage=ccdExposure.getMaskedImage(),
682  fwhm=self.config.fwhm,
683  growFootprints=self.config.growSaturationFootprintSize,
684  maskName=self.config.saturatedMaskName,
685  )
686 
687  def suspectDetection(self, exposure, amp):
688  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
689 
690  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
691  This is intended to indicate pixels that may be affected by unknown systematics;
692  for example if non-linearity corrections above a certain level are unstable
693  then that would be a useful value for suspectLevel. A value of `nan` indicates
694  that no such level exists and no pixels are to be masked as suspicious.
695 
696  \param[in,out] exposure exposure to process; only the amp DataSec is processed
697  \param[in] amp amplifier device data
698  """
699  suspectLevel = amp.getSuspectLevel()
700  if math.isnan(suspectLevel):
701  return
702 
703  maskedImage = exposure.getMaskedImage()
704  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
705  isrFunctions.makeThresholdMask(
706  maskedImage=dataView,
707  threshold=suspectLevel,
708  growFootprints=0,
709  maskName=self.config.suspectMaskName,
710  )
711 
712  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
713  """!Mask defects using mask plane "BAD" and interpolate over them, in place
714 
715  \param[in,out] ccdExposure exposure to process
716  \param[in] defectBaseList a list of defects to mask and interpolate
717 
718  \warning: call this after CCD assembly, since defects may cross amplifier boundaries
719  """
720  maskedImage = ccdExposure.getMaskedImage()
721  defectList = []
722  for d in defectBaseList:
723  bbox = d.getBBox()
724  nd = measAlg.Defect(bbox)
725  defectList.append(nd)
726  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
727  isrFunctions.interpolateDefectList(
728  maskedImage=maskedImage,
729  defectList=defectList,
730  fwhm=self.config.fwhm,
731  )
732 
733  def maskAndInterpNan(self, exposure):
734  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
735 
736  We mask and interpolate over all NaNs, including those
737  that are masked with other bits (because those may or may
738  not be interpolated over later, and we want to remove all
739  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
740  is used to preserve the historical name.
741 
742  \param[in,out] exposure exposure to process
743  """
744  maskedImage = exposure.getMaskedImage()
745 
746  # Find and mask NaNs
747  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
748  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
749  numNans = maskNans(maskedImage, maskVal)
750  self.metadata.set("NUMNANS", numNans)
751 
752  # Interpolate over these previously-unmasked NaNs
753  if numNans > 0:
754  self.log.warn("There were %i unmasked NaNs", numNans)
755  nanDefectList = isrFunctions.getDefectListFromMask(
756  maskedImage=maskedImage,
757  maskName='UNMASKEDNAN',
758  )
759  isrFunctions.interpolateDefectList(
760  maskedImage=exposure.getMaskedImage(),
761  defectList=nanDefectList,
762  fwhm=self.config.fwhm,
763  )
764 
765  def overscanCorrection(self, exposure, amp):
766  """!Apply overscan correction, in place
767 
768  \param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
769  \param[in] amp amplifier device data
770  """
771  if not amp.getHasRawInfo():
772  raise RuntimeError("This method must be executed on an amp with raw information.")
773 
774  if amp.getRawHorizontalOverscanBBox().isEmpty():
775  self.log.info("No Overscan region. Not performing Overscan Correction.")
776  return None
777 
778  maskedImage = exposure.getMaskedImage()
779  dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
780 
781  expImage = exposure.getMaskedImage().getImage()
782  overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
783 
784  isrFunctions.overscanCorrection(
785  ampMaskedImage=dataView,
786  overscanImage=overscanImage,
787  fitType=self.config.overscanFitType,
788  order=self.config.overscanOrder,
789  collapseRej=self.config.overscanRej,
790  )
791 
792  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
793  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
794 
795  \param[in,out] ccdExposure exposure to process
796  \param[in] fpPolygon Polygon in focal plane coordinates
797  """
798  # Get ccd corners in focal plane coordinates
799  ccd = ccdExposure.getDetector()
800  fpCorners = ccd.getCorners(FOCAL_PLANE)
801  ccdPolygon = Polygon(fpCorners)
802 
803  # Get intersection of ccd corners with fpPolygon
804  intersect = ccdPolygon.intersectionSingle(fpPolygon)
805 
806  # Transform back to pixel positions and build new polygon
807  ccdPoints = [ccd.transform(ccd.makeCameraPoint(x, FOCAL_PLANE), PIXELS).getPoint() for x in intersect]
808  validPolygon = Polygon(ccdPoints)
809  ccdExposure.getInfo().setValidPolygon(validPolygon)
810 
811  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
812  """Apply brighter fatter correction in place for the image
813 
814  This correction takes a kernel that has been derived from flat field images to
815  redistribute the charge. The gradient of the kernel is the deflection
816  field due to the accumulated charge.
817 
818  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
819  using the following equation:
820 
821  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
822 
823  To evaluate the derivative term we expand it as follows:
824 
825  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
826 
827  Because we use the measured counts instead of the incident counts we apply the correction
828  iteratively to reconstruct the original counts and the correction. We stop iterating when the
829  summed difference between the current corrected image and the one from the previous iteration
830  is below the threshold. We do not require convergence because the number of iterations is
831  too large a computational cost. How we define the threshold still needs to be evaluated, the
832  current default was shown to work reasonably well on a small set of images. For more information
833  on the method see DocuShare Document-19407.
834 
835  The edges as defined by the kernel are not corrected because they have spurious values
836  due to the convolution.
837  """
838  self.log.info("Applying brighter fatter correction")
839 
840  image = exposure.getMaskedImage().getImage()
841 
842  # The image needs to be units of electrons/holes
843  with self.gainContext(exposure, image, applyGain):
844 
845  kLx = numpy.shape(kernel)[0]
846  kLy = numpy.shape(kernel)[1]
847  kernelImage = afwImage.ImageD(kLx, kLy)
848  kernelImage.getArray()[:, :] = kernel
849  tempImage = image.clone()
850 
851  nanIndex = numpy.isnan(tempImage.getArray())
852  tempImage.getArray()[nanIndex] = 0.
853 
854  outImage = afwImage.ImageF(image.getDimensions())
855  corr = numpy.zeros_like(image.getArray())
856  prev_image = numpy.zeros_like(image.getArray())
857  convCntrl = afwMath.ConvolutionControl(False, True, 1)
858  fixedKernel = afwMath.FixedKernel(kernelImage)
859 
860  # Define boundary by convolution region. The region that the correction will be
861  # calculated for is one fewer in each dimension because of the second derivative terms.
862  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
863  startX = kLx//2
864  endX = -kLx//2
865  startY = kLy//2
866  endY = -kLy//2
867 
868  for iteration in range(maxIter):
869 
870  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
871  tmpArray = tempImage.getArray()
872  outArray = outImage.getArray()
873 
874  # First derivative term
875  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
876  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
877  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
878 
879  # Second derivative term
880  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
881  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
882  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
883 
884  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
885 
886  tmpArray[:, :] = image.getArray()[:, :]
887  tmpArray[nanIndex] = 0.
888  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
889 
890  if iteration > 0:
891  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
892 
893  if diff < threshold:
894  break
895  prev_image[:, :] = tmpArray[:, :]
896 
897  if iteration == maxIter - 1:
898  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
899 
900  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
901  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
902  corr[startY + 1:endY - 1, startX + 1:endX - 1]
903 
904 
905  @contextmanager
906  def gainContext(self, exp, image, apply):
907  """Context manager that applies and removes gain
908  """
909  if apply:
910  ccd = exp.getDetector()
911  for amp in ccd:
912  sim = image.Factory(image, amp.getBBox())
913  sim *= amp.getGain()
914 
915  try:
916  yield exp
917  finally:
918  if apply:
919  ccd = exp.getDetector()
920  for amp in ccd:
921  sim = image.Factory(image, amp.getBBox())
922  sim /= amp.getGain()
923 
924 
925  @contextmanager
926  def flatContext(self, exp, flat, dark=None):
927  """Context manager that applies and removes flats and darks,
928  if the task is configured to apply them.
929  """
930  if self.config.doDark and dark is not None:
931  self.darkCorrection(exp, dark)
932  if self.config.doFlat:
933  self.flatCorrection(exp, flat)
934  try:
935  yield exp
936  finally:
937  if self.config.doFlat:
938  self.flatCorrection(exp, flat, invert=True)
939  if self.config.doDark and dark is not None:
940  self.darkCorrection(exp, dark, invert=True)
941 
942 
943 class FakeAmp(object):
944  """A Detector-like object that supports returning gain and saturation level"""
945 
946  def __init__(self, exposure, config):
947  self._bbox = exposure.getBBox(afwImage.LOCAL)
948  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
949  self._gain = config.gain
950  self._readNoise = config.readNoise
951  self._saturation = config.saturation
952 
953  def getBBox(self):
954  return self._bbox
955 
956  def getRawBBox(self):
957  return self._bbox
958 
959  def getHasRawInfo(self):
960  return True # but see getRawHorizontalOverscanBBox()
961 
963  return self._RawHorizontalOverscanBBox
964 
965  def getGain(self):
966  return self._gain
967 
968  def getReadNoise(self):
969  return self._readNoise
970 
971  def getSaturation(self):
972  return self._saturation
973 
974  def getSuspectLevel(self):
975  return float("NaN")
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
Definition: isrTask.py:811
def runDataRef(self, sensorRef)
Perform instrument signature removal on a ButlerDataRef of a Sensor.
Definition: isrTask.py:523
def gainContext(self, exp, image, apply)
Definition: isrTask.py:906
def __init__(self, args, kwargs)
Constructor for IsrTask.
Definition: isrTask.py:335
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:347
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:733
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:671
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:390
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:261
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:962
def convertIntToFloat(self, exposure)
Definition: isrTask.py:546
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:615
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:630
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:571
def doLinearize(self, detector)
Is linearization wanted for this detector?
Definition: isrTask.py:592
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:792
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
Definition: isrTask.py:563
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:926
def updateVariance(self, ampExposure, amp)
Set the variance plane based on the image plane, plus amplifier gain and read noise.
Definition: isrTask.py:602
def overscanCorrection(self, exposure, amp)
Apply overscan correction, in place.
Definition: isrTask.py:765
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:687
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:712
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:655
def __init__(self, exposure, config)
Definition: isrTask.py:946