lsst.ip.isr  13.0-11-gf01aa92+14
 All Classes Namespaces Files Functions Variables Typedefs Macros Groups
isrTask.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 from builtins import object
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 import numpy
27 
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.math as afwMath
34 from lsst.daf.persistence import ButlerDataRef
35 from lsstDebug import getDebugFrame
36 from lsst.afw.display import getDisplay
37 from . import isrFunctions
38 from .assembleCcdTask import AssembleCcdTask
39 from .fringe import FringeTask
40 from lsst.afw.geom.polygon import Polygon
41 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
42 from contextlib import contextmanager
43 from .isr import maskNans
44 
45 
46 class IsrTaskConfig(pexConfig.Config):
47  doBias = pexConfig.Field(
48  dtype=bool,
49  doc="Apply bias frame correction?",
50  default=True,
51  )
52  doDark = pexConfig.Field(
53  dtype=bool,
54  doc="Apply dark frame correction?",
55  default=True,
56  )
57  doFlat = pexConfig.Field(
58  dtype=bool,
59  doc="Apply flat field correction?",
60  default=True,
61  )
62  doFringe = pexConfig.Field(
63  dtype=bool,
64  doc="Apply fringe correction?",
65  default=True,
66  )
67  doDefect = pexConfig.Field(
68  dtype=bool,
69  doc="Apply correction for CCD defects, e.g. hot pixels?",
70  default=True,
71  )
72  doWrite = pexConfig.Field(
73  dtype=bool,
74  doc="Persist postISRCCD?",
75  default=True,
76  )
77  assembleCcd = pexConfig.ConfigurableField(
78  target=AssembleCcdTask,
79  doc="CCD assembly task",
80  )
81  gain = pexConfig.Field(
82  dtype=float,
83  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
84  default=float("NaN"),
85  )
86  readNoise = pexConfig.Field(
87  dtype=float,
88  doc="The read noise to use if no Detector is present in the Exposure",
89  default=0.0,
90  )
91  saturation = pexConfig.Field(
92  dtype=float,
93  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
94  default=float("NaN"),
95  )
96  fringeAfterFlat = pexConfig.Field(
97  dtype=bool,
98  doc="Do fringe subtraction after flat-fielding?",
99  default=True,
100  )
101  fringe = pexConfig.ConfigurableField(
102  target=FringeTask,
103  doc="Fringe subtraction task",
104  )
105  fwhm = pexConfig.Field(
106  dtype=float,
107  doc="FWHM of PSF (arcsec)",
108  default=1.0,
109  )
110  saturatedMaskName = pexConfig.Field(
111  dtype=str,
112  doc="Name of mask plane to use in saturation detection and interpolation",
113  default="SAT",
114  )
115  suspectMaskName = pexConfig.Field(
116  dtype=str,
117  doc="Name of mask plane to use for suspect pixels",
118  default="SUSPECT",
119  )
120  flatScalingType = pexConfig.ChoiceField(
121  dtype=str,
122  doc="The method for scaling the flat on the fly.",
123  default='USER',
124  allowed={
125  "USER": "Scale by flatUserScale",
126  "MEAN": "Scale by the inverse of the mean",
127  "MEDIAN": "Scale by the inverse of the median",
128  },
129  )
130  flatUserScale = pexConfig.Field(
131  dtype=float,
132  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
133  default=1.0,
134  )
135  overscanFitType = pexConfig.ChoiceField(
136  dtype=str,
137  doc="The method for fitting the overscan bias level.",
138  default='MEDIAN',
139  allowed={
140  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
141  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
142  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
143  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
144  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
145  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
146  "MEAN": "Correct using the mean of the overscan region",
147  "MEDIAN": "Correct using the median of the overscan region",
148  },
149  )
150  overscanOrder = pexConfig.Field(
151  dtype=int,
152  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
153  "or number of spline knots if overscan fit type is a spline."),
154  default=1,
155  )
156  overscanRej = pexConfig.Field(
157  dtype=float,
158  doc="Rejection threshold (sigma) for collapsing overscan before fit",
159  default=3.0,
160  )
161  growSaturationFootprintSize = pexConfig.Field(
162  dtype=int,
163  doc="Number of pixels by which to grow the saturation footprints",
164  default=1,
165  )
166  fluxMag0T1 = pexConfig.Field(
167  dtype=float,
168  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
169  default=1e10,
170  )
171  keysToRemoveFromAssembledCcd = pexConfig.ListField(
172  dtype=str,
173  doc="fields to remove from the metadata of the assembled ccd.",
174  default=[],
175  )
176  doAssembleIsrExposures = pexConfig.Field(
177  dtype=bool,
178  default=False,
179  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
180  )
181  doAssembleCcd = pexConfig.Field(
182  dtype=bool,
183  default=True,
184  doc="Assemble amp-level exposures into a ccd-level exposure?"
185  )
186  doLinearize = pexConfig.Field(
187  dtype=bool,
188  doc="Correct for nonlinearity of the detector's response?",
189  default=True,
190  )
191  doBrighterFatter = pexConfig.Field(
192  dtype=bool,
193  default=False,
194  doc="Apply the brighter fatter correction"
195  )
196  brighterFatterKernelFile = pexConfig.Field(
197  dtype=str,
198  default='',
199  doc="Kernel file used for the brighter fatter correction"
200  )
201  brighterFatterMaxIter = pexConfig.Field(
202  dtype=int,
203  default=10,
204  doc="Maximum number of iterations for the brighter fatter correction"
205  )
206  brighterFatterThreshold = pexConfig.Field(
207  dtype=float,
208  default=1000,
209  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
210  " absolute value of the difference between the current corrected image and the one"
211  " from the previous iteration summed over all the pixels."
212  )
213  brighterFatterApplyGain = pexConfig.Field(
214  dtype=bool,
215  default=True,
216  doc="Should the gain be applied when applying the brighter fatter correction?"
217  )
218  datasetType = pexConfig.Field(
219  dtype=str,
220  doc="Dataset type for input data; users will typically leave this alone, "
221  "but camera-specific ISR tasks will override it",
222  default="raw",
223  )
224  fallbackFilterName = pexConfig.Field(dtype=str,
225  doc="Fallback default filter name for calibrations", optional=True)
226 
227 ## \addtogroup LSST_task_documentation
228 ## \{
229 ## \page IsrTask
230 ## \ref IsrTask_ "IsrTask"
231 ## \copybrief IsrTask
232 ## \}
233 
234 
235 class IsrTask(pipeBase.CmdLineTask):
236  """!
237  \anchor IsrTask_
238 
239  \brief Apply common instrument signature correction algorithms to a raw frame.
240 
241  \section ip_isr_isr_Contents Contents
242 
243  - \ref ip_isr_isr_Purpose
244  - \ref ip_isr_isr_Initialize
245  - \ref ip_isr_isr_IO
246  - \ref ip_isr_isr_Config
247  - \ref ip_isr_isr_Debug
248 
249 
250  \section ip_isr_isr_Purpose Description
251 
252  The process for correcting imaging data is very similar from camera to camera.
253  This task provides a vanilla implementation of doing these corrections, including
254  the ability to turn certain corrections off if they are not needed.
255  The inputs to the primary method, run, are a raw exposure to be corrected and the
256  calibration data products. The raw input is a single chip sized mosaic of all amps
257  including overscans and other non-science pixels.
258  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
259  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
260  This task may not meet all needs and it is expected that it will be subclassed for
261  specific applications.
262 
263  \section ip_isr_isr_Initialize Task initialization
264 
265  \copydoc \_\_init\_\_
266 
267  \section ip_isr_isr_IO Inputs/Outputs to the run method
268 
269  \copydoc run
270 
271  \section ip_isr_isr_Config Configuration parameters
272 
273  See \ref IsrTaskConfig
274 
275  \section ip_isr_isr_Debug Debug variables
276 
277  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
278  flag \c --debug, \c -d to import \b debug.py from your \c PYTHONPATH; see <a
279  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
280  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
281 
282  The available variables in IsrTask are:
283  <DL>
284  <DT> \c display
285  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
286  <DL>
287  <DT> postISRCCD
288  <DD> display exposure after ISR has been applied
289  </DL>
290  </DL>
291 
292  For example, put something like
293  \code{.py}
294  import lsstDebug
295  def DebugInfo(name):
296  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
297  if name == "lsst.ip.isrFunctions.isrTask":
298  di.display = {'postISRCCD':2}
299  return di
300  lsstDebug.Info = DebugInfo
301  \endcode
302  into your debug.py file and run the commandline task with the \c --debug flag.
303 
304  <HR>
305  """
306  ConfigClass = IsrTaskConfig
307  _DefaultName = "isr"
308 
309  def __init__(self, *args, **kwargs):
310  '''!Constructor for IsrTask
311  \param[in] *args -- a list of positional arguments passed on to the Task constructor
312  \param[in] **kwargs -- a dictionary of keyword arguments passed on to the Task constructor
313  Call the lsst.pipe.base.task.Task.__init__ method
314  Then setup the assembly and fringe correction subtasks
315  '''
316  pipeBase.Task.__init__(self, *args, **kwargs)
317  self.makeSubtask("assembleCcd")
318  self.makeSubtask("fringe")
319 
320  def readIsrData(self, dataRef, rawExposure):
321  """!Retrieve necessary frames for instrument signature removal
322  \param[in] dataRef -- a daf.persistence.butlerSubset.ButlerDataRef
323  of the detector data to be processed
324  \param[in] rawExposure -- a reference raw exposure that will later be
325  corrected with the retrieved calibration data;
326  should not be modified in this method.
327  \return a pipeBase.Struct with fields containing kwargs expected by run()
328  - bias: exposure of bias frame
329  - dark: exposure of dark frame
330  - flat: exposure of flat field
331  - defects: list of detects
332  - fringeStruct: a pipeBase.Struct with field fringes containing
333  exposure of fringe frame or list of fringe exposure
334  """
335  ccd = rawExposure.getDetector()
336 
337  biasExposure = self.getIsrExposure(dataRef, "bias") if self.config.doBias else None
338  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
339  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
340  darkExposure = self.getIsrExposure(dataRef, "dark") if self.config.doDark else None
341  flatExposure = self.getIsrExposure(dataRef, "flat") if self.config.doFlat else None
342  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
343  defectList = dataRef.get("defects") if self.config.doDefect else None
344 
345  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
346  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
347  if self.config.doAssembleIsrExposures else None)
348  else:
349  fringeStruct = pipeBase.Struct(fringes=None)
350 
351  # Struct should include only kwargs to run()
352  return pipeBase.Struct(bias=biasExposure,
353  linearizer=linearizer,
354  dark=darkExposure,
355  flat=flatExposure,
356  defects=defectList,
357  fringes=fringeStruct,
358  bfKernel=brighterFatterKernel
359  )
360 
361  @pipeBase.timeMethod
362  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
363  fringes=None, bfKernel=None):
364  """!Perform instrument signature removal on an exposure
365 
366  Steps include:
367  - Detect saturation, apply overscan correction, bias, dark and flat
368  - Perform CCD assembly
369  - Interpolate over defects, saturated pixels and all NaNs
370 
371  \param[in] ccdExposure -- lsst.afw.image.exposure of detector data
372  \param[in] bias -- exposure of bias frame
373  \param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
374  \param[in] dark -- exposure of dark frame
375  \param[in] flat -- exposure of flatfield
376  \param[in] defects -- list of detects
377  \param[in] fringes -- a pipeBase.Struct with field fringes containing
378  exposure of fringe frame or list of fringe exposure
379  \param[in] bfKernel -- kernel for brighter-fatter correction
380 
381  \return a pipeBase.Struct with field:
382  - exposure
383  """
384 
385  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
386  if isinstance(ccdExposure, ButlerDataRef):
387  return self.runDataRef(ccdExposure)
388 
389  ccd = ccdExposure.getDetector()
390 
391  # Validate Input
392  if self.config.doBias and bias is None:
393  raise RuntimeError("Must supply a bias exposure if config.doBias True")
394  if self.doLinearize(ccd) and linearizer is None:
395  raise RuntimeError("Must supply a linearizer if config.doBias True")
396  if self.config.doDark and dark is None:
397  raise RuntimeError("Must supply a dark exposure if config.doDark True")
398  if self.config.doFlat and flat is None:
399  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
400  if self.config.doBrighterFatter and bfKernel is None:
401  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
402  if fringes is None:
403  fringes = pipeBase.Struct(fringes=None)
404  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
405  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
406  if self.config.doDefect and defects is None:
407  raise RuntimeError("Must supply defects if config.doDefect True")
408 
409  ccdExposure = self.convertIntToFloat(ccdExposure)
410 
411  if not ccd:
412  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
413  ccd = [FakeAmp(ccdExposure, self.config)]
414 
415  for amp in ccd:
416  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
417  if ccdExposure.getBBox().contains(amp.getBBox()):
418  self.saturationDetection(ccdExposure, amp)
419  self.suspectDetection(ccdExposure, amp)
420  self.overscanCorrection(ccdExposure, amp)
421 
422  if self.config.doAssembleCcd:
423  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
424 
425  if self.config.doBias:
426  self.biasCorrection(ccdExposure, bias)
427 
428  if self.doLinearize(ccd):
429  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
430 
431  for amp in ccd:
432  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
433  if ccdExposure.getBBox().contains(amp.getBBox()):
434  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
435  self.updateVariance(ampExposure, amp)
436 
437  if self.config.doBrighterFatter:
438  self.brighterFatterCorrection(ccdExposure, bfKernel,
439  self.config.brighterFatterMaxIter,
440  self.config.brighterFatterThreshold,
441  self.config.brighterFatterApplyGain,
442  )
443 
444  if self.config.doDark:
445  self.darkCorrection(ccdExposure, dark)
446 
447  if self.config.doFringe and not self.config.fringeAfterFlat:
448  self.fringe.run(ccdExposure, **fringes.getDict())
449 
450  if self.config.doFlat:
451  self.flatCorrection(ccdExposure, flat)
452 
453  if self.config.doDefect:
454  self.maskAndInterpDefect(ccdExposure, defects)
455 
456  self.saturationInterpolation(ccdExposure)
457 
458  self.maskAndInterpNan(ccdExposure)
459 
460  if self.config.doFringe and self.config.fringeAfterFlat:
461  self.fringe.run(ccdExposure, **fringes.getDict())
462 
463  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
464  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
465 
466  frame = getDebugFrame(self._display, "postISRCCD")
467  if frame:
468  getDisplay(frame).mtv(ccdExposure)
469 
470  return pipeBase.Struct(
471  exposure=ccdExposure,
472  )
473 
474  @pipeBase.timeMethod
475  def runDataRef(self, sensorRef):
476  """!Perform instrument signature removal on a ButlerDataRef of a Sensor
477 
478  - Read in necessary detrending/isr/calibration data
479  - Process raw exposure in run()
480  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
481 
482  \param[in] sensorRef -- daf.persistence.butlerSubset.ButlerDataRef of the
483  detector data to be processed
484  \return a pipeBase.Struct with fields:
485  - exposure: the exposure after application of ISR
486  """
487  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
488  ccdExposure = sensorRef.get('raw')
489  isrData = self.readIsrData(sensorRef, ccdExposure)
490 
491  result = self.run(ccdExposure, **isrData.getDict())
492 
493  if self.config.doWrite:
494  sensorRef.put(result.exposure, "postISRCCD")
495 
496  return result
497 
498  def convertIntToFloat(self, exposure):
499  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
500  """
501  if isinstance(exposure, afwImage.ExposureF):
502  # Nothing to be done
503  return exposure
504  if not hasattr(exposure, "convertF"):
505  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
506 
507  newexposure = exposure.convertF()
508  maskedImage = newexposure.getMaskedImage()
509  varArray = maskedImage.getVariance().getArray()
510  varArray[:, :] = 1
511  maskArray = maskedImage.getMask().getArray()
512  maskArray[:, :] = 0
513  return newexposure
514 
515  def biasCorrection(self, exposure, biasExposure):
516  """!Apply bias correction in place
517 
518  \param[in,out] exposure exposure to process
519  \param[in] biasExposure bias exposure of same size as exposure
520  """
521  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
522 
523  def darkCorrection(self, exposure, darkExposure):
524  """!Apply dark correction in place
525 
526  \param[in,out] exposure exposure to process
527  \param[in] darkExposure dark exposure of same size as exposure
528  """
529  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
530  if math.isnan(expScale):
531  raise RuntimeError("Exposure darktime is NAN")
532  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
533  if math.isnan(darkScale):
534  raise RuntimeError("Dark calib darktime is NAN")
535  isrFunctions.darkCorrection(
536  maskedImage=exposure.getMaskedImage(),
537  darkMaskedImage=darkExposure.getMaskedImage(),
538  expScale=expScale,
539  darkScale=darkScale,
540  )
541 
542  def doLinearize(self, detector):
543  """!Is linearization wanted for this detector?
544 
545  Checks config.doLinearize and the linearity type of the first amplifier.
546 
547  \param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
548  """
549  return self.config.doLinearize and \
550  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
551 
552  def updateVariance(self, ampExposure, amp):
553  """!Set the variance plane based on the image plane, plus amplifier gain and read noise
554 
555  \param[in,out] ampExposure exposure to process
556  \param[in] amp amplifier detector information
557  """
558  if not math.isnan(amp.getGain()):
559  isrFunctions.updateVariance(
560  maskedImage=ampExposure.getMaskedImage(),
561  gain=amp.getGain(),
562  readNoise=amp.getReadNoise(),
563  )
564 
565  def flatCorrection(self, exposure, flatExposure):
566  """!Apply flat correction in place
567 
568  \param[in,out] exposure exposure to process
569  \param[in] flatExposure flatfield exposure same size as exposure
570  """
571  isrFunctions.flatCorrection(
572  maskedImage=exposure.getMaskedImage(),
573  flatMaskedImage=flatExposure.getMaskedImage(),
574  scalingType=self.config.flatScalingType,
575  userScale=self.config.flatUserScale,
576  )
577 
578  def getIsrExposure(self, dataRef, datasetType, immediate=True):
579  """!Retrieve a calibration dataset for removing instrument signature
580 
581  \param[in] dataRef data reference for exposure
582  \param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
583  \param[in] immediate if True, disable butler proxies to enable error
584  handling within this routine
585  \return exposure
586  """
587  try:
588  exp = dataRef.get(datasetType, immediate=immediate)
589  except Exception as exc1:
590  if not self.config.fallbackFilterName:
591  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
592  try:
593  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
594  except Exception as exc2:
595  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
596  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
597  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
598 
599  if self.config.doAssembleIsrExposures:
600  exp = self.assembleCcd.assembleCcd(exp)
601  return exp
602 
603  def saturationDetection(self, exposure, amp):
604  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
605 
606  \param[in,out] exposure exposure to process; only the amp DataSec is processed
607  \param[in] amp amplifier device data
608  """
609  if not math.isnan(amp.getSaturation()):
610  maskedImage = exposure.getMaskedImage()
611  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
612  isrFunctions.makeThresholdMask(
613  maskedImage=dataView,
614  threshold=amp.getSaturation(),
615  growFootprints=0,
616  maskName=self.config.saturatedMaskName,
617  )
618 
619  def saturationInterpolation(self, ccdExposure):
620  """!Interpolate over saturated pixels, in place
621 
622  \param[in,out] ccdExposure exposure to process
623 
624  \warning:
625  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
626  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
627  """
628  isrFunctions.interpolateFromMask(
629  maskedImage=ccdExposure.getMaskedImage(),
630  fwhm=self.config.fwhm,
631  growFootprints=self.config.growSaturationFootprintSize,
632  maskName=self.config.saturatedMaskName,
633  )
634 
635  def suspectDetection(self, exposure, amp):
636  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
637 
638  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
639  This is intended to indicate pixels that may be affected by unknown systematics;
640  for example if non-linearity corrections above a certain level are unstable
641  then that would be a useful value for suspectLevel. A value of `nan` indicates
642  that no such level exists and no pixels are to be masked as suspicious.
643 
644  \param[in,out] exposure exposure to process; only the amp DataSec is processed
645  \param[in] amp amplifier device data
646  """
647  suspectLevel = amp.getSuspectLevel()
648  if math.isnan(suspectLevel):
649  return
650 
651  maskedImage = exposure.getMaskedImage()
652  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
653  isrFunctions.makeThresholdMask(
654  maskedImage=dataView,
655  threshold=suspectLevel,
656  growFootprints=0,
657  maskName=self.config.suspectMaskName,
658  )
659 
660  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
661  """!Mask defects using mask plane "BAD" and interpolate over them, in place
662 
663  \param[in,out] ccdExposure exposure to process
664  \param[in] defectBaseList a list of defects to mask and interpolate
665 
666  \warning: call this after CCD assembly, since defects may cross amplifier boundaries
667  """
668  maskedImage = ccdExposure.getMaskedImage()
669  defectList = []
670  for d in defectBaseList:
671  bbox = d.getBBox()
672  nd = measAlg.Defect(bbox)
673  defectList.append(nd)
674  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
675  isrFunctions.interpolateDefectList(
676  maskedImage=maskedImage,
677  defectList=defectList,
678  fwhm=self.config.fwhm,
679  )
680 
681  def maskAndInterpNan(self, exposure):
682  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
683 
684  We mask and interpolate over all NaNs, including those
685  that are masked with other bits (because those may or may
686  not be interpolated over later, and we want to remove all
687  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
688  is used to preserve the historical name.
689 
690  \param[in,out] exposure exposure to process
691  """
692  maskedImage = exposure.getMaskedImage()
693 
694  # Find and mask NaNs
695  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
696  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
697  numNans = maskNans(maskedImage, maskVal)
698  self.metadata.set("NUMNANS", numNans)
699 
700  # Interpolate over these previously-unmasked NaNs
701  if numNans > 0:
702  self.log.warn("There were %i unmasked NaNs", numNans)
703  nanDefectList = isrFunctions.getDefectListFromMask(
704  maskedImage=maskedImage,
705  maskName='UNMASKEDNAN',
706  growFootprints=0,
707  )
708  isrFunctions.interpolateDefectList(
709  maskedImage=exposure.getMaskedImage(),
710  defectList=nanDefectList,
711  fwhm=self.config.fwhm,
712  )
713 
714  def overscanCorrection(self, exposure, amp):
715  """!Apply overscan correction, in place
716 
717  \param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
718  \param[in] amp amplifier device data
719  """
720  if not amp.getHasRawInfo():
721  raise RuntimeError("This method must be executed on an amp with raw information.")
722 
723  if amp.getRawHorizontalOverscanBBox().isEmpty():
724  self.log.info("No Overscan region. Not performing Overscan Correction.")
725  return None
726 
727  maskedImage = exposure.getMaskedImage()
728  dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
729 
730  expImage = exposure.getMaskedImage().getImage()
731  overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
732 
733  isrFunctions.overscanCorrection(
734  ampMaskedImage=dataView,
735  overscanImage=overscanImage,
736  fitType=self.config.overscanFitType,
737  order=self.config.overscanOrder,
738  collapseRej=self.config.overscanRej,
739  )
740 
741  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
742  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
743 
744  \param[in,out] ccdExposure exposure to process
745  \param[in] fpPolygon Polygon in focal plane coordinates
746  """
747  # Get ccd corners in focal plane coordinates
748  ccd = ccdExposure.getDetector()
749  fpCorners = ccd.getCorners(FOCAL_PLANE)
750  ccdPolygon = Polygon(fpCorners)
751 
752  # Get intersection of ccd corners with fpPolygon
753  intersect = ccdPolygon.intersectionSingle(fpPolygon)
754 
755  # Transform back to pixel positions and build new polygon
756  ccdPoints = [ccd.transform(ccd.makeCameraPoint(x, FOCAL_PLANE), PIXELS).getPoint() for x in intersect]
757  validPolygon = Polygon(ccdPoints)
758  ccdExposure.getInfo().setValidPolygon(validPolygon)
759 
760  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
761  """Apply brighter fatter correction in place for the image
762 
763  This correction takes a kernel that has been derived from flat field images to
764  redistribute the charge. The gradient of the kernel is the deflection
765  field due to the accumulated charge.
766 
767  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
768  using the following equation:
769 
770  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
771 
772  To evaluate the derivative term we expand it as follows:
773 
774  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
775 
776  Because we use the measured counts instead of the incident counts we apply the correction
777  iteratively to reconstruct the original counts and the correction. We stop iterating when the
778  summed difference between the current corrected image and the one from the previous iteration
779  is below the threshold. We do not require convergence because the number of iterations is
780  too large a computational cost. How we define the threshold still needs to be evaluated, the
781  current default was shown to work reasonably well on a small set of images. For more information
782  on the method see DocuShare Document-19407.
783 
784  The edges as defined by the kernel are not corrected because they have spurious values
785  due to the convolution.
786  """
787  self.log.info("Applying brighter fatter correction")
788 
789  image = exposure.getMaskedImage().getImage()
790 
791  # The image needs to be units of electrons/holes
792  with self.gainContext(exposure, image, applyGain):
793 
794  kLx = numpy.shape(kernel)[0]
795  kLy = numpy.shape(kernel)[1]
796  kernelImage = afwImage.ImageD(kLx, kLy)
797  kernelImage.getArray()[:, :] = kernel
798  tempImage = image.clone()
799 
800  nanIndex = numpy.isnan(tempImage.getArray())
801  tempImage.getArray()[nanIndex] = 0.
802 
803  outImage = afwImage.ImageF(image.getDimensions())
804  corr = numpy.zeros_like(image.getArray())
805  prev_image = numpy.zeros_like(image.getArray())
806  convCntrl = afwMath.ConvolutionControl(False, True, 1)
807  fixedKernel = afwMath.FixedKernel(kernelImage)
808 
809  # Define boundary by convolution region. The region that the correction will be
810  # calculated for is one fewer in each dimension because of the second derivative terms.
811  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
812  startX = kLx//2
813  endX = -kLx//2
814  startY = kLy//2
815  endY = -kLy//2
816 
817  for iteration in range(maxIter):
818 
819  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
820  tmpArray = tempImage.getArray()
821  outArray = outImage.getArray()
822 
823  # First derivative term
824  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
825  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
826  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
827 
828  # Second derivative term
829  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
830  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
831  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
832 
833  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
834 
835  tmpArray[:, :] = image.getArray()[:, :]
836  tmpArray[nanIndex] = 0.
837  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
838 
839  if iteration > 0:
840  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
841 
842  if diff < threshold:
843  break
844  prev_image[:, :] = tmpArray[:, :]
845 
846  if iteration == maxIter - 1:
847  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
848 
849  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
850  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
851  corr[startY + 1:endY - 1, startX + 1:endX - 1]
852 
853 
854  @contextmanager
855  def gainContext(self, exp, image, apply):
856  """Context manager that applies and removes gain
857  """
858  if apply:
859  ccd = exp.getDetector()
860  for amp in ccd:
861  sim = image.Factory(image, amp.getBBox())
862  sim *= amp.getGain()
863 
864  try:
865  yield exp
866  finally:
867  if apply:
868  ccd = exp.getDetector()
869  for amp in ccd:
870  sim = image.Factory(image, amp.getBBox())
871  sim /= amp.getGain()
872 
873 
874 class FakeAmp(object):
875  """A Detector-like object that supports returning gain and saturation level"""
876 
877  def __init__(self, exposure, config):
878  self._bbox = exposure.getBBox(afwImage.LOCAL)
879  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
880  self._gain = config.gain
881  self._readNoise = config.readNoise
882  self._saturation = config.saturation
883 
884  def getBBox(self):
885  return self._bbox
886 
887  def getRawBBox(self):
888  return self._bbox
889 
890  def getHasRawInfo(self):
891  return True # but see getRawHorizontalOverscanBBox()
892 
894  return self._RawHorizontalOverscanBBox
895 
896  def getGain(self):
897  return self._gain
898 
899  def getReadNoise(self):
900  return self._readNoise
901 
902  def getSaturation(self):
903  return self._saturation
904 
905  def getSuspectLevel(self):
906  return float("NaN")
def doLinearize
Is linearization wanted for this detector?
Definition: isrTask.py:542
def getIsrExposure
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:578
def maskAndInterpNan
Mask NaNs using mask plane &quot;UNMASKEDNAN&quot; and interpolate over them, in place.
Definition: isrTask.py:681
def flatCorrection
Apply flat correction in place.
Definition: isrTask.py:565
def run
Perform instrument signature removal on an exposure.
Definition: isrTask.py:363
def __init__
Constructor for IsrTask.
Definition: isrTask.py:309
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:235
def runDataRef
Perform instrument signature removal on a ButlerDataRef of a Sensor.
Definition: isrTask.py:475
def suspectDetection
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:635
def updateVariance
Set the variance plane based on the image plane, plus amplifier gain and read noise.
Definition: isrTask.py:552
def maskAndInterpDefect
Mask defects using mask plane &quot;BAD&quot; and interpolate over them, in place.
Definition: isrTask.py:660
def setValidPolygonIntersect
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:741
def overscanCorrection
Apply overscan correction, in place.
Definition: isrTask.py:714
def saturationDetection
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:603
def darkCorrection
Apply dark correction in place.
Definition: isrTask.py:523
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def saturationInterpolation
Interpolate over saturated pixels, in place.
Definition: isrTask.py:619
def biasCorrection
Apply bias correction in place.
Definition: isrTask.py:515
def readIsrData
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:320