lsst.ip.isr  14.0-8-gb81b6e9+1
isrTask.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 from builtins import object
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 import numpy
27 
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.math as afwMath
34 from lsst.daf.persistence import ButlerDataRef
35 from lsstDebug import getDebugFrame
36 from lsst.afw.display import getDisplay
37 from . import isrFunctions
38 from .assembleCcdTask import AssembleCcdTask
39 from .fringe import FringeTask
40 from lsst.afw.geom.polygon import Polygon
41 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
42 from contextlib import contextmanager
43 from .isr import maskNans
44 from .crosstalk import CrosstalkTask
45 
46 
47 class IsrTaskConfig(pexConfig.Config):
48  doBias = pexConfig.Field(
49  dtype=bool,
50  doc="Apply bias frame correction?",
51  default=True,
52  )
53  doDark = pexConfig.Field(
54  dtype=bool,
55  doc="Apply dark frame correction?",
56  default=True,
57  )
58  doFlat = pexConfig.Field(
59  dtype=bool,
60  doc="Apply flat field correction?",
61  default=True,
62  )
63  doFringe = pexConfig.Field(
64  dtype=bool,
65  doc="Apply fringe correction?",
66  default=True,
67  )
68  doDefect = pexConfig.Field(
69  dtype=bool,
70  doc="Apply correction for CCD defects, e.g. hot pixels?",
71  default=True,
72  )
73  doWrite = pexConfig.Field(
74  dtype=bool,
75  doc="Persist postISRCCD?",
76  default=True,
77  )
78  assembleCcd = pexConfig.ConfigurableField(
79  target=AssembleCcdTask,
80  doc="CCD assembly task",
81  )
82  gain = pexConfig.Field(
83  dtype=float,
84  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
85  default=float("NaN"),
86  )
87  readNoise = pexConfig.Field(
88  dtype=float,
89  doc="The read noise to use if no Detector is present in the Exposure",
90  default=0.0,
91  )
92  saturation = pexConfig.Field(
93  dtype=float,
94  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
95  default=float("NaN"),
96  )
97  fringeAfterFlat = pexConfig.Field(
98  dtype=bool,
99  doc="Do fringe subtraction after flat-fielding?",
100  default=True,
101  )
102  fringe = pexConfig.ConfigurableField(
103  target=FringeTask,
104  doc="Fringe subtraction task",
105  )
106  fwhm = pexConfig.Field(
107  dtype=float,
108  doc="FWHM of PSF (arcsec)",
109  default=1.0,
110  )
111  saturatedMaskName = pexConfig.Field(
112  dtype=str,
113  doc="Name of mask plane to use in saturation detection and interpolation",
114  default="SAT",
115  )
116  suspectMaskName = pexConfig.Field(
117  dtype=str,
118  doc="Name of mask plane to use for suspect pixels",
119  default="SUSPECT",
120  )
121  flatScalingType = pexConfig.ChoiceField(
122  dtype=str,
123  doc="The method for scaling the flat on the fly.",
124  default='USER',
125  allowed={
126  "USER": "Scale by flatUserScale",
127  "MEAN": "Scale by the inverse of the mean",
128  "MEDIAN": "Scale by the inverse of the median",
129  },
130  )
131  flatUserScale = pexConfig.Field(
132  dtype=float,
133  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
134  default=1.0,
135  )
136  overscanFitType = pexConfig.ChoiceField(
137  dtype=str,
138  doc="The method for fitting the overscan bias level.",
139  default='MEDIAN',
140  allowed={
141  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
142  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
143  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
144  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
145  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
146  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
147  "MEAN": "Correct using the mean of the overscan region",
148  "MEDIAN": "Correct using the median of the overscan region",
149  },
150  )
151  overscanOrder = pexConfig.Field(
152  dtype=int,
153  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
154  "or number of spline knots if overscan fit type is a spline."),
155  default=1,
156  )
157  overscanRej = pexConfig.Field(
158  dtype=float,
159  doc="Rejection threshold (sigma) for collapsing overscan before fit",
160  default=3.0,
161  )
162  growSaturationFootprintSize = pexConfig.Field(
163  dtype=int,
164  doc="Number of pixels by which to grow the saturation footprints",
165  default=1,
166  )
167  doSaturationInterpolation = pexConfig.Field(
168  dtype=bool,
169  doc="Perform interpolation over pixels masked as saturated?",
170  default=True,
171  )
172  doNanInterpAfterFlat = pexConfig.Field(
173  dtype=bool,
174  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
175  "also have to interpolate them before flat-fielding."),
176  default=False,
177  )
178  fluxMag0T1 = pexConfig.Field(
179  dtype=float,
180  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
181  default=1e10,
182  )
183  keysToRemoveFromAssembledCcd = pexConfig.ListField(
184  dtype=str,
185  doc="fields to remove from the metadata of the assembled ccd.",
186  default=[],
187  )
188  doAssembleIsrExposures = pexConfig.Field(
189  dtype=bool,
190  default=False,
191  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
192  )
193  doAssembleCcd = pexConfig.Field(
194  dtype=bool,
195  default=True,
196  doc="Assemble amp-level exposures into a ccd-level exposure?"
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)"
202  )
203  doLinearize = pexConfig.Field(
204  dtype=bool,
205  doc="Correct for nonlinearity of the detector's response?",
206  default=True,
207  )
208  doCrosstalk = pexConfig.Field(
209  dtype=bool,
210  doc="Apply intra-CCD crosstalk correction?",
211  default=False,
212  )
213  crosstalk = pexConfig.ConfigurableField(
214  target=CrosstalkTask,
215  doc="Intra-CCD crosstalk correction",
216  )
217  doBrighterFatter = pexConfig.Field(
218  dtype=bool,
219  default=False,
220  doc="Apply the brighter fatter correction"
221  )
222  brighterFatterKernelFile = pexConfig.Field(
223  dtype=str,
224  default='',
225  doc="Kernel file used for the brighter fatter correction"
226  )
227  brighterFatterMaxIter = pexConfig.Field(
228  dtype=int,
229  default=10,
230  doc="Maximum number of iterations for the brighter fatter correction"
231  )
232  brighterFatterThreshold = pexConfig.Field(
233  dtype=float,
234  default=1000,
235  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
236  " absolute value of the difference between the current corrected image and the one"
237  " from the previous iteration summed over all the pixels."
238  )
239  brighterFatterApplyGain = pexConfig.Field(
240  dtype=bool,
241  default=True,
242  doc="Should the gain be applied when applying the brighter fatter correction?"
243  )
244  datasetType = pexConfig.Field(
245  dtype=str,
246  doc="Dataset type for input data; users will typically leave this alone, "
247  "but camera-specific ISR tasks will override it",
248  default="raw",
249  )
250  fallbackFilterName = pexConfig.Field(dtype=str,
251  doc="Fallback default filter name for calibrations", optional=True)
252  doAttachTransmissionCurve = pexConfig.Field(
253  dtype=bool,
254  default=False,
255  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
256  )
257  doUseOpticsTransmission = pexConfig.Field(
258  dtype=bool,
259  default=True,
260  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
261  )
262  doUseFilterTransmission = pexConfig.Field(
263  dtype=bool,
264  default=True,
265  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
266  )
267  doUseSensorTransmission = pexConfig.Field(
268  dtype=bool,
269  default=True,
270  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
271  )
272  doUseAtmosphereTransmission = pexConfig.Field(
273  dtype=bool,
274  default=True,
275  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
276  )
277 
278 
279 
285 
286 
287 class IsrTask(pipeBase.CmdLineTask):
288  """!
289  \anchor IsrTask_
290 
291  \brief Apply common instrument signature correction algorithms to a raw frame.
292 
293  \section ip_isr_isr_Contents Contents
294 
295  - \ref ip_isr_isr_Purpose
296  - \ref ip_isr_isr_Initialize
297  - \ref ip_isr_isr_IO
298  - \ref ip_isr_isr_Config
299  - \ref ip_isr_isr_Debug
300 
301 
302  \section ip_isr_isr_Purpose Description
303 
304  The process for correcting imaging data is very similar from camera to camera.
305  This task provides a vanilla implementation of doing these corrections, including
306  the ability to turn certain corrections off if they are not needed.
307  The inputs to the primary method, run, are a raw exposure to be corrected and the
308  calibration data products. The raw input is a single chip sized mosaic of all amps
309  including overscans and other non-science pixels.
310  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
311  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
312  This task may not meet all needs and it is expected that it will be subclassed for
313  specific applications.
314 
315  \section ip_isr_isr_Initialize Task initialization
316 
317  \copydoc \_\_init\_\_
318 
319  \section ip_isr_isr_IO Inputs/Outputs to the run method
320 
321  \copydoc run
322 
323  \section ip_isr_isr_Config Configuration parameters
324 
325  See \ref IsrTaskConfig
326 
327  \section ip_isr_isr_Debug Debug variables
328 
329  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
330  flag \c --debug, \c -d to import \b debug.py from your \c PYTHONPATH; see <a
331  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
332  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
333 
334  The available variables in IsrTask are:
335  <DL>
336  <DT> \c display
337  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
338  <DL>
339  <DT> postISRCCD
340  <DD> display exposure after ISR has been applied
341  </DL>
342  </DL>
343 
344  For example, put something like
345  \code{.py}
346  import lsstDebug
347  def DebugInfo(name):
348  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
349  if name == "lsst.ip.isrFunctions.isrTask":
350  di.display = {'postISRCCD':2}
351  return di
352  lsstDebug.Info = DebugInfo
353  \endcode
354  into your debug.py file and run the commandline task with the \c --debug flag.
355 
356  <HR>
357  """
358  ConfigClass = IsrTaskConfig
359  _DefaultName = "isr"
360 
361  def __init__(self, *args, **kwargs):
362  '''!Constructor for IsrTask
363  @param[in] *args a list of positional arguments passed on to the Task constructor
364  @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor
365  Call the lsst.pipe.base.task.Task.__init__ method
366  Then setup the assembly and fringe correction subtasks
367  '''
368  pipeBase.Task.__init__(self, *args, **kwargs)
369  self.makeSubtask("assembleCcd")
370  self.makeSubtask("fringe")
371  self.makeSubtask("crosstalk")
372 
373  def readIsrData(self, dataRef, rawExposure):
374  """!Retrieve necessary frames for instrument signature removal
375  @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef
376  of the detector data to be processed
377  @param[in] rawExposure a reference raw exposure that will later be
378  corrected with the retrieved calibration data;
379  should not be modified in this method.
380  \return a pipeBase.Struct with fields containing kwargs expected by run()
381  - bias: exposure of bias frame
382  - dark: exposure of dark frame
383  - flat: exposure of flat field
384  - defects: list of detects
385  - fringeStruct: a pipeBase.Struct with field fringes containing
386  exposure of fringe frame or list of fringe exposure
387  """
388  ccd = rawExposure.getDetector()
389 
390  biasExposure = self.getIsrExposure(dataRef, "bias") if self.config.doBias else None
391  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
392  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
393  darkExposure = self.getIsrExposure(dataRef, "dark") if self.config.doDark else None
394  flatExposure = self.getIsrExposure(dataRef, "flat") if self.config.doFlat else None
395  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
396  defectList = dataRef.get("defects") if self.config.doDefect else None
397 
398  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
399  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
400  if self.config.doAssembleIsrExposures else None)
401  else:
402  fringeStruct = pipeBase.Struct(fringes=None)
403 
404  if self.config.doAttachTransmissionCurve:
405  opticsTransmission = (dataRef.get("transmission_optics")
406  if self.config.doUseOpticsTransmission else None)
407  filterTransmission = (dataRef.get("transmission_filter")
408  if self.config.doUseFilterTransmission else None)
409  sensorTransmission = (dataRef.get("transmission_sensor")
410  if self.config.doUseSensorTransmission else None)
411  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
412  if self.config.doUseAtmosphereTransmission else None)
413  else:
414  opticsTransmission = None
415  filterTransmission = None
416  sensorTransmission = None
417  atmosphereTransmission = None
418 
419  # Struct should include only kwargs to run()
420  return pipeBase.Struct(bias=biasExposure,
421  linearizer=linearizer,
422  dark=darkExposure,
423  flat=flatExposure,
424  defects=defectList,
425  fringes=fringeStruct,
426  bfKernel=brighterFatterKernel,
427  opticsTransmission=opticsTransmission,
428  filterTransmission=filterTransmission,
429  sensorTransmission=sensorTransmission,
430  atmosphereTransmission=atmosphereTransmission,
431  )
432 
433  @pipeBase.timeMethod
434  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
435  fringes=None, bfKernel=None,
436  opticsTransmission=None, filterTransmission=None,
437  sensorTransmission=None, atmosphereTransmission=None):
438  """!Perform instrument signature removal on an exposure
439 
440  Steps include:
441  - Detect saturation, apply overscan correction, bias, dark and flat
442  - Perform CCD assembly
443  - Interpolate over defects, saturated pixels and all NaNs
444 
445  @param[in] ccdExposure lsst.afw.image.exposure of detector data
446  @param[in] bias exposure of bias frame
447  @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
448  @param[in] dark exposure of dark frame
449  @param[in] flat exposure of flatfield
450  @param[in] defects list of detects
451  @param[in] fringes a pipeBase.Struct with field fringes containing
452  exposure of fringe frame or list of fringe exposure
453  @param[in] bfKernel kernel for brighter-fatter correction
454  @param[in] opticsTransmission a TransmissionCurve for the optics
455  @param[in] filterTransmission a TransmissionCurve for the filter
456  @param[in] sensorTransmission a TransmissionCurve for the sensor
457  @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere
458 
459  @return a pipeBase.Struct with field:
460  - exposure
461  """
462  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
463  if isinstance(ccdExposure, ButlerDataRef):
464  return self.runDataRef(ccdExposure)
465 
466  ccd = ccdExposure.getDetector()
467 
468  # Validate Input
469  if self.config.doBias and bias is None:
470  raise RuntimeError("Must supply a bias exposure if config.doBias True")
471  if self.doLinearize(ccd) and linearizer is None:
472  raise RuntimeError("Must supply a linearizer if config.doBias True")
473  if self.config.doDark and dark is None:
474  raise RuntimeError("Must supply a dark exposure if config.doDark True")
475  if self.config.doFlat and flat is None:
476  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
477  if self.config.doBrighterFatter and bfKernel is None:
478  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
479  if fringes is None:
480  fringes = pipeBase.Struct(fringes=None)
481  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
482  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
483  if self.config.doDefect and defects is None:
484  raise RuntimeError("Must supply defects if config.doDefect True")
485 
486  ccdExposure = self.convertIntToFloat(ccdExposure)
487 
488  if not ccd:
489  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
490  ccd = [FakeAmp(ccdExposure, self.config)]
491 
492  for amp in ccd:
493  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
494  if ccdExposure.getBBox().contains(amp.getBBox()):
495  self.saturationDetection(ccdExposure, amp)
496  self.suspectDetection(ccdExposure, amp)
497  self.overscanCorrection(ccdExposure, amp)
498 
499  if self.config.doAssembleCcd:
500  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
501  if self.config.expectWcs and not ccdExposure.getWcs():
502  self.log.warn("No WCS found in input exposure")
503 
504  if self.config.doBias:
505  self.biasCorrection(ccdExposure, bias)
506 
507  if self.doLinearize(ccd):
508  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
509 
510  if self.config.doCrosstalk:
511  self.crosstalk.run(ccdExposure)
512 
513  for amp in ccd:
514  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
515  if ccdExposure.getBBox().contains(amp.getBBox()):
516  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
517  self.updateVariance(ampExposure, amp)
518 
519  interpolationDone = False
520 
521  if self.config.doBrighterFatter:
522 
523  # We need to apply flats and darks before we can interpolate, and we
524  # need to interpolate before we do B-F, but we do B-F without the
525  # flats and darks applied so we can work in units of electrons or holes.
526  # This context manager applies and then removes the darks and flats.
527  with self.flatContext(ccdExposure, flat, dark):
528  if self.config.doDefect:
529  self.maskAndInterpDefect(ccdExposure, defects)
530  if self.config.doSaturationInterpolation:
531  self.saturationInterpolation(ccdExposure)
532  self.maskAndInterpNan(ccdExposure)
533  interpolationDone = True
534 
535  self.brighterFatterCorrection(ccdExposure, bfKernel,
536  self.config.brighterFatterMaxIter,
537  self.config.brighterFatterThreshold,
538  self.config.brighterFatterApplyGain,
539  )
540 
541  if self.config.doDark:
542  self.darkCorrection(ccdExposure, dark)
543 
544  if self.config.doFringe and not self.config.fringeAfterFlat:
545  self.fringe.run(ccdExposure, **fringes.getDict())
546 
547  if self.config.doFlat:
548  self.flatCorrection(ccdExposure, flat)
549 
550  if not interpolationDone:
551  if self.config.doDefect:
552  self.maskAndInterpDefect(ccdExposure, defects)
553  if self.config.doSaturationInterpolation:
554  self.saturationInterpolation(ccdExposure)
555  if not interpolationDone or self.config.doNanInterpAfterFlat:
556  self.maskAndInterpNan(ccdExposure)
557 
558  if self.config.doFringe and self.config.fringeAfterFlat:
559  self.fringe.run(ccdExposure, **fringes.getDict())
560 
561  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
562  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
563 
564  if self.config.doAttachTransmissionCurve:
565  self.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
566  filterTransmission=filterTransmission,
567  sensorTransmission=sensorTransmission,
568  atmosphereTransmission=atmosphereTransmission)
569 
570  frame = getDebugFrame(self._display, "postISRCCD")
571  if frame:
572  getDisplay(frame).mtv(ccdExposure)
573 
574  return pipeBase.Struct(
575  exposure=ccdExposure,
576  )
577 
578  @pipeBase.timeMethod
579  def runDataRef(self, sensorRef):
580  """!Perform instrument signature removal on a ButlerDataRef of a Sensor
581 
582  - Read in necessary detrending/isr/calibration data
583  - Process raw exposure in run()
584  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
585 
586  @param[in] sensorRef daf.persistence.butlerSubset.ButlerDataRef of the
587  detector data to be processed
588  @return a pipeBase.Struct with fields:
589  - exposure: the exposure after application of ISR
590  """
591  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
592  ccdExposure = sensorRef.get('raw')
593  isrData = self.readIsrData(sensorRef, ccdExposure)
594 
595  result = self.run(ccdExposure, **isrData.getDict())
596 
597  if self.config.doWrite:
598  sensorRef.put(result.exposure, "postISRCCD")
599 
600  return result
601 
602  def convertIntToFloat(self, exposure):
603  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
604  """
605  if isinstance(exposure, afwImage.ExposureF):
606  # Nothing to be done
607  return exposure
608  if not hasattr(exposure, "convertF"):
609  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
610 
611  newexposure = exposure.convertF()
612  maskedImage = newexposure.getMaskedImage()
613  varArray = maskedImage.getVariance().getArray()
614  varArray[:, :] = 1
615  maskArray = maskedImage.getMask().getArray()
616  maskArray[:, :] = 0
617  return newexposure
618 
619  def biasCorrection(self, exposure, biasExposure):
620  """!Apply bias correction in place
621 
622  @param[in,out] exposure exposure to process
623  @param[in] biasExposure bias exposure of same size as exposure
624  """
625  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
626 
627  def darkCorrection(self, exposure, darkExposure, invert=False):
628  """!Apply dark correction in place
629 
630  @param[in,out] exposure exposure to process
631  @param[in] darkExposure dark exposure of same size as exposure
632  @param[in] invert if True, remove the dark from an already-corrected image
633  """
634  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
635  if math.isnan(expScale):
636  raise RuntimeError("Exposure darktime is NAN")
637  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
638  if math.isnan(darkScale):
639  raise RuntimeError("Dark calib darktime is NAN")
640  isrFunctions.darkCorrection(
641  maskedImage=exposure.getMaskedImage(),
642  darkMaskedImage=darkExposure.getMaskedImage(),
643  expScale=expScale,
644  darkScale=darkScale,
645  invert=invert
646  )
647 
648  def doLinearize(self, detector):
649  """!Is linearization wanted for this detector?
650 
651  Checks config.doLinearize and the linearity type of the first amplifier.
652 
653  @param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
654  """
655  return self.config.doLinearize and \
656  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
657 
658  def updateVariance(self, ampExposure, amp):
659  """!Set the variance plane based on the image plane, plus amplifier gain and read noise
660 
661  @param[in,out] ampExposure exposure to process
662  @param[in] amp amplifier detector information
663  """
664  gain = amp.getGain()
665  if not math.isnan(gain):
666  if gain <= 0:
667  patchedGain = 1.0
668  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" % (amp.getName(), gain, patchedGain))
669  gain = patchedGain
670 
671  isrFunctions.updateVariance(
672  maskedImage=ampExposure.getMaskedImage(),
673  gain=gain,
674  readNoise=amp.getReadNoise(),
675  )
676 
677  def flatCorrection(self, exposure, flatExposure, invert=False):
678  """!Apply flat correction in place
679 
680  @param[in,out] exposure exposure to process
681  @param[in] flatExposure flatfield exposure same size as exposure
682  @param[in] invert if True, unflatten an already-flattened image instead.
683  """
684  isrFunctions.flatCorrection(
685  maskedImage=exposure.getMaskedImage(),
686  flatMaskedImage=flatExposure.getMaskedImage(),
687  scalingType=self.config.flatScalingType,
688  userScale=self.config.flatUserScale,
689  invert=invert
690  )
691 
692  def getIsrExposure(self, dataRef, datasetType, immediate=True):
693  """!Retrieve a calibration dataset for removing instrument signature
694 
695  @param[in] dataRef data reference for exposure
696  @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
697  @param[in] immediate if True, disable butler proxies to enable error
698  handling within this routine
699  \return exposure
700  """
701  try:
702  exp = dataRef.get(datasetType, immediate=immediate)
703  except Exception as exc1:
704  if not self.config.fallbackFilterName:
705  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
706  try:
707  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
708  except Exception as exc2:
709  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
710  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
711  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
712 
713  if self.config.doAssembleIsrExposures:
714  exp = self.assembleCcd.assembleCcd(exp)
715  return exp
716 
717  def saturationDetection(self, exposure, amp):
718  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
719 
720  @param[in,out] exposure exposure to process; only the amp DataSec is processed
721  @param[in] amp amplifier device data
722  """
723  if not math.isnan(amp.getSaturation()):
724  maskedImage = exposure.getMaskedImage()
725  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
726  isrFunctions.makeThresholdMask(
727  maskedImage=dataView,
728  threshold=amp.getSaturation(),
729  growFootprints=0,
730  maskName=self.config.saturatedMaskName,
731  )
732 
733  def saturationInterpolation(self, ccdExposure):
734  """!Interpolate over saturated pixels, in place
735 
736  @param[in,out] ccdExposure exposure to process
737 
738  \warning:
739  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
740  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
741  """
742  isrFunctions.interpolateFromMask(
743  maskedImage=ccdExposure.getMaskedImage(),
744  fwhm=self.config.fwhm,
745  growFootprints=self.config.growSaturationFootprintSize,
746  maskName=self.config.saturatedMaskName,
747  )
748 
749  def suspectDetection(self, exposure, amp):
750  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
751 
752  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
753  This is intended to indicate pixels that may be affected by unknown systematics;
754  for example if non-linearity corrections above a certain level are unstable
755  then that would be a useful value for suspectLevel. A value of `nan` indicates
756  that no such level exists and no pixels are to be masked as suspicious.
757 
758  @param[in,out] exposure exposure to process; only the amp DataSec is processed
759  @param[in] amp amplifier device data
760  """
761  suspectLevel = amp.getSuspectLevel()
762  if math.isnan(suspectLevel):
763  return
764 
765  maskedImage = exposure.getMaskedImage()
766  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
767  isrFunctions.makeThresholdMask(
768  maskedImage=dataView,
769  threshold=suspectLevel,
770  growFootprints=0,
771  maskName=self.config.suspectMaskName,
772  )
773 
774  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
775  """!Mask defects using mask plane "BAD" and interpolate over them, in place
776 
777  @param[in,out] ccdExposure exposure to process
778  @param[in] defectBaseList a list of defects to mask and interpolate
779 
780  \warning: call this after CCD assembly, since defects may cross amplifier boundaries
781  """
782  maskedImage = ccdExposure.getMaskedImage()
783  defectList = []
784  for d in defectBaseList:
785  bbox = d.getBBox()
786  nd = measAlg.Defect(bbox)
787  defectList.append(nd)
788  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
789  isrFunctions.interpolateDefectList(
790  maskedImage=maskedImage,
791  defectList=defectList,
792  fwhm=self.config.fwhm,
793  )
794 
795  def maskAndInterpNan(self, exposure):
796  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
797 
798  We mask and interpolate over all NaNs, including those
799  that are masked with other bits (because those may or may
800  not be interpolated over later, and we want to remove all
801  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
802  is used to preserve the historical name.
803 
804  @param[in,out] exposure exposure to process
805  """
806  maskedImage = exposure.getMaskedImage()
807 
808  # Find and mask NaNs
809  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
810  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
811  numNans = maskNans(maskedImage, maskVal)
812  self.metadata.set("NUMNANS", numNans)
813 
814  # Interpolate over these previously-unmasked NaNs
815  if numNans > 0:
816  self.log.warn("There were %i unmasked NaNs", numNans)
817  nanDefectList = isrFunctions.getDefectListFromMask(
818  maskedImage=maskedImage,
819  maskName='UNMASKEDNAN',
820  )
821  isrFunctions.interpolateDefectList(
822  maskedImage=exposure.getMaskedImage(),
823  defectList=nanDefectList,
824  fwhm=self.config.fwhm,
825  )
826 
827  def overscanCorrection(self, exposure, amp):
828  """!Apply overscan correction, in place
829 
830  @param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
831  @param[in] amp amplifier device data
832  """
833  if not amp.getHasRawInfo():
834  raise RuntimeError("This method must be executed on an amp with raw information.")
835 
836  if amp.getRawHorizontalOverscanBBox().isEmpty():
837  self.log.info("No Overscan region. Not performing Overscan Correction.")
838  return None
839 
840  maskedImage = exposure.getMaskedImage()
841  dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
842 
843  expImage = exposure.getMaskedImage().getImage()
844  overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
845 
846  isrFunctions.overscanCorrection(
847  ampMaskedImage=dataView,
848  overscanImage=overscanImage,
849  fitType=self.config.overscanFitType,
850  order=self.config.overscanOrder,
851  collapseRej=self.config.overscanRej,
852  )
853 
854  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
855  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
856 
857  @param[in,out] ccdExposure exposure to process
858  @param[in] fpPolygon Polygon in focal plane coordinates
859  """
860  # Get ccd corners in focal plane coordinates
861  ccd = ccdExposure.getDetector()
862  fpCorners = ccd.getCorners(FOCAL_PLANE)
863  ccdPolygon = Polygon(fpCorners)
864 
865  # Get intersection of ccd corners with fpPolygon
866  intersect = ccdPolygon.intersectionSingle(fpPolygon)
867 
868  # Transform back to pixel positions and build new polygon
869  ccdPoints = [ccd.transform(ccd.makeCameraPoint(x, FOCAL_PLANE), PIXELS).getPoint() for x in intersect]
870  validPolygon = Polygon(ccdPoints)
871  ccdExposure.getInfo().setValidPolygon(validPolygon)
872 
873  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
874  """Apply brighter fatter correction in place for the image
875 
876  This correction takes a kernel that has been derived from flat field images to
877  redistribute the charge. The gradient of the kernel is the deflection
878  field due to the accumulated charge.
879 
880  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
881  using the following equation:
882 
883  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
884 
885  To evaluate the derivative term we expand it as follows:
886 
887  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
888 
889  Because we use the measured counts instead of the incident counts we apply the correction
890  iteratively to reconstruct the original counts and the correction. We stop iterating when the
891  summed difference between the current corrected image and the one from the previous iteration
892  is below the threshold. We do not require convergence because the number of iterations is
893  too large a computational cost. How we define the threshold still needs to be evaluated, the
894  current default was shown to work reasonably well on a small set of images. For more information
895  on the method see DocuShare Document-19407.
896 
897  The edges as defined by the kernel are not corrected because they have spurious values
898  due to the convolution.
899  """
900  self.log.info("Applying brighter fatter correction")
901 
902  image = exposure.getMaskedImage().getImage()
903 
904  # The image needs to be units of electrons/holes
905  with self.gainContext(exposure, image, applyGain):
906 
907  kLx = numpy.shape(kernel)[0]
908  kLy = numpy.shape(kernel)[1]
909  kernelImage = afwImage.ImageD(kLx, kLy)
910  kernelImage.getArray()[:, :] = kernel
911  tempImage = image.clone()
912 
913  nanIndex = numpy.isnan(tempImage.getArray())
914  tempImage.getArray()[nanIndex] = 0.
915 
916  outImage = afwImage.ImageF(image.getDimensions())
917  corr = numpy.zeros_like(image.getArray())
918  prev_image = numpy.zeros_like(image.getArray())
919  convCntrl = afwMath.ConvolutionControl(False, True, 1)
920  fixedKernel = afwMath.FixedKernel(kernelImage)
921 
922  # Define boundary by convolution region. The region that the correction will be
923  # calculated for is one fewer in each dimension because of the second derivative terms.
924  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
925  startX = kLx//2
926  endX = -kLx//2
927  startY = kLy//2
928  endY = -kLy//2
929 
930  for iteration in range(maxIter):
931 
932  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
933  tmpArray = tempImage.getArray()
934  outArray = outImage.getArray()
935 
936  with numpy.errstate(invalid="ignore", over="ignore"):
937  # First derivative term
938  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
939  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
940  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
941 
942  # Second derivative term
943  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
944  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
945  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
946 
947  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
948 
949  tmpArray[:, :] = image.getArray()[:, :]
950  tmpArray[nanIndex] = 0.
951  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
952 
953  if iteration > 0:
954  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
955 
956  if diff < threshold:
957  break
958  prev_image[:, :] = tmpArray[:, :]
959 
960  if iteration == maxIter - 1:
961  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
962 
963  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
964  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
965  corr[startY + 1:endY - 1, startX + 1:endX - 1]
966 
967  def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None,
968  sensorTransmission=None, atmosphereTransmission=None):
969  """Attach a TransmissionCurve to an Exposure, given separate curves for
970  different components.
971 
972  Parameters
973  ----------
974  exposure : `lsst.afw.image.Exposure`
975  Exposure object to modify by attaching the product of all given
976  ``TransmissionCurves`` in post-assembly trimmed detector
977  coordinates. Must have a valid ``Detector`` attached that matches
978  the detector associated with sensorTransmission.
979  opticsTransmission : `lsst.afw.image.TransmissionCurve`
980  A ``TransmissionCurve`` that represents the throughput of the
981  optics, to be evaluated in focal-plane coordinates.
982  filterTransmission : `lsst.afw.image.TransmissionCurve`
983  A ``TransmissionCurve`` that represents the throughput of the
984  filter itself, to be evaluated in focal-plane coordinates.
985  sensorTransmission : `lsst.afw.image.TransmissionCurve`
986  A ``TransmissionCurve`` that represents the throughput of the
987  sensor itself, to be evaluated in post-assembly trimmed detector
988  coordinates.
989  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
990  A ``TransmissionCurve`` that represents the throughput of the
991  atmosphere, assumed to be spatially constant.
992 
993  All ``TransmissionCurve`` arguments are optional; if none are provided,
994  the attached ``TransmissionCurve`` will have unit transmission
995  everywhere.
996 
997  Returns
998  -------
999  combined : ``lsst.afw.image.TransmissionCurve``
1000  The TransmissionCurve attached to the exposure.
1001  """
1002  return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1003  filterTransmission=filterTransmission,
1004  sensorTransmission=sensorTransmission,
1005  atmosphereTransmission=atmosphereTransmission)
1006 
1007  @contextmanager
1008  def gainContext(self, exp, image, apply):
1009  """Context manager that applies and removes gain
1010  """
1011  if apply:
1012  ccd = exp.getDetector()
1013  for amp in ccd:
1014  sim = image.Factory(image, amp.getBBox())
1015  sim *= amp.getGain()
1016 
1017  try:
1018  yield exp
1019  finally:
1020  if apply:
1021  ccd = exp.getDetector()
1022  for amp in ccd:
1023  sim = image.Factory(image, amp.getBBox())
1024  sim /= amp.getGain()
1025 
1026 
1027  @contextmanager
1028  def flatContext(self, exp, flat, dark=None):
1029  """Context manager that applies and removes flats and darks,
1030  if the task is configured to apply them.
1031  """
1032  if self.config.doDark and dark is not None:
1033  self.darkCorrection(exp, dark)
1034  if self.config.doFlat:
1035  self.flatCorrection(exp, flat)
1036  try:
1037  yield exp
1038  finally:
1039  if self.config.doFlat:
1040  self.flatCorrection(exp, flat, invert=True)
1041  if self.config.doDark and dark is not None:
1042  self.darkCorrection(exp, dark, invert=True)
1043 
1044 
1045 class FakeAmp(object):
1046  """A Detector-like object that supports returning gain and saturation level"""
1047 
1048  def __init__(self, exposure, config):
1049  self._bbox = exposure.getBBox(afwImage.LOCAL)
1050  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
1051  self._gain = config.gain
1052  self._readNoise = config.readNoise
1053  self._saturation = config.saturation
1054 
1055  def getBBox(self):
1056  return self._bbox
1057 
1058  def getRawBBox(self):
1059  return self._bbox
1060 
1061  def getHasRawInfo(self):
1062  return True # but see getRawHorizontalOverscanBBox()
1063 
1065  return self._RawHorizontalOverscanBBox
1066 
1067  def getGain(self):
1068  return self._gain
1069 
1070  def getReadNoise(self):
1071  return self._readNoise
1072 
1073  def getSaturation(self):
1074  return self._saturation
1075 
1076  def getSuspectLevel(self):
1077  return float("NaN")
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
Definition: isrTask.py:873
def runDataRef(self, sensorRef)
Perform instrument signature removal on a ButlerDataRef of a Sensor.
Definition: isrTask.py:579
def gainContext(self, exp, image, apply)
Definition: isrTask.py:1008
def __init__(self, args, kwargs)
Constructor for IsrTask.
Definition: isrTask.py:361
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:373
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Definition: isrTask.py:968
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:795
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:733
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:287
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:1064
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:437
def convertIntToFloat(self, exposure)
Definition: isrTask.py:602
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:677
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:692
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:627
def doLinearize(self, detector)
Is linearization wanted for this detector?
Definition: isrTask.py:648
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:854
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
Definition: isrTask.py:619
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:1028
def updateVariance(self, ampExposure, amp)
Set the variance plane based on the image plane, plus amplifier gain and read noise.
Definition: isrTask.py:658
def overscanCorrection(self, exposure, amp)
Apply overscan correction, in place.
Definition: isrTask.py:827
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:749
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:774
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:717
def __init__(self, exposure, config)
Definition: isrTask.py:1048