lsst.ip.isr  14.0-4-g3dd7f2d+1
isrTask.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 from builtins import object
4 #
5 # LSST Data Management System
6 # Copyright 2008-2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 import numpy
27 
28 import lsst.afw.geom as afwGeom
29 import lsst.afw.image as afwImage
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.math as afwMath
34 from lsst.daf.persistence import ButlerDataRef
35 from lsstDebug import getDebugFrame
36 from lsst.afw.display import getDisplay
37 from . import isrFunctions
38 from .assembleCcdTask import AssembleCcdTask
39 from .fringe import FringeTask
40 from lsst.afw.geom.polygon import Polygon
41 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
42 from contextlib import contextmanager
43 from .isr import maskNans
44 from .crosstalk import CrosstalkTask
45 
46 
47 class IsrTaskConfig(pexConfig.Config):
48  doBias = pexConfig.Field(
49  dtype=bool,
50  doc="Apply bias frame correction?",
51  default=True,
52  )
53  doDark = pexConfig.Field(
54  dtype=bool,
55  doc="Apply dark frame correction?",
56  default=True,
57  )
58  doFlat = pexConfig.Field(
59  dtype=bool,
60  doc="Apply flat field correction?",
61  default=True,
62  )
63  doFringe = pexConfig.Field(
64  dtype=bool,
65  doc="Apply fringe correction?",
66  default=True,
67  )
68  doDefect = pexConfig.Field(
69  dtype=bool,
70  doc="Apply correction for CCD defects, e.g. hot pixels?",
71  default=True,
72  )
73  doWrite = pexConfig.Field(
74  dtype=bool,
75  doc="Persist postISRCCD?",
76  default=True,
77  )
78  assembleCcd = pexConfig.ConfigurableField(
79  target=AssembleCcdTask,
80  doc="CCD assembly task",
81  )
82  gain = pexConfig.Field(
83  dtype=float,
84  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
85  default=float("NaN"),
86  )
87  readNoise = pexConfig.Field(
88  dtype=float,
89  doc="The read noise to use if no Detector is present in the Exposure",
90  default=0.0,
91  )
92  saturation = pexConfig.Field(
93  dtype=float,
94  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
95  default=float("NaN"),
96  )
97  fringeAfterFlat = pexConfig.Field(
98  dtype=bool,
99  doc="Do fringe subtraction after flat-fielding?",
100  default=True,
101  )
102  fringe = pexConfig.ConfigurableField(
103  target=FringeTask,
104  doc="Fringe subtraction task",
105  )
106  fwhm = pexConfig.Field(
107  dtype=float,
108  doc="FWHM of PSF (arcsec)",
109  default=1.0,
110  )
111  saturatedMaskName = pexConfig.Field(
112  dtype=str,
113  doc="Name of mask plane to use in saturation detection and interpolation",
114  default="SAT",
115  )
116  suspectMaskName = pexConfig.Field(
117  dtype=str,
118  doc="Name of mask plane to use for suspect pixels",
119  default="SUSPECT",
120  )
121  flatScalingType = pexConfig.ChoiceField(
122  dtype=str,
123  doc="The method for scaling the flat on the fly.",
124  default='USER',
125  allowed={
126  "USER": "Scale by flatUserScale",
127  "MEAN": "Scale by the inverse of the mean",
128  "MEDIAN": "Scale by the inverse of the median",
129  },
130  )
131  flatUserScale = pexConfig.Field(
132  dtype=float,
133  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
134  default=1.0,
135  )
136  overscanFitType = pexConfig.ChoiceField(
137  dtype=str,
138  doc="The method for fitting the overscan bias level.",
139  default='MEDIAN',
140  allowed={
141  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
142  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
143  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
144  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
145  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
146  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
147  "MEAN": "Correct using the mean of the overscan region",
148  "MEDIAN": "Correct using the median of the overscan region",
149  },
150  )
151  overscanOrder = pexConfig.Field(
152  dtype=int,
153  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
154  "or number of spline knots if overscan fit type is a spline."),
155  default=1,
156  )
157  overscanRej = pexConfig.Field(
158  dtype=float,
159  doc="Rejection threshold (sigma) for collapsing overscan before fit",
160  default=3.0,
161  )
162  growSaturationFootprintSize = pexConfig.Field(
163  dtype=int,
164  doc="Number of pixels by which to grow the saturation footprints",
165  default=1,
166  )
167  doSaturationInterpolation = pexConfig.Field(
168  dtype=bool,
169  doc="Perform interpolation over pixels masked as saturated?",
170  default=True,
171  )
172  doNanInterpAfterFlat = pexConfig.Field(
173  dtype=bool,
174  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
175  "also have to interpolate them before flat-fielding."),
176  default=False,
177  )
178  fluxMag0T1 = pexConfig.Field(
179  dtype=float,
180  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
181  default=1e10,
182  )
183  keysToRemoveFromAssembledCcd = pexConfig.ListField(
184  dtype=str,
185  doc="fields to remove from the metadata of the assembled ccd.",
186  default=[],
187  )
188  doAssembleIsrExposures = pexConfig.Field(
189  dtype=bool,
190  default=False,
191  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
192  )
193  doAssembleCcd = pexConfig.Field(
194  dtype=bool,
195  default=True,
196  doc="Assemble amp-level exposures into a ccd-level exposure?"
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)"
202  )
203  doLinearize = pexConfig.Field(
204  dtype=bool,
205  doc="Correct for nonlinearity of the detector's response?",
206  default=True,
207  )
208  doCrosstalk = pexConfig.Field(
209  dtype=bool,
210  doc="Apply intra-CCD crosstalk correction?",
211  default=False,
212  )
213  crosstalk = pexConfig.ConfigurableField(
214  target=CrosstalkTask,
215  doc="Intra-CCD crosstalk correction",
216  )
217  doBrighterFatter = pexConfig.Field(
218  dtype=bool,
219  default=False,
220  doc="Apply the brighter fatter correction"
221  )
222  brighterFatterKernelFile = pexConfig.Field(
223  dtype=str,
224  default='',
225  doc="Kernel file used for the brighter fatter correction"
226  )
227  brighterFatterMaxIter = pexConfig.Field(
228  dtype=int,
229  default=10,
230  doc="Maximum number of iterations for the brighter fatter correction"
231  )
232  brighterFatterThreshold = pexConfig.Field(
233  dtype=float,
234  default=1000,
235  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
236  " absolute value of the difference between the current corrected image and the one"
237  " from the previous iteration summed over all the pixels."
238  )
239  brighterFatterApplyGain = pexConfig.Field(
240  dtype=bool,
241  default=True,
242  doc="Should the gain be applied when applying the brighter fatter correction?"
243  )
244  datasetType = pexConfig.Field(
245  dtype=str,
246  doc="Dataset type for input data; users will typically leave this alone, "
247  "but camera-specific ISR tasks will override it",
248  default="raw",
249  )
250  fallbackFilterName = pexConfig.Field(dtype=str,
251  doc="Fallback default filter name for calibrations", optional=True)
252  doAttachTransmissionCurve = pexConfig.Field(
253  dtype=bool,
254  default=False,
255  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
256  )
257  doUseOpticsTransmission = pexConfig.Field(
258  dtype=bool,
259  default=True,
260  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
261  )
262  doUseFilterTransmission = pexConfig.Field(
263  dtype=bool,
264  default=True,
265  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
266  )
267  doUseSensorTransmission = pexConfig.Field(
268  dtype=bool,
269  default=True,
270  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
271  )
272  doUseAtmosphereTransmission = pexConfig.Field(
273  dtype=bool,
274  default=True,
275  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
276  )
277 
278 
279 
285 
286 
287 class IsrTask(pipeBase.CmdLineTask):
288  """!
289  \anchor IsrTask_
290 
291  \brief Apply common instrument signature correction algorithms to a raw frame.
292 
293  \section ip_isr_isr_Contents Contents
294 
295  - \ref ip_isr_isr_Purpose
296  - \ref ip_isr_isr_Initialize
297  - \ref ip_isr_isr_IO
298  - \ref ip_isr_isr_Config
299  - \ref ip_isr_isr_Debug
300 
301 
302  \section ip_isr_isr_Purpose Description
303 
304  The process for correcting imaging data is very similar from camera to camera.
305  This task provides a vanilla implementation of doing these corrections, including
306  the ability to turn certain corrections off if they are not needed.
307  The inputs to the primary method, run, are a raw exposure to be corrected and the
308  calibration data products. The raw input is a single chip sized mosaic of all amps
309  including overscans and other non-science pixels.
310  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
311  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
312  This task may not meet all needs and it is expected that it will be subclassed for
313  specific applications.
314 
315  \section ip_isr_isr_Initialize Task initialization
316 
317  \copydoc \_\_init\_\_
318 
319  \section ip_isr_isr_IO Inputs/Outputs to the run method
320 
321  \copydoc run
322 
323  \section ip_isr_isr_Config Configuration parameters
324 
325  See \ref IsrTaskConfig
326 
327  \section ip_isr_isr_Debug Debug variables
328 
329  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
330  flag \c --debug, \c -d to import \b debug.py from your \c PYTHONPATH; see <a
331  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
332  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
333 
334  The available variables in IsrTask are:
335  <DL>
336  <DT> \c display
337  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
338  <DL>
339  <DT> postISRCCD
340  <DD> display exposure after ISR has been applied
341  </DL>
342  </DL>
343 
344  For example, put something like
345  \code{.py}
346  import lsstDebug
347  def DebugInfo(name):
348  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
349  if name == "lsst.ip.isrFunctions.isrTask":
350  di.display = {'postISRCCD':2}
351  return di
352  lsstDebug.Info = DebugInfo
353  \endcode
354  into your debug.py file and run the commandline task with the \c --debug flag.
355 
356  <HR>
357  """
358  ConfigClass = IsrTaskConfig
359  _DefaultName = "isr"
360 
361  def __init__(self, *args, **kwargs):
362  '''!Constructor for IsrTask
363  @param[in] *args a list of positional arguments passed on to the Task constructor
364  @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor
365  Call the lsst.pipe.base.task.Task.__init__ method
366  Then setup the assembly and fringe correction subtasks
367  '''
368  pipeBase.Task.__init__(self, *args, **kwargs)
369  self.makeSubtask("assembleCcd")
370  self.makeSubtask("fringe")
371  self.makeSubtask("crosstalk")
372 
373  def readIsrData(self, dataRef, rawExposure):
374  """!Retrieve necessary frames for instrument signature removal
375  @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef
376  of the detector data to be processed
377  @param[in] rawExposure a reference raw exposure that will later be
378  corrected with the retrieved calibration data;
379  should not be modified in this method.
380  \return a pipeBase.Struct with fields containing kwargs expected by run()
381  - bias: exposure of bias frame
382  - dark: exposure of dark frame
383  - flat: exposure of flat field
384  - defects: list of detects
385  - fringeStruct: a pipeBase.Struct with field fringes containing
386  exposure of fringe frame or list of fringe exposure
387  """
388  ccd = rawExposure.getDetector()
389 
390  biasExposure = self.getIsrExposure(dataRef, "bias") if self.config.doBias else None
391  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
392  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
393  darkExposure = self.getIsrExposure(dataRef, "dark") if self.config.doDark else None
394  flatExposure = self.getIsrExposure(dataRef, "flat") if self.config.doFlat else None
395  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
396  defectList = dataRef.get("defects") if self.config.doDefect else None
397 
398  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
399  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
400  if self.config.doAssembleIsrExposures else None)
401  else:
402  fringeStruct = pipeBase.Struct(fringes=None)
403 
404  if self.config.doAttachTransmissionCurve:
405  opticsTransmission = (dataRef.get("transmission_optics")
406  if self.config.doUseOpticsTransmission else None)
407  filterTransmission = (dataRef.get("transmission_filter")
408  if self.config.doUseFilterTransmission else None)
409  sensorTransmission = (dataRef.get("transmission_sensor")
410  if self.config.doUseSensorTransmission else None)
411  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
412  if self.config.doUseAtmosphereTransmission else None)
413  else:
414  opticsTransmission = None
415  filterTransmission = None
416  sensorTransmission = None
417  atmosphereTransmission = None
418 
419  # Struct should include only kwargs to run()
420  return pipeBase.Struct(bias=biasExposure,
421  linearizer=linearizer,
422  dark=darkExposure,
423  flat=flatExposure,
424  defects=defectList,
425  fringes=fringeStruct,
426  bfKernel=brighterFatterKernel,
427  opticsTransmission=opticsTransmission,
428  filterTransmission=filterTransmission,
429  sensorTransmission=sensorTransmission,
430  atmosphereTransmission=atmosphereTransmission,
431  )
432 
433  @pipeBase.timeMethod
434  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
435  fringes=None, bfKernel=None,
436  opticsTransmission=None, filterTransmission=None,
437  sensorTransmission=None, atmosphereTransmission=None):
438  """!Perform instrument signature removal on an exposure
439 
440  Steps include:
441  - Detect saturation, apply overscan correction, bias, dark and flat
442  - Perform CCD assembly
443  - Interpolate over defects, saturated pixels and all NaNs
444 
445  @param[in] ccdExposure lsst.afw.image.exposure of detector data
446  @param[in] bias exposure of bias frame
447  @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
448  @param[in] dark exposure of dark frame
449  @param[in] flat exposure of flatfield
450  @param[in] defects list of detects
451  @param[in] fringes a pipeBase.Struct with field fringes containing
452  exposure of fringe frame or list of fringe exposure
453  @param[in] bfKernel kernel for brighter-fatter correction
454  @param[in] opticsTransmission a TransmissionCurve for the optics
455  @param[in] filterTransmission a TransmissionCurve for the filter
456  @param[in] sensorTransmission a TransmissionCurve for the sensor
457  @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere
458 
459  @return a pipeBase.Struct with field:
460  - exposure
461  """
462  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
463  if isinstance(ccdExposure, ButlerDataRef):
464  return self.runDataRef(ccdExposure)
465 
466  ccd = ccdExposure.getDetector()
467 
468  # Validate Input
469  if self.config.doBias and bias is None:
470  raise RuntimeError("Must supply a bias exposure if config.doBias True")
471  if self.doLinearize(ccd) and linearizer is None:
472  raise RuntimeError("Must supply a linearizer if config.doBias True")
473  if self.config.doDark and dark is None:
474  raise RuntimeError("Must supply a dark exposure if config.doDark True")
475  if self.config.doFlat and flat is None:
476  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
477  if self.config.doBrighterFatter and bfKernel is None:
478  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
479  if fringes is None:
480  fringes = pipeBase.Struct(fringes=None)
481  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
482  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
483  if self.config.doDefect and defects is None:
484  raise RuntimeError("Must supply defects if config.doDefect True")
485 
486  ccdExposure = self.convertIntToFloat(ccdExposure)
487 
488  if not ccd:
489  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
490  ccd = [FakeAmp(ccdExposure, self.config)]
491 
492  for amp in ccd:
493  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
494  if ccdExposure.getBBox().contains(amp.getBBox()):
495  self.saturationDetection(ccdExposure, amp)
496  self.suspectDetection(ccdExposure, amp)
497  self.overscanCorrection(ccdExposure, amp)
498 
499  if self.config.doAssembleCcd:
500  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
501  if self.config.expectWcs and not ccdExposure.getWcs():
502  self.log.warn("No WCS found in input exposure")
503 
504  if self.config.doBias:
505  self.biasCorrection(ccdExposure, bias)
506 
507  if self.doLinearize(ccd):
508  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
509 
510  if self.config.doCrosstalk:
511  self.crosstalk.run(ccdExposure)
512 
513  for amp in ccd:
514  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
515  if ccdExposure.getBBox().contains(amp.getBBox()):
516  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
517  self.updateVariance(ampExposure, amp)
518 
519  interpolationDone = False
520 
521  if self.config.doBrighterFatter:
522 
523  # We need to apply flats and darks before we can interpolate, and we
524  # need to interpolate before we do B-F, but we do B-F without the
525  # flats and darks applied so we can work in units of electrons or holes.
526  # This context manager applies and then removes the darks and flats.
527  with self.flatContext(ccdExposure, flat, dark):
528  if self.config.doDefect:
529  self.maskAndInterpDefect(ccdExposure, defects)
530  if self.config.doSaturationInterpolation:
531  self.saturationInterpolation(ccdExposure)
532  self.maskAndInterpNan(ccdExposure)
533  interpolationDone = True
534 
535  self.brighterFatterCorrection(ccdExposure, bfKernel,
536  self.config.brighterFatterMaxIter,
537  self.config.brighterFatterThreshold,
538  self.config.brighterFatterApplyGain,
539  )
540 
541  if self.config.doDark:
542  self.darkCorrection(ccdExposure, dark)
543 
544  if self.config.doFringe and not self.config.fringeAfterFlat:
545  self.fringe.run(ccdExposure, **fringes.getDict())
546 
547  if self.config.doFlat:
548  self.flatCorrection(ccdExposure, flat)
549 
550  if not interpolationDone:
551  if self.config.doDefect:
552  self.maskAndInterpDefect(ccdExposure, defects)
553  if self.config.doSaturationInterpolation:
554  self.saturationInterpolation(ccdExposure)
555  if not interpolationDone or self.config.doNanInterpAfterFlat:
556  self.maskAndInterpNan(ccdExposure)
557 
558  if self.config.doFringe and self.config.fringeAfterFlat:
559  self.fringe.run(ccdExposure, **fringes.getDict())
560 
561  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
562  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
563 
564  if self.config.doAttachTransmissionCurve:
565  self.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
566  filterTransmission=filterTransmission,
567  sensorTransmission=sensorTransmission,
568  atmosphereTransmission=atmosphereTransmission)
569 
570  frame = getDebugFrame(self._display, "postISRCCD")
571  if frame:
572  getDisplay(frame).mtv(ccdExposure)
573 
574  return pipeBase.Struct(
575  exposure=ccdExposure,
576  )
577 
578  @pipeBase.timeMethod
579  def runDataRef(self, sensorRef):
580  """!Perform instrument signature removal on a ButlerDataRef of a Sensor
581 
582  - Read in necessary detrending/isr/calibration data
583  - Process raw exposure in run()
584  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
585 
586  @param[in] sensorRef daf.persistence.butlerSubset.ButlerDataRef of the
587  detector data to be processed
588  @return a pipeBase.Struct with fields:
589  - exposure: the exposure after application of ISR
590  """
591  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
592  ccdExposure = sensorRef.get('raw')
593  isrData = self.readIsrData(sensorRef, ccdExposure)
594 
595  result = self.run(ccdExposure, **isrData.getDict())
596 
597  if self.config.doWrite:
598  sensorRef.put(result.exposure, "postISRCCD")
599 
600  return result
601 
602  def convertIntToFloat(self, exposure):
603  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
604  """
605  if isinstance(exposure, afwImage.ExposureF):
606  # Nothing to be done
607  return exposure
608  if not hasattr(exposure, "convertF"):
609  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
610 
611  newexposure = exposure.convertF()
612  maskedImage = newexposure.getMaskedImage()
613  varArray = maskedImage.getVariance().getArray()
614  varArray[:, :] = 1
615  maskArray = maskedImage.getMask().getArray()
616  maskArray[:, :] = 0
617  return newexposure
618 
619  def biasCorrection(self, exposure, biasExposure):
620  """!Apply bias correction in place
621 
622  @param[in,out] exposure exposure to process
623  @param[in] biasExposure bias exposure of same size as exposure
624  """
625  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
626 
627  def darkCorrection(self, exposure, darkExposure, invert=False):
628  """!Apply dark correction in place
629 
630  @param[in,out] exposure exposure to process
631  @param[in] darkExposure dark exposure of same size as exposure
632  @param[in] invert if True, remove the dark from an already-corrected image
633  """
634  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
635  if math.isnan(expScale):
636  raise RuntimeError("Exposure darktime is NAN")
637  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
638  if math.isnan(darkScale):
639  raise RuntimeError("Dark calib darktime is NAN")
640  isrFunctions.darkCorrection(
641  maskedImage=exposure.getMaskedImage(),
642  darkMaskedImage=darkExposure.getMaskedImage(),
643  expScale=expScale,
644  darkScale=darkScale,
645  invert=invert
646  )
647 
648  def doLinearize(self, detector):
649  """!Is linearization wanted for this detector?
650 
651  Checks config.doLinearize and the linearity type of the first amplifier.
652 
653  @param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
654  """
655  return self.config.doLinearize and \
656  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
657 
658  def updateVariance(self, ampExposure, amp):
659  """!Set the variance plane based on the image plane, plus amplifier gain and read noise
660 
661  @param[in,out] ampExposure exposure to process
662  @param[in] amp amplifier detector information
663  """
664  if not math.isnan(amp.getGain()):
665  isrFunctions.updateVariance(
666  maskedImage=ampExposure.getMaskedImage(),
667  gain=amp.getGain(),
668  readNoise=amp.getReadNoise(),
669  )
670 
671  def flatCorrection(self, exposure, flatExposure, invert=False):
672  """!Apply flat correction in place
673 
674  @param[in,out] exposure exposure to process
675  @param[in] flatExposure flatfield exposure same size as exposure
676  @param[in] invert if True, unflatten an already-flattened image instead.
677  """
678  isrFunctions.flatCorrection(
679  maskedImage=exposure.getMaskedImage(),
680  flatMaskedImage=flatExposure.getMaskedImage(),
681  scalingType=self.config.flatScalingType,
682  userScale=self.config.flatUserScale,
683  invert=invert
684  )
685 
686  def getIsrExposure(self, dataRef, datasetType, immediate=True):
687  """!Retrieve a calibration dataset for removing instrument signature
688 
689  @param[in] dataRef data reference for exposure
690  @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
691  @param[in] immediate if True, disable butler proxies to enable error
692  handling within this routine
693  \return exposure
694  """
695  try:
696  exp = dataRef.get(datasetType, immediate=immediate)
697  except Exception as exc1:
698  if not self.config.fallbackFilterName:
699  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
700  try:
701  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
702  except Exception as exc2:
703  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
704  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
705  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
706 
707  if self.config.doAssembleIsrExposures:
708  exp = self.assembleCcd.assembleCcd(exp)
709  return exp
710 
711  def saturationDetection(self, exposure, amp):
712  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
713 
714  @param[in,out] exposure exposure to process; only the amp DataSec is processed
715  @param[in] amp amplifier device data
716  """
717  if not math.isnan(amp.getSaturation()):
718  maskedImage = exposure.getMaskedImage()
719  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
720  isrFunctions.makeThresholdMask(
721  maskedImage=dataView,
722  threshold=amp.getSaturation(),
723  growFootprints=0,
724  maskName=self.config.saturatedMaskName,
725  )
726 
727  def saturationInterpolation(self, ccdExposure):
728  """!Interpolate over saturated pixels, in place
729 
730  @param[in,out] ccdExposure exposure to process
731 
732  \warning:
733  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
734  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
735  """
736  isrFunctions.interpolateFromMask(
737  maskedImage=ccdExposure.getMaskedImage(),
738  fwhm=self.config.fwhm,
739  growFootprints=self.config.growSaturationFootprintSize,
740  maskName=self.config.saturatedMaskName,
741  )
742 
743  def suspectDetection(self, exposure, amp):
744  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
745 
746  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
747  This is intended to indicate pixels that may be affected by unknown systematics;
748  for example if non-linearity corrections above a certain level are unstable
749  then that would be a useful value for suspectLevel. A value of `nan` indicates
750  that no such level exists and no pixels are to be masked as suspicious.
751 
752  @param[in,out] exposure exposure to process; only the amp DataSec is processed
753  @param[in] amp amplifier device data
754  """
755  suspectLevel = amp.getSuspectLevel()
756  if math.isnan(suspectLevel):
757  return
758 
759  maskedImage = exposure.getMaskedImage()
760  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
761  isrFunctions.makeThresholdMask(
762  maskedImage=dataView,
763  threshold=suspectLevel,
764  growFootprints=0,
765  maskName=self.config.suspectMaskName,
766  )
767 
768  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
769  """!Mask defects using mask plane "BAD" and interpolate over them, in place
770 
771  @param[in,out] ccdExposure exposure to process
772  @param[in] defectBaseList a list of defects to mask and interpolate
773 
774  \warning: call this after CCD assembly, since defects may cross amplifier boundaries
775  """
776  maskedImage = ccdExposure.getMaskedImage()
777  defectList = []
778  for d in defectBaseList:
779  bbox = d.getBBox()
780  nd = measAlg.Defect(bbox)
781  defectList.append(nd)
782  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
783  isrFunctions.interpolateDefectList(
784  maskedImage=maskedImage,
785  defectList=defectList,
786  fwhm=self.config.fwhm,
787  )
788 
789  def maskAndInterpNan(self, exposure):
790  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
791 
792  We mask and interpolate over all NaNs, including those
793  that are masked with other bits (because those may or may
794  not be interpolated over later, and we want to remove all
795  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
796  is used to preserve the historical name.
797 
798  @param[in,out] exposure exposure to process
799  """
800  maskedImage = exposure.getMaskedImage()
801 
802  # Find and mask NaNs
803  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
804  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
805  numNans = maskNans(maskedImage, maskVal)
806  self.metadata.set("NUMNANS", numNans)
807 
808  # Interpolate over these previously-unmasked NaNs
809  if numNans > 0:
810  self.log.warn("There were %i unmasked NaNs", numNans)
811  nanDefectList = isrFunctions.getDefectListFromMask(
812  maskedImage=maskedImage,
813  maskName='UNMASKEDNAN',
814  )
815  isrFunctions.interpolateDefectList(
816  maskedImage=exposure.getMaskedImage(),
817  defectList=nanDefectList,
818  fwhm=self.config.fwhm,
819  )
820 
821  def overscanCorrection(self, exposure, amp):
822  """!Apply overscan correction, in place
823 
824  @param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
825  @param[in] amp amplifier device data
826  """
827  if not amp.getHasRawInfo():
828  raise RuntimeError("This method must be executed on an amp with raw information.")
829 
830  if amp.getRawHorizontalOverscanBBox().isEmpty():
831  self.log.info("No Overscan region. Not performing Overscan Correction.")
832  return None
833 
834  maskedImage = exposure.getMaskedImage()
835  dataView = maskedImage.Factory(maskedImage, amp.getRawDataBBox())
836 
837  expImage = exposure.getMaskedImage().getImage()
838  overscanImage = expImage.Factory(expImage, amp.getRawHorizontalOverscanBBox())
839 
840  isrFunctions.overscanCorrection(
841  ampMaskedImage=dataView,
842  overscanImage=overscanImage,
843  fitType=self.config.overscanFitType,
844  order=self.config.overscanOrder,
845  collapseRej=self.config.overscanRej,
846  )
847 
848  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
849  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
850 
851  @param[in,out] ccdExposure exposure to process
852  @param[in] fpPolygon Polygon in focal plane coordinates
853  """
854  # Get ccd corners in focal plane coordinates
855  ccd = ccdExposure.getDetector()
856  fpCorners = ccd.getCorners(FOCAL_PLANE)
857  ccdPolygon = Polygon(fpCorners)
858 
859  # Get intersection of ccd corners with fpPolygon
860  intersect = ccdPolygon.intersectionSingle(fpPolygon)
861 
862  # Transform back to pixel positions and build new polygon
863  ccdPoints = [ccd.transform(ccd.makeCameraPoint(x, FOCAL_PLANE), PIXELS).getPoint() for x in intersect]
864  validPolygon = Polygon(ccdPoints)
865  ccdExposure.getInfo().setValidPolygon(validPolygon)
866 
867  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
868  """Apply brighter fatter correction in place for the image
869 
870  This correction takes a kernel that has been derived from flat field images to
871  redistribute the charge. The gradient of the kernel is the deflection
872  field due to the accumulated charge.
873 
874  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
875  using the following equation:
876 
877  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
878 
879  To evaluate the derivative term we expand it as follows:
880 
881  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
882 
883  Because we use the measured counts instead of the incident counts we apply the correction
884  iteratively to reconstruct the original counts and the correction. We stop iterating when the
885  summed difference between the current corrected image and the one from the previous iteration
886  is below the threshold. We do not require convergence because the number of iterations is
887  too large a computational cost. How we define the threshold still needs to be evaluated, the
888  current default was shown to work reasonably well on a small set of images. For more information
889  on the method see DocuShare Document-19407.
890 
891  The edges as defined by the kernel are not corrected because they have spurious values
892  due to the convolution.
893  """
894  self.log.info("Applying brighter fatter correction")
895 
896  image = exposure.getMaskedImage().getImage()
897 
898  # The image needs to be units of electrons/holes
899  with self.gainContext(exposure, image, applyGain):
900 
901  kLx = numpy.shape(kernel)[0]
902  kLy = numpy.shape(kernel)[1]
903  kernelImage = afwImage.ImageD(kLx, kLy)
904  kernelImage.getArray()[:, :] = kernel
905  tempImage = image.clone()
906 
907  nanIndex = numpy.isnan(tempImage.getArray())
908  tempImage.getArray()[nanIndex] = 0.
909 
910  outImage = afwImage.ImageF(image.getDimensions())
911  corr = numpy.zeros_like(image.getArray())
912  prev_image = numpy.zeros_like(image.getArray())
913  convCntrl = afwMath.ConvolutionControl(False, True, 1)
914  fixedKernel = afwMath.FixedKernel(kernelImage)
915 
916  # Define boundary by convolution region. The region that the correction will be
917  # calculated for is one fewer in each dimension because of the second derivative terms.
918  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
919  startX = kLx//2
920  endX = -kLx//2
921  startY = kLy//2
922  endY = -kLy//2
923 
924  for iteration in range(maxIter):
925 
926  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
927  tmpArray = tempImage.getArray()
928  outArray = outImage.getArray()
929 
930  # First derivative term
931  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
932  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
933  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
934 
935  # Second derivative term
936  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
937  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
938  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
939 
940  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
941 
942  tmpArray[:, :] = image.getArray()[:, :]
943  tmpArray[nanIndex] = 0.
944  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
945 
946  if iteration > 0:
947  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
948 
949  if diff < threshold:
950  break
951  prev_image[:, :] = tmpArray[:, :]
952 
953  if iteration == maxIter - 1:
954  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
955 
956  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
957  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
958  corr[startY + 1:endY - 1, startX + 1:endX - 1]
959 
960  def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None,
961  sensorTransmission=None, atmosphereTransmission=None):
962  """Attach a TransmissionCurve to an Exposure, given separate curves for
963  different components.
964 
965  Parameters
966  ----------
967  exposure : `lsst.afw.image.Exposure`
968  Exposure object to modify by attaching the product of all given
969  ``TransmissionCurves`` in post-assembly trimmed detector
970  coordinates. Must have a valid ``Detector`` attached that matches
971  the detector associated with sensorTransmission.
972  opticsTransmission : `lsst.afw.image.TransmissionCurve`
973  A ``TransmissionCurve`` that represents the throughput of the
974  optics, to be evaluated in focal-plane coordinates.
975  filterTransmission : `lsst.afw.image.TransmissionCurve`
976  A ``TransmissionCurve`` that represents the throughput of the
977  filter itself, to be evaluated in focal-plane coordinates.
978  sensorTransmission : `lsst.afw.image.TransmissionCurve`
979  A ``TransmissionCurve`` that represents the throughput of the
980  sensor itself, to be evaluated in post-assembly trimmed detector
981  coordinates.
982  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
983  A ``TransmissionCurve`` that represents the throughput of the
984  atmosphere, assumed to be spatially constant.
985 
986  All ``TransmissionCurve`` arguments are optional; if none are provided,
987  the attached ``TransmissionCurve`` will have unit transmission
988  everywhere.
989 
990  Returns
991  -------
992  combined : ``lsst.afw.image.TransmissionCurve``
993  The TransmissionCurve attached to the exposure.
994  """
995  return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
996  filterTransmission=filterTransmission,
997  sensorTransmission=sensorTransmission,
998  atmosphereTransmission=atmosphereTransmission)
999 
1000  @contextmanager
1001  def gainContext(self, exp, image, apply):
1002  """Context manager that applies and removes gain
1003  """
1004  if apply:
1005  ccd = exp.getDetector()
1006  for amp in ccd:
1007  sim = image.Factory(image, amp.getBBox())
1008  sim *= amp.getGain()
1009 
1010  try:
1011  yield exp
1012  finally:
1013  if apply:
1014  ccd = exp.getDetector()
1015  for amp in ccd:
1016  sim = image.Factory(image, amp.getBBox())
1017  sim /= amp.getGain()
1018 
1019 
1020  @contextmanager
1021  def flatContext(self, exp, flat, dark=None):
1022  """Context manager that applies and removes flats and darks,
1023  if the task is configured to apply them.
1024  """
1025  if self.config.doDark and dark is not None:
1026  self.darkCorrection(exp, dark)
1027  if self.config.doFlat:
1028  self.flatCorrection(exp, flat)
1029  try:
1030  yield exp
1031  finally:
1032  if self.config.doFlat:
1033  self.flatCorrection(exp, flat, invert=True)
1034  if self.config.doDark and dark is not None:
1035  self.darkCorrection(exp, dark, invert=True)
1036 
1037 
1038 class FakeAmp(object):
1039  """A Detector-like object that supports returning gain and saturation level"""
1040 
1041  def __init__(self, exposure, config):
1042  self._bbox = exposure.getBBox(afwImage.LOCAL)
1043  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
1044  self._gain = config.gain
1045  self._readNoise = config.readNoise
1046  self._saturation = config.saturation
1047 
1048  def getBBox(self):
1049  return self._bbox
1050 
1051  def getRawBBox(self):
1052  return self._bbox
1053 
1054  def getHasRawInfo(self):
1055  return True # but see getRawHorizontalOverscanBBox()
1056 
1058  return self._RawHorizontalOverscanBBox
1059 
1060  def getGain(self):
1061  return self._gain
1062 
1063  def getReadNoise(self):
1064  return self._readNoise
1065 
1066  def getSaturation(self):
1067  return self._saturation
1068 
1069  def getSuspectLevel(self):
1070  return float("NaN")
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
Definition: isrTask.py:867
def runDataRef(self, sensorRef)
Perform instrument signature removal on a ButlerDataRef of a Sensor.
Definition: isrTask.py:579
def gainContext(self, exp, image, apply)
Definition: isrTask.py:1001
def __init__(self, args, kwargs)
Constructor for IsrTask.
Definition: isrTask.py:361
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:373
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Definition: isrTask.py:961
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:789
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:727
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:287
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:1057
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:437
def convertIntToFloat(self, exposure)
Definition: isrTask.py:602
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:671
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:686
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:627
def doLinearize(self, detector)
Is linearization wanted for this detector?
Definition: isrTask.py:648
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:848
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
Definition: isrTask.py:619
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:1021
def updateVariance(self, ampExposure, amp)
Set the variance plane based on the image plane, plus amplifier gain and read noise.
Definition: isrTask.py:658
def overscanCorrection(self, exposure, amp)
Apply overscan correction, in place.
Definition: isrTask.py:821
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:743
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:768
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:711
def __init__(self, exposure, config)
Definition: isrTask.py:1041