lsst.ip.isr  16.0-14-g6c7ed55+9
isrTask.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.meas.algorithms as measAlg
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.afw.math as afwMath
31 from lsst.daf.persistence import ButlerDataRef
32 from lsstDebug import getDebugFrame
33 from lsst.afw.display import getDisplay
34 from . import isrFunctions
35 from .assembleCcdTask import AssembleCcdTask
36 from .fringe import FringeTask
37 from lsst.afw.geom import Polygon
38 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
39 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, FIELD_ANGLE, NullLinearityType
40 from contextlib import contextmanager
41 from .isr import maskNans
42 from .crosstalk import CrosstalkTask
43 
44 
45 class IsrTaskConfig(pexConfig.Config):
46  doBias = pexConfig.Field(
47  dtype=bool,
48  doc="Apply bias frame correction?",
49  default=True,
50  )
51  doDark = pexConfig.Field(
52  dtype=bool,
53  doc="Apply dark frame correction?",
54  default=True,
55  )
56  doFlat = pexConfig.Field(
57  dtype=bool,
58  doc="Apply flat field correction?",
59  default=True,
60  )
61  doFringe = pexConfig.Field(
62  dtype=bool,
63  doc="Apply fringe correction?",
64  default=True,
65  )
66  doDefect = pexConfig.Field(
67  dtype=bool,
68  doc="Apply correction for CCD defects, e.g. hot pixels?",
69  default=True,
70  )
71  doAddDistortionModel = pexConfig.Field(
72  dtype=bool,
73  doc="Apply a distortion model based on camera geometry to the WCS?",
74  default=True,
75  )
76  doWrite = pexConfig.Field(
77  dtype=bool,
78  doc="Persist postISRCCD?",
79  default=True,
80  )
81  biasDataProductName = pexConfig.Field(
82  dtype=str,
83  doc="Name of the bias data product",
84  default="bias",
85  )
86  darkDataProductName = pexConfig.Field(
87  dtype=str,
88  doc="Name of the dark data product",
89  default="dark",
90  )
91  flatDataProductName = pexConfig.Field(
92  dtype=str,
93  doc="Name of the flat data product",
94  default="flat",
95  )
96  assembleCcd = pexConfig.ConfigurableField(
97  target=AssembleCcdTask,
98  doc="CCD assembly task",
99  )
100  gain = pexConfig.Field(
101  dtype=float,
102  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
103  default=float("NaN"),
104  )
105  readNoise = pexConfig.Field(
106  dtype=float,
107  doc="The read noise to use if no Detector is present in the Exposure",
108  default=0.0,
109  )
110  saturation = pexConfig.Field(
111  dtype=float,
112  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
113  default=float("NaN"),
114  )
115  fringeAfterFlat = pexConfig.Field(
116  dtype=bool,
117  doc="Do fringe subtraction after flat-fielding?",
118  default=True,
119  )
120  fringe = pexConfig.ConfigurableField(
121  target=FringeTask,
122  doc="Fringe subtraction task",
123  )
124  fwhm = pexConfig.Field(
125  dtype=float,
126  doc="FWHM of PSF (arcsec)",
127  default=1.0,
128  )
129  saturatedMaskName = pexConfig.Field(
130  dtype=str,
131  doc="Name of mask plane to use in saturation detection and interpolation",
132  default="SAT",
133  )
134  suspectMaskName = pexConfig.Field(
135  dtype=str,
136  doc="Name of mask plane to use for suspect pixels",
137  default="SUSPECT",
138  )
139  flatScalingType = pexConfig.ChoiceField(
140  dtype=str,
141  doc="The method for scaling the flat on the fly.",
142  default='USER',
143  allowed={
144  "USER": "Scale by flatUserScale",
145  "MEAN": "Scale by the inverse of the mean",
146  "MEDIAN": "Scale by the inverse of the median",
147  },
148  )
149  flatUserScale = pexConfig.Field(
150  dtype=float,
151  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
152  default=1.0,
153  )
154  overscanFitType = pexConfig.ChoiceField(
155  dtype=str,
156  doc="The method for fitting the overscan bias level.",
157  default='MEDIAN',
158  allowed={
159  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
160  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
161  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
162  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
163  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
164  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
165  "MEAN": "Correct using the mean of the overscan region",
166  "MEANCLIP": "Correct using a clipped mean of the overscan region",
167  "MEDIAN": "Correct using the median of the overscan region",
168  },
169  )
170  overscanOrder = pexConfig.Field(
171  dtype=int,
172  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
173  "or number of spline knots if overscan fit type is a spline."),
174  default=1,
175  )
176  overscanNumSigmaClip = pexConfig.Field(
177  dtype=float,
178  doc="Rejection threshold (sigma) for collapsing overscan before fit",
179  default=3.0,
180  )
181 
182  overscanNumLeadingColumnsToSkip = pexConfig.Field(
183  dtype=int,
184  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
185  default=0,
186  )
187  overscanNumTrailingColumnsToSkip = pexConfig.Field(
188  dtype=int,
189  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
190  default=0,
191  )
192  growSaturationFootprintSize = pexConfig.Field(
193  dtype=int,
194  doc="Number of pixels by which to grow the saturation footprints",
195  default=1,
196  )
197  doSaturationInterpolation = pexConfig.Field(
198  dtype=bool,
199  doc="Perform interpolation over pixels masked as saturated?",
200  default=True,
201  )
202  doNanInterpAfterFlat = pexConfig.Field(
203  dtype=bool,
204  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
205  "also have to interpolate them before flat-fielding."),
206  default=False,
207  )
208  fluxMag0T1 = pexConfig.Field(
209  dtype=float,
210  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
211  default=1e10,
212  )
213  keysToRemoveFromAssembledCcd = pexConfig.ListField(
214  dtype=str,
215  doc="fields to remove from the metadata of the assembled ccd.",
216  default=[],
217  )
218  doAssembleIsrExposures = pexConfig.Field(
219  dtype=bool,
220  default=False,
221  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
222  )
223  doAssembleCcd = pexConfig.Field(
224  dtype=bool,
225  default=True,
226  doc="Assemble amp-level exposures into a ccd-level exposure?"
227  )
228  expectWcs = pexConfig.Field(
229  dtype=bool,
230  default=True,
231  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)"
232  )
233  doLinearize = pexConfig.Field(
234  dtype=bool,
235  doc="Correct for nonlinearity of the detector's response?",
236  default=True,
237  )
238  doCrosstalk = pexConfig.Field(
239  dtype=bool,
240  doc="Apply intra-CCD crosstalk correction?",
241  default=False,
242  )
243  crosstalk = pexConfig.ConfigurableField(
244  target=CrosstalkTask,
245  doc="Intra-CCD crosstalk correction",
246  )
247  doBrighterFatter = pexConfig.Field(
248  dtype=bool,
249  default=False,
250  doc="Apply the brighter fatter correction"
251  )
252  brighterFatterLevel = pexConfig.ChoiceField(
253  doc="The level at which to correct for brighter-fatter",
254  dtype=str, default="DETECTOR",
255  allowed={
256  "AMP": "Every amplifier treated separately",
257  "DETECTOR": "One kernel per detector",
258  }
259  )
260  brighterFatterKernelFile = pexConfig.Field(
261  dtype=str,
262  default='',
263  doc="Kernel file used for the brighter fatter correction"
264  )
265  brighterFatterMaxIter = pexConfig.Field(
266  dtype=int,
267  default=10,
268  doc="Maximum number of iterations for the brighter fatter correction"
269  )
270  brighterFatterThreshold = pexConfig.Field(
271  dtype=float,
272  default=1000,
273  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
274  " absolute value of the difference between the current corrected image and the one"
275  " from the previous iteration summed over all the pixels."
276  )
277  brighterFatterApplyGain = pexConfig.Field(
278  dtype=bool,
279  default=True,
280  doc="Should the gain be applied when applying the brighter fatter correction?"
281  )
282  datasetType = pexConfig.Field(
283  dtype=str,
284  doc="Dataset type for input data; users will typically leave this alone, "
285  "but camera-specific ISR tasks will override it",
286  default="raw",
287  )
288  fallbackFilterName = pexConfig.Field(dtype=str,
289  doc="Fallback default filter name for calibrations", optional=True)
290  doAttachTransmissionCurve = pexConfig.Field(
291  dtype=bool,
292  default=False,
293  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
294  )
295  doUseOpticsTransmission = pexConfig.Field(
296  dtype=bool,
297  default=True,
298  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
299  )
300  doUseFilterTransmission = pexConfig.Field(
301  dtype=bool,
302  default=True,
303  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
304  )
305  doUseSensorTransmission = pexConfig.Field(
306  dtype=bool,
307  default=True,
308  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
309  )
310  doUseAtmosphereTransmission = pexConfig.Field(
311  dtype=bool,
312  default=True,
313  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
314  )
315  doEmpiricalReadNoise = pexConfig.Field(
316  dtype=bool,
317  default=False,
318  doc="Calculate empirical read noise instead of value from AmpInfo data?"
319  )
320 
321 
327 
328 
329 class IsrTask(pipeBase.CmdLineTask):
330  r"""!
331  @anchor IsrTask_
332 
333  @brief Apply common instrument signature correction algorithms to a raw frame.
334 
335  @section ip_isr_isr_Contents Contents
336 
337  - @ref ip_isr_isr_Purpose
338  - @ref ip_isr_isr_Initialize
339  - @ref ip_isr_isr_IO
340  - @ref ip_isr_isr_Config
341  - @ref ip_isr_isr_Debug
342 
343 
344  @section ip_isr_isr_Purpose Description
345 
346  The process for correcting imaging data is very similar from camera to camera.
347  This task provides a vanilla implementation of doing these corrections, including
348  the ability to turn certain corrections off if they are not needed.
349  The inputs to the primary method, run, are a raw exposure to be corrected and the
350  calibration data products. The raw input is a single chip sized mosaic of all amps
351  including overscans and other non-science pixels.
352  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
353  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
354  This task may not meet all needs and it is expected that it will be subclassed for
355  specific applications.
356 
357  @section ip_isr_isr_Initialize Task initialization
358 
359  @copydoc \_\_init\_\_
360 
361  @section ip_isr_isr_IO Inputs/Outputs to the run method
362 
363  @copydoc run
364 
365  @section ip_isr_isr_Config Configuration parameters
366 
367  See @ref IsrTaskConfig
368 
369  @section ip_isr_isr_Debug Debug variables
370 
371  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
372  flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a
373  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
374  Using lsstDebug to control debugging output</a> for more about @b debug.py files.
375 
376  The available variables in IsrTask are:
377  <DL>
378  <DT> @c display
379  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
380  <DL>
381  <DT> postISRCCD
382  <DD> display exposure after ISR has been applied
383  </DL>
384  </DL>
385 
386  For example, put something like
387  @code{.py}
388  import lsstDebug
389  def DebugInfo(name):
390  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
391  if name == "lsst.ip.isr.isrTask":
392  di.display = {'postISRCCD':2}
393  return di
394  lsstDebug.Info = DebugInfo
395  @endcode
396  into your debug.py file and run the commandline task with the @c --debug flag.
397 
398  <HR>
399  """
400  ConfigClass = IsrTaskConfig
401  _DefaultName = "isr"
402 
403  def __init__(self, *args, **kwargs):
404  '''!Constructor for IsrTask
405  @param[in] *args a list of positional arguments passed on to the Task constructor
406  @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor
407  Call the lsst.pipe.base.task.Task.__init__ method
408  Then setup the assembly and fringe correction subtasks
409  '''
410  pipeBase.Task.__init__(self, *args, **kwargs)
411  self.makeSubtask("assembleCcd")
412  self.makeSubtask("fringe")
413  self.makeSubtask("crosstalk")
414 
415  def readIsrData(self, dataRef, rawExposure):
416  """!Retrieve necessary frames for instrument signature removal
417  @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef
418  of the detector data to be processed
419  @param[in] rawExposure a reference raw exposure that will later be
420  corrected with the retrieved calibration data;
421  should not be modified in this method.
422  @return a pipeBase.Struct with fields containing kwargs expected by run()
423  - bias: exposure of bias frame
424  - dark: exposure of dark frame
425  - flat: exposure of flat field
426  - defects: list of detects
427  - fringeStruct: a pipeBase.Struct with field fringes containing
428  exposure of fringe frame or list of fringe exposure
429  """
430  ccd = rawExposure.getDetector()
431 
432  biasExposure = self.getIsrExposure(dataRef, self.config.biasDataProductName) \
433  if self.config.doBias else None
434  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
435  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
436  darkExposure = self.getIsrExposure(dataRef, self.config.darkDataProductName) \
437  if self.config.doDark else None
438  flatExposure = self.getIsrExposure(dataRef, self.config.flatDataProductName) \
439  if self.config.doFlat else None
440  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
441  defectList = dataRef.get("defects") if self.config.doDefect else None
442 
443  if self.config.doCrosstalk:
444  crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
445  else:
446  crosstalkSources = None
447 
448  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
449  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
450  if self.config.doAssembleIsrExposures else None)
451  else:
452  fringeStruct = pipeBase.Struct(fringes=None)
453 
454  if self.config.doAttachTransmissionCurve:
455  opticsTransmission = (dataRef.get("transmission_optics")
456  if self.config.doUseOpticsTransmission else None)
457  filterTransmission = (dataRef.get("transmission_filter")
458  if self.config.doUseFilterTransmission else None)
459  sensorTransmission = (dataRef.get("transmission_sensor")
460  if self.config.doUseSensorTransmission else None)
461  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
462  if self.config.doUseAtmosphereTransmission else None)
463  else:
464  opticsTransmission = None
465  filterTransmission = None
466  sensorTransmission = None
467  atmosphereTransmission = None
468 
469  # Struct should include only kwargs to run()
470  return pipeBase.Struct(bias=biasExposure,
471  linearizer=linearizer,
472  dark=darkExposure,
473  flat=flatExposure,
474  defects=defectList,
475  fringes=fringeStruct,
476  bfKernel=brighterFatterKernel,
477  opticsTransmission=opticsTransmission,
478  filterTransmission=filterTransmission,
479  sensorTransmission=sensorTransmission,
480  atmosphereTransmission=atmosphereTransmission,
481  crosstalkSources=crosstalkSources,
482  )
483 
484  @pipeBase.timeMethod
485  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
486  fringes=None, bfKernel=None, camera=None,
487  opticsTransmission=None, filterTransmission=None,
488  sensorTransmission=None, atmosphereTransmission=None,
489  crosstalkSources=None):
490  """!Perform instrument signature removal on an exposure
491 
492  Steps include:
493  - Detect saturation, apply overscan correction, bias, dark and flat
494  - Perform CCD assembly
495  - Interpolate over defects, saturated pixels and all NaNs
496 
497  @param[in] ccdExposure lsst.afw.image.exposure of detector data
498  @param[in] bias exposure of bias frame
499  @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
500  @param[in] dark exposure of dark frame
501  @param[in] flat exposure of flatfield
502  @param[in] defects list of detects
503  @param[in] fringes a pipeBase.Struct with field fringes containing
504  exposure of fringe frame or list of fringe exposure
505  @param[in] bfKernel kernel for brighter-fatter correction, an
506  lsst.cp.pipe.makeBrighterFatterKernel.BrighterFatterKernel object
507  @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera;
508  used by addDistortionModel
509  @param[in] opticsTransmission a TransmissionCurve for the optics
510  @param[in] filterTransmission a TransmissionCurve for the filter
511  @param[in] sensorTransmission a TransmissionCurve for the sensor
512  @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere
513  @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk
514 
515  @return a pipeBase.Struct with field:
516  - exposure
517  """
518  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
519  if isinstance(ccdExposure, ButlerDataRef):
520  return self.runDataRef(ccdExposure)
521 
522  ccd = ccdExposure.getDetector()
523 
524  # Validate Input
525  if self.config.doBias and bias is None:
526  raise RuntimeError("Must supply a bias exposure if config.doBias True")
527  if self.doLinearize(ccd) and linearizer is None:
528  raise RuntimeError("Must supply a linearizer if config.doLinearize True")
529  if self.config.doDark and dark is None:
530  raise RuntimeError("Must supply a dark exposure if config.doDark True")
531  if self.config.doFlat and flat is None:
532  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
533  if self.config.doBrighterFatter and bfKernel is None:
534  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
535  if fringes is None:
536  fringes = pipeBase.Struct(fringes=None)
537  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
538  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
539  if self.config.doDefect and defects is None:
540  raise RuntimeError("Must supply defects if config.doDefect True")
541  if self.config.doAddDistortionModel and camera is None:
542  raise RuntimeError("Must supply camera if config.doAddDistortionModel True")
543 
544  ccdExposure = self.convertIntToFloat(ccdExposure)
545 
546  if not ccd:
547  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
548  ccd = [FakeAmp(ccdExposure, self.config)]
549 
550  overscans = []
551  for amp in ccd:
552  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
553  if ccdExposure.getBBox().contains(amp.getBBox()):
554  self.saturationDetection(ccdExposure, amp)
555  self.suspectDetection(ccdExposure, amp)
556  overscanResults = self.overscanCorrection(ccdExposure, amp)
557  overscans.append(overscanResults.overscanImage if overscanResults is not None else None)
558  else:
559  overscans.append(None)
560 
561  if self.config.doCrosstalk:
562  self.crosstalk.run(ccdExposure, crosstalkSources)
563 
564  if self.config.doAssembleCcd:
565  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
566  if self.config.expectWcs and not ccdExposure.getWcs():
567  self.log.warn("No WCS found in input exposure")
568 
569  if self.config.doBias:
570  self.biasCorrection(ccdExposure, bias)
571 
572  if self.doLinearize(ccd):
573  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
574 
575  assert len(ccd) == len(overscans)
576  for amp, overscanImage in zip(ccd, overscans):
577  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
578  if ccdExposure.getBBox().contains(amp.getBBox()):
579  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
580  self.updateVariance(ampExposure, amp, overscanImage)
581 
582  interpolationDone = False
583 
584  if self.config.doBrighterFatter:
585 
586  # We need to apply flats and darks before we can interpolate, and we
587  # need to interpolate before we do B-F, but we do B-F without the
588  # flats and darks applied so we can work in units of electrons or holes.
589  # This context manager applies and then removes the darks and flats.
590  with self.flatContext(ccdExposure, flat, dark):
591  if self.config.doDefect:
592  self.maskAndInterpDefect(ccdExposure, defects)
593  if self.config.doSaturationInterpolation:
594  self.saturationInterpolation(ccdExposure)
595  self.maskAndInterpNan(ccdExposure)
596  interpolationDone = True
597 
598  if self.config.brighterFatterLevel == 'DETECTOR':
599  kernelElement = bfKernel.kernel[ccdExposure.getDetector().getId()]
600  else:
601  # TODO: DM-15631 for implementing this
602  raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented")
603  self.brighterFatterCorrection(ccdExposure, kernelElement,
604  self.config.brighterFatterMaxIter,
605  self.config.brighterFatterThreshold,
606  self.config.brighterFatterApplyGain,
607  )
608 
609  if self.config.doDark:
610  self.darkCorrection(ccdExposure, dark)
611 
612  if self.config.doFringe and not self.config.fringeAfterFlat:
613  self.fringe.run(ccdExposure, **fringes.getDict())
614 
615  if self.config.doFlat:
616  self.flatCorrection(ccdExposure, flat)
617 
618  if not interpolationDone:
619  if self.config.doDefect:
620  self.maskAndInterpDefect(ccdExposure, defects)
621  if self.config.doSaturationInterpolation:
622  self.saturationInterpolation(ccdExposure)
623  if not interpolationDone or self.config.doNanInterpAfterFlat:
624  self.maskAndInterpNan(ccdExposure)
625 
626  if self.config.doFringe and self.config.fringeAfterFlat:
627  self.fringe.run(ccdExposure, **fringes.getDict())
628 
629  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
630  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
631 
632  if self.config.doAddDistortionModel:
633  self.addDistortionModel(exposure=ccdExposure, camera=camera)
634 
635  if self.config.doAttachTransmissionCurve:
636  self.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
637  filterTransmission=filterTransmission,
638  sensorTransmission=sensorTransmission,
639  atmosphereTransmission=atmosphereTransmission)
640 
641  frame = getDebugFrame(self._display, "postISRCCD")
642  if frame:
643  display = getDisplay(frame)
644  display.scale('asinh', 'zscale')
645  display.mtv(ccdExposure)
646 
647  return pipeBase.Struct(
648  exposure=ccdExposure,
649  )
650 
651  @pipeBase.timeMethod
652  def runDataRef(self, sensorRef):
653  """Perform instrument signature removal on a ButlerDataRef of a Sensor
654 
655  - Read in necessary detrending/isr/calibration data
656  - Process raw exposure in run()
657  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
658 
659  Parameters
660  ----------
661  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
662  DataRef of the detector data to be processed
663 
664  Returns
665  -------
666  result : `pipeBase.Struct`
667  Struct contains field "exposure," which is the exposure after application of ISR
668  """
669  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
670  ccdExposure = sensorRef.get('raw')
671  camera = sensorRef.get("camera")
672  if camera is None and self.config.doAddDistortionModel:
673  raise RuntimeError("config.doAddDistortionModel is True "
674  "but could not get a camera from the butler")
675  isrData = self.readIsrData(sensorRef, ccdExposure)
676 
677  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
678 
679  if self.config.doWrite:
680  sensorRef.put(result.exposure, "postISRCCD")
681 
682  return result
683 
684  def convertIntToFloat(self, exposure):
685  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
686  """
687  if isinstance(exposure, afwImage.ExposureF):
688  # Nothing to be done
689  return exposure
690  if not hasattr(exposure, "convertF"):
691  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
692 
693  newexposure = exposure.convertF()
694  newexposure.variance[:] = 1
695  newexposure.mask[:] = 0x0
696 
697  return newexposure
698 
699  def biasCorrection(self, exposure, biasExposure):
700  """!Apply bias correction in place
701 
702  @param[in,out] exposure exposure to process
703  @param[in] biasExposure bias exposure of same size as exposure
704  """
705  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
706 
707  def darkCorrection(self, exposure, darkExposure, invert=False):
708  """!Apply dark correction in place
709 
710  @param[in,out] exposure exposure to process
711  @param[in] darkExposure dark exposure of same size as exposure
712  @param[in] invert if True, remove the dark from an already-corrected image
713  """
714  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
715  if math.isnan(expScale):
716  raise RuntimeError("Exposure darktime is NAN")
717  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
718  if math.isnan(darkScale):
719  raise RuntimeError("Dark calib darktime is NAN")
720  isrFunctions.darkCorrection(
721  maskedImage=exposure.getMaskedImage(),
722  darkMaskedImage=darkExposure.getMaskedImage(),
723  expScale=expScale,
724  darkScale=darkScale,
725  invert=invert
726  )
727 
728  def doLinearize(self, detector):
729  """!Is linearization wanted for this detector?
730 
731  Checks config.doLinearize and the linearity type of the first amplifier.
732 
733  @param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
734  """
735  return self.config.doLinearize and \
736  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
737 
738  def updateVariance(self, ampExposure, amp, overscanImage=None):
739  """Set the variance plane using the amplifier gain and read noise
740 
741  The read noise is calculated from the ``overscanImage`` if the
742  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
743  the value from the amplifier data is used.
744 
745  Parameters
746  ----------
747  ampExposure : `lsst.afw.image.Exposure`
748  Exposure to process.
749  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
750  Amplifier detector data.
751  overscanImage : `lsst.afw.image.MaskedImage`, optional.
752  Image of overscan, required only for empirical read noise.
753  """
754  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
755  gain = amp.getGain()
756  if not math.isnan(gain):
757  if gain <= 0:
758  patchedGain = 1.0
759  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
760  (amp.getName(), gain, patchedGain))
761  gain = patchedGain
762 
763  if self.config.doEmpiricalReadNoise and overscanImage is not None:
764  stats = afwMath.StatisticsControl()
765  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
766  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
767  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
768  else:
769  readNoise = amp.getReadNoise()
770 
771  isrFunctions.updateVariance(
772  maskedImage=ampExposure.getMaskedImage(),
773  gain=gain,
774  readNoise=readNoise,
775  )
776 
777  def flatCorrection(self, exposure, flatExposure, invert=False):
778  """!Apply flat correction in place
779 
780  @param[in,out] exposure exposure to process
781  @param[in] flatExposure flatfield exposure same size as exposure
782  @param[in] invert if True, unflatten an already-flattened image instead.
783  """
784  isrFunctions.flatCorrection(
785  maskedImage=exposure.getMaskedImage(),
786  flatMaskedImage=flatExposure.getMaskedImage(),
787  scalingType=self.config.flatScalingType,
788  userScale=self.config.flatUserScale,
789  invert=invert
790  )
791 
792  def getIsrExposure(self, dataRef, datasetType, immediate=True):
793  """!Retrieve a calibration dataset for removing instrument signature
794 
795  @param[in] dataRef data reference for exposure
796  @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
797  @param[in] immediate if True, disable butler proxies to enable error
798  handling within this routine
799  @return exposure
800  """
801  try:
802  exp = dataRef.get(datasetType, immediate=immediate)
803  except Exception as exc1:
804  if not self.config.fallbackFilterName:
805  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
806  try:
807  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
808  except Exception as exc2:
809  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
810  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
811  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
812 
813  if self.config.doAssembleIsrExposures:
814  exp = self.assembleCcd.assembleCcd(exp)
815  return exp
816 
817  def saturationDetection(self, exposure, amp):
818  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
819 
820  @param[in,out] exposure exposure to process; only the amp DataSec is processed
821  @param[in] amp amplifier device data
822  """
823  if not math.isnan(amp.getSaturation()):
824  maskedImage = exposure.getMaskedImage()
825  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
826  isrFunctions.makeThresholdMask(
827  maskedImage=dataView,
828  threshold=amp.getSaturation(),
829  growFootprints=0,
830  maskName=self.config.saturatedMaskName,
831  )
832 
833  def saturationInterpolation(self, ccdExposure):
834  """!Interpolate over saturated pixels, in place
835 
836  @param[in,out] ccdExposure exposure to process
837 
838  @warning:
839  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
840  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
841  """
842  isrFunctions.interpolateFromMask(
843  maskedImage=ccdExposure.getMaskedImage(),
844  fwhm=self.config.fwhm,
845  growFootprints=self.config.growSaturationFootprintSize,
846  maskName=self.config.saturatedMaskName,
847  )
848 
849  def suspectDetection(self, exposure, amp):
850  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
851 
852  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
853  This is intended to indicate pixels that may be affected by unknown systematics;
854  for example if non-linearity corrections above a certain level are unstable
855  then that would be a useful value for suspectLevel. A value of `nan` indicates
856  that no such level exists and no pixels are to be masked as suspicious.
857 
858  @param[in,out] exposure exposure to process; only the amp DataSec is processed
859  @param[in] amp amplifier device data
860  """
861  suspectLevel = amp.getSuspectLevel()
862  if math.isnan(suspectLevel):
863  return
864 
865  maskedImage = exposure.getMaskedImage()
866  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
867  isrFunctions.makeThresholdMask(
868  maskedImage=dataView,
869  threshold=suspectLevel,
870  growFootprints=0,
871  maskName=self.config.suspectMaskName,
872  )
873 
874  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
875  """!Mask defects using mask plane "BAD" and interpolate over them, in place
876 
877  @param[in,out] ccdExposure exposure to process
878  @param[in] defectBaseList a list of defects to mask and interpolate
879 
880  @warning: call this after CCD assembly, since defects may cross amplifier boundaries
881  """
882  maskedImage = ccdExposure.getMaskedImage()
883  defectList = []
884  for d in defectBaseList:
885  bbox = d.getBBox()
886  nd = measAlg.Defect(bbox)
887  defectList.append(nd)
888  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
889  isrFunctions.interpolateDefectList(
890  maskedImage=maskedImage,
891  defectList=defectList,
892  fwhm=self.config.fwhm,
893  )
894 
895  def maskAndInterpNan(self, exposure):
896  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
897 
898  We mask and interpolate over all NaNs, including those
899  that are masked with other bits (because those may or may
900  not be interpolated over later, and we want to remove all
901  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
902  is used to preserve the historical name.
903 
904  @param[in,out] exposure exposure to process
905  """
906  maskedImage = exposure.getMaskedImage()
907 
908  # Find and mask NaNs
909  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
910  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
911  numNans = maskNans(maskedImage, maskVal)
912  self.metadata.set("NUMNANS", numNans)
913 
914  # Interpolate over these previously-unmasked NaNs
915  if numNans > 0:
916  self.log.warn("There were %i unmasked NaNs", numNans)
917  nanDefectList = isrFunctions.getDefectListFromMask(
918  maskedImage=maskedImage,
919  maskName='UNMASKEDNAN',
920  )
921  isrFunctions.interpolateDefectList(
922  maskedImage=exposure.getMaskedImage(),
923  defectList=nanDefectList,
924  fwhm=self.config.fwhm,
925  )
926 
927  def overscanCorrection(self, exposure, amp):
928  """Apply overscan correction, in-place
929 
930  Parameters
931  ----------
932  exposure : `lsst.afw.image.Exposure`
933  Exposure to process; must include both data and bias regions.
934  amp : `lsst.afw.table.AmpInfoRecord`
935  Amplifier device data.
936 
937  Results
938  -------
939  result : `lsst.pipe.base.Struct` or `NoneType`
940  `None` if there is no overscan; otherwise, this is a
941  result struct with components:
942 
943  - ``imageFit``: Value(s) removed from image (scalar or
944  `lsst.afw.image.Image`).
945  - ``overscanFit``: Value(s) removed from overscan (scalar or
946  `lsst.afw.image.Image`).
947  - ``overscanImage``: Image of the overscan, post-subtraction
948  (`lsst.afw.image.Image`).
949  """
950  if not amp.getHasRawInfo():
951  raise RuntimeError("This method must be executed on an amp with raw information.")
952 
953  if amp.getRawHorizontalOverscanBBox().isEmpty():
954  self.log.info("No Overscan region. Not performing Overscan Correction.")
955  return None
956 
957  oscanBBox = amp.getRawHorizontalOverscanBBox()
958 
959  # afw.cameraGeom.assembleImage.makeUpdatedDetector doesn't update readoutCorner; DM-15559
960  x0, x1 = oscanBBox.getBeginX(), oscanBBox.getEndX()
961 
962  prescanBBox = amp.getRawPrescanBBox()
963  if oscanBBox.getBeginX() > prescanBBox.getBeginX(): # amp is at the right
964  x0 += self.config.overscanNumLeadingColumnsToSkip
965  x1 -= self.config.overscanNumTrailingColumnsToSkip
966  else:
967  x0 += self.config.overscanNumTrailingColumnsToSkip
968  x1 -= self.config.overscanNumLeadingColumnsToSkip
969 
970  oscanBBox = afwGeom.BoxI(afwGeom.PointI(x0, oscanBBox.getBeginY()),
971  afwGeom.PointI(x1 - 1, oscanBBox.getEndY() - 1))
972 
973  maskedImage = exposure.maskedImage
974  dataView = maskedImage[amp.getRawDataBBox()]
975  overscanImage = maskedImage[oscanBBox]
976 
977  sctrl = afwMath.StatisticsControl()
978  sctrl.setNumSigmaClip(self.config.overscanNumSigmaClip)
979 
980  results = isrFunctions.overscanCorrection(
981  ampMaskedImage=dataView,
982  overscanImage=overscanImage,
983  fitType=self.config.overscanFitType,
984  order=self.config.overscanOrder,
985  statControl=sctrl,
986  )
987  results.overscanImage = overscanImage
988  return results
989 
990  def addDistortionModel(self, exposure, camera):
991  """!Update the WCS in exposure with a distortion model based on camera geometry
992 
993  Add a model for optical distortion based on geometry found in `camera`
994  and the `exposure`'s detector. The raw input exposure is assumed
995  have a TAN WCS that has no compensation for optical distortion.
996  Two other possibilities are:
997  - The raw input exposure already has a model for optical distortion,
998  as is the case for raw DECam data.
999  In that case you should set config.doAddDistortionModel False.
1000  - The raw input exposure has a model for distortion, but it has known
1001  deficiencies severe enough to be worth fixing (e.g. because they
1002  cause problems for fitting a better WCS). In that case you should
1003  override this method with a version suitable for your raw data.
1004 
1005  @param[in,out] exposure exposure to process; must include a Detector and a WCS;
1006  the WCS of the exposure is modified in place
1007  @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera
1008  """
1009  self.log.info("Adding a distortion model to the WCS")
1010  wcs = exposure.getWcs()
1011  if wcs is None:
1012  raise RuntimeError("exposure has no WCS")
1013  if camera is None:
1014  raise RuntimeError("camera is None")
1015  detector = exposure.getDetector()
1016  if detector is None:
1017  raise RuntimeError("exposure has no Detector")
1018  pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
1019  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
1020  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
1021  exposure.setWcs(distortedWcs)
1022 
1023  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
1024  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
1025 
1026  @param[in,out] ccdExposure exposure to process
1027  @param[in] fpPolygon Polygon in focal plane coordinates
1028  """
1029  # Get ccd corners in focal plane coordinates
1030  ccd = ccdExposure.getDetector()
1031  fpCorners = ccd.getCorners(FOCAL_PLANE)
1032  ccdPolygon = Polygon(fpCorners)
1033 
1034  # Get intersection of ccd corners with fpPolygon
1035  intersect = ccdPolygon.intersectionSingle(fpPolygon)
1036 
1037  # Transform back to pixel positions and build new polygon
1038  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
1039  validPolygon = Polygon(ccdPoints)
1040  ccdExposure.getInfo().setValidPolygon(validPolygon)
1041 
1042  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
1043  """Apply brighter fatter correction in place for the image
1044 
1045  This correction takes a kernel that has been derived from flat field images to
1046  redistribute the charge. The gradient of the kernel is the deflection
1047  field due to the accumulated charge.
1048 
1049  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
1050  using the following equation:
1051 
1052  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
1053 
1054  To evaluate the derivative term we expand it as follows:
1055 
1056  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
1057 
1058  Because we use the measured counts instead of the incident counts we apply the correction
1059  iteratively to reconstruct the original counts and the correction. We stop iterating when the
1060  summed difference between the current corrected image and the one from the previous iteration
1061  is below the threshold. We do not require convergence because the number of iterations is
1062  too large a computational cost. How we define the threshold still needs to be evaluated, the
1063  current default was shown to work reasonably well on a small set of images. For more information
1064  on the method see DocuShare Document-19407.
1065 
1066  The edges as defined by the kernel are not corrected because they have spurious values
1067  due to the convolution.
1068  """
1069  self.log.info("Applying brighter fatter correction")
1070 
1071  image = exposure.getMaskedImage().getImage()
1072 
1073  # The image needs to be units of electrons/holes
1074  with self.gainContext(exposure, image, applyGain):
1075 
1076  kLx = numpy.shape(kernel)[0]
1077  kLy = numpy.shape(kernel)[1]
1078  kernelImage = afwImage.ImageD(kLx, kLy)
1079  kernelImage.getArray()[:, :] = kernel
1080  tempImage = image.clone()
1081 
1082  nanIndex = numpy.isnan(tempImage.getArray())
1083  tempImage.getArray()[nanIndex] = 0.
1084 
1085  outImage = afwImage.ImageF(image.getDimensions())
1086  corr = numpy.zeros_like(image.getArray())
1087  prev_image = numpy.zeros_like(image.getArray())
1088  convCntrl = afwMath.ConvolutionControl(False, True, 1)
1089  fixedKernel = afwMath.FixedKernel(kernelImage)
1090 
1091  # Define boundary by convolution region. The region that the correction will be
1092  # calculated for is one fewer in each dimension because of the second derivative terms.
1093  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
1094  startX = kLx//2
1095  endX = -kLx//2
1096  startY = kLy//2
1097  endY = -kLy//2
1098 
1099  for iteration in range(maxIter):
1100 
1101  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1102  tmpArray = tempImage.getArray()
1103  outArray = outImage.getArray()
1104 
1105  with numpy.errstate(invalid="ignore", over="ignore"):
1106  # First derivative term
1107  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1108  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1109  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1110 
1111  # Second derivative term
1112  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1113  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1114  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1115 
1116  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1117 
1118  tmpArray[:, :] = image.getArray()[:, :]
1119  tmpArray[nanIndex] = 0.
1120  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1121 
1122  if iteration > 0:
1123  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1124 
1125  if diff < threshold:
1126  break
1127  prev_image[:, :] = tmpArray[:, :]
1128 
1129  if iteration == maxIter - 1:
1130  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
1131 
1132  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
1133  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1134  corr[startY + 1:endY - 1, startX + 1:endX - 1]
1135 
1136  def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None,
1137  sensorTransmission=None, atmosphereTransmission=None):
1138  """Attach a TransmissionCurve to an Exposure, given separate curves for
1139  different components.
1140 
1141  Parameters
1142  ----------
1143  exposure : `lsst.afw.image.Exposure`
1144  Exposure object to modify by attaching the product of all given
1145  ``TransmissionCurves`` in post-assembly trimmed detector
1146  coordinates. Must have a valid ``Detector`` attached that matches
1147  the detector associated with sensorTransmission.
1148  opticsTransmission : `lsst.afw.image.TransmissionCurve`
1149  A ``TransmissionCurve`` that represents the throughput of the
1150  optics, to be evaluated in focal-plane coordinates.
1151  filterTransmission : `lsst.afw.image.TransmissionCurve`
1152  A ``TransmissionCurve`` that represents the throughput of the
1153  filter itself, to be evaluated in focal-plane coordinates.
1154  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1155  A ``TransmissionCurve`` that represents the throughput of the
1156  sensor itself, to be evaluated in post-assembly trimmed detector
1157  coordinates.
1158  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1159  A ``TransmissionCurve`` that represents the throughput of the
1160  atmosphere, assumed to be spatially constant.
1161 
1162  All ``TransmissionCurve`` arguments are optional; if none are provided,
1163  the attached ``TransmissionCurve`` will have unit transmission
1164  everywhere.
1165 
1166  Returns
1167  -------
1168  combined : ``lsst.afw.image.TransmissionCurve``
1169  The TransmissionCurve attached to the exposure.
1170  """
1171  return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1172  filterTransmission=filterTransmission,
1173  sensorTransmission=sensorTransmission,
1174  atmosphereTransmission=atmosphereTransmission)
1175 
1176  @contextmanager
1177  def gainContext(self, exp, image, apply):
1178  """Context manager that applies and removes gain
1179  """
1180  if apply:
1181  ccd = exp.getDetector()
1182  for amp in ccd:
1183  sim = image.Factory(image, amp.getBBox())
1184  sim *= amp.getGain()
1185 
1186  try:
1187  yield exp
1188  finally:
1189  if apply:
1190  ccd = exp.getDetector()
1191  for amp in ccd:
1192  sim = image.Factory(image, amp.getBBox())
1193  sim /= amp.getGain()
1194 
1195  @contextmanager
1196  def flatContext(self, exp, flat, dark=None):
1197  """Context manager that applies and removes flats and darks,
1198  if the task is configured to apply them.
1199  """
1200  if self.config.doDark and dark is not None:
1201  self.darkCorrection(exp, dark)
1202  if self.config.doFlat:
1203  self.flatCorrection(exp, flat)
1204  try:
1205  yield exp
1206  finally:
1207  if self.config.doFlat:
1208  self.flatCorrection(exp, flat, invert=True)
1209  if self.config.doDark and dark is not None:
1210  self.darkCorrection(exp, dark, invert=True)
1211 
1212 
1213 class FakeAmp(object):
1214  """A Detector-like object that supports returning gain and saturation level"""
1215 
1216  def __init__(self, exposure, config):
1217  self._bbox = exposure.getBBox(afwImage.LOCAL)
1218  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
1219  self._gain = config.gain
1220  self._readNoise = config.readNoise
1221  self._saturation = config.saturation
1222 
1223  def getBBox(self):
1224  return self._bbox
1225 
1226  def getRawBBox(self):
1227  return self._bbox
1228 
1229  def getHasRawInfo(self):
1230  return True # but see getRawHorizontalOverscanBBox()
1231 
1233  return self._RawHorizontalOverscanBBox
1234 
1235  def getGain(self):
1236  return self._gain
1237 
1238  def getReadNoise(self):
1239  return self._readNoise
1240 
1241  def getSaturation(self):
1242  return self._saturation
1243 
1244  def getSuspectLevel(self):
1245  return float("NaN")
1246 
1247 
1248 class RunIsrConfig(pexConfig.Config):
1249  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
1250 
1251 
1257 
1258 
1259 class RunIsrTask(pipeBase.CmdLineTask):
1260  """Task to wrap the default IsrTask to allow it to be retargeted.
1261 
1262  The standard IsrTask can be called directly from a command line
1263  program, but doing so removes the ability of the task to be
1264  retargeted. As most cameras override some set of the IsrTask
1265  methods, this would remove those data-specific methods in the
1266  output post-ISR images. This wrapping class fixes the issue,
1267  allowing identical post-ISR images to be generated by both the
1268  processCcd and isrTask code.
1269  """
1270  ConfigClass = RunIsrConfig
1271  _DefaultName = "runIsr"
1272 
1273  def __init__(self, *args, **kwargs):
1274  super().__init__(*args, **kwargs)
1275  self.makeSubtask("isr")
1276 
1277  def runDataRef(self, dataRef):
1278  """
1279  Parameters
1280  ----------
1281  dataRef : `lsst.daf.persistence.ButlerDataRef`
1282  data reference of the detector data to be processed
1283 
1284  Returns
1285  -------
1286  result : `pipeBase.Struct`
1287  Result struct with component:
1288 
1289  - exposure : `lsst.afw.image.Exposure`
1290  Post-ISR processed exposure.
1291  """
1292  return self.isr.runDataRef(dataRef)
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
Definition: isrTask.py:1042
def runDataRef(self, sensorRef)
Definition: isrTask.py:652
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:489
def gainContext(self, exp, image, apply)
Definition: isrTask.py:1177
def __init__(self, args, kwargs)
Constructor for IsrTask.
Definition: isrTask.py:403
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:415
def runDataRef(self, dataRef)
Definition: isrTask.py:1277
def __init__(self, args, kwargs)
Definition: isrTask.py:1273
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Definition: isrTask.py:1137
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:895
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:833
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:329
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:1232
def convertIntToFloat(self, exposure)
Definition: isrTask.py:684
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:777
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:792
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:707
def doLinearize(self, detector)
Is linearization wanted for this detector?
Definition: isrTask.py:728
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
Definition: isrTask.py:990
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:1023
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
Definition: isrTask.py:699
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:1196
def overscanCorrection(self, exposure, amp)
Definition: isrTask.py:927
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:738
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:849
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:874
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:817
def __init__(self, exposure, config)
Definition: isrTask.py:1216