lsst.ip.isr  16.0-11-gf78ffa7
isrTask.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.meas.algorithms as measAlg
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.afw.math as afwMath
31 from lsst.daf.persistence import ButlerDataRef
32 from lsstDebug import getDebugFrame
33 from lsst.afw.display import getDisplay
34 from . import isrFunctions
35 from .assembleCcdTask import AssembleCcdTask
36 from .fringe import FringeTask
37 from lsst.afw.geom import Polygon
38 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
39 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, FIELD_ANGLE, NullLinearityType
40 from contextlib import contextmanager
41 from .isr import maskNans
42 from .crosstalk import CrosstalkTask
43 
44 
45 class IsrTaskConfig(pexConfig.Config):
46  doBias = pexConfig.Field(
47  dtype=bool,
48  doc="Apply bias frame correction?",
49  default=True,
50  )
51  doDark = pexConfig.Field(
52  dtype=bool,
53  doc="Apply dark frame correction?",
54  default=True,
55  )
56  doFlat = pexConfig.Field(
57  dtype=bool,
58  doc="Apply flat field correction?",
59  default=True,
60  )
61  doFringe = pexConfig.Field(
62  dtype=bool,
63  doc="Apply fringe correction?",
64  default=True,
65  )
66  doDefect = pexConfig.Field(
67  dtype=bool,
68  doc="Apply correction for CCD defects, e.g. hot pixels?",
69  default=True,
70  )
71  doAddDistortionModel = pexConfig.Field(
72  dtype=bool,
73  doc="Apply a distortion model based on camera geometry to the WCS?",
74  default=True,
75  )
76  doWrite = pexConfig.Field(
77  dtype=bool,
78  doc="Persist postISRCCD?",
79  default=True,
80  )
81  biasDataProductName = pexConfig.Field(
82  dtype=str,
83  doc="Name of the bias data product",
84  default="bias",
85  )
86  darkDataProductName = pexConfig.Field(
87  dtype=str,
88  doc="Name of the dark data product",
89  default="dark",
90  )
91  flatDataProductName = pexConfig.Field(
92  dtype=str,
93  doc="Name of the flat data product",
94  default="flat",
95  )
96  assembleCcd = pexConfig.ConfigurableField(
97  target=AssembleCcdTask,
98  doc="CCD assembly task",
99  )
100  gain = pexConfig.Field(
101  dtype=float,
102  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
103  default=float("NaN"),
104  )
105  readNoise = pexConfig.Field(
106  dtype=float,
107  doc="The read noise to use if no Detector is present in the Exposure",
108  default=0.0,
109  )
110  saturation = pexConfig.Field(
111  dtype=float,
112  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
113  default=float("NaN"),
114  )
115  fringeAfterFlat = pexConfig.Field(
116  dtype=bool,
117  doc="Do fringe subtraction after flat-fielding?",
118  default=True,
119  )
120  fringe = pexConfig.ConfigurableField(
121  target=FringeTask,
122  doc="Fringe subtraction task",
123  )
124  fwhm = pexConfig.Field(
125  dtype=float,
126  doc="FWHM of PSF (arcsec)",
127  default=1.0,
128  )
129  saturatedMaskName = pexConfig.Field(
130  dtype=str,
131  doc="Name of mask plane to use in saturation detection and interpolation",
132  default="SAT",
133  )
134  suspectMaskName = pexConfig.Field(
135  dtype=str,
136  doc="Name of mask plane to use for suspect pixels",
137  default="SUSPECT",
138  )
139  flatScalingType = pexConfig.ChoiceField(
140  dtype=str,
141  doc="The method for scaling the flat on the fly.",
142  default='USER',
143  allowed={
144  "USER": "Scale by flatUserScale",
145  "MEAN": "Scale by the inverse of the mean",
146  "MEDIAN": "Scale by the inverse of the median",
147  },
148  )
149  flatUserScale = pexConfig.Field(
150  dtype=float,
151  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
152  default=1.0,
153  )
154  overscanFitType = pexConfig.ChoiceField(
155  dtype=str,
156  doc="The method for fitting the overscan bias level.",
157  default='MEDIAN',
158  allowed={
159  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
160  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
161  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
162  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
163  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
164  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
165  "MEAN": "Correct using the mean of the overscan region",
166  "MEDIAN": "Correct using the median of the overscan region",
167  },
168  )
169  overscanOrder = pexConfig.Field(
170  dtype=int,
171  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
172  "or number of spline knots if overscan fit type is a spline."),
173  default=1,
174  )
175  overscanRej = pexConfig.Field(
176  dtype=float,
177  doc="Rejection threshold (sigma) for collapsing overscan before fit",
178  default=3.0,
179  )
180 
181  overscanNumLeadingColumnsToSkip = pexConfig.Field(
182  dtype=int,
183  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
184  default=0,
185  )
186  overscanNumTrailingColumnsToSkip = pexConfig.Field(
187  dtype=int,
188  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
189  default=0,
190  )
191  growSaturationFootprintSize = pexConfig.Field(
192  dtype=int,
193  doc="Number of pixels by which to grow the saturation footprints",
194  default=1,
195  )
196  doSaturationInterpolation = pexConfig.Field(
197  dtype=bool,
198  doc="Perform interpolation over pixels masked as saturated?",
199  default=True,
200  )
201  doNanInterpAfterFlat = pexConfig.Field(
202  dtype=bool,
203  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
204  "also have to interpolate them before flat-fielding."),
205  default=False,
206  )
207  fluxMag0T1 = pexConfig.Field(
208  dtype=float,
209  doc="The approximate flux of a zero-magnitude object in a one-second exposure",
210  default=1e10,
211  )
212  keysToRemoveFromAssembledCcd = pexConfig.ListField(
213  dtype=str,
214  doc="fields to remove from the metadata of the assembled ccd.",
215  default=[],
216  )
217  doAssembleIsrExposures = pexConfig.Field(
218  dtype=bool,
219  default=False,
220  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
221  )
222  doAssembleCcd = pexConfig.Field(
223  dtype=bool,
224  default=True,
225  doc="Assemble amp-level exposures into a ccd-level exposure?"
226  )
227  expectWcs = pexConfig.Field(
228  dtype=bool,
229  default=True,
230  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)"
231  )
232  doLinearize = pexConfig.Field(
233  dtype=bool,
234  doc="Correct for nonlinearity of the detector's response?",
235  default=True,
236  )
237  doCrosstalk = pexConfig.Field(
238  dtype=bool,
239  doc="Apply intra-CCD crosstalk correction?",
240  default=False,
241  )
242  crosstalk = pexConfig.ConfigurableField(
243  target=CrosstalkTask,
244  doc="Intra-CCD crosstalk correction",
245  )
246  doBrighterFatter = pexConfig.Field(
247  dtype=bool,
248  default=False,
249  doc="Apply the brighter fatter correction"
250  )
251  brighterFatterLevel = pexConfig.ChoiceField(
252  doc="The level at which to correct for brighter-fatter",
253  dtype=str, default="DETECTOR",
254  allowed={
255  "AMP": "Every amplifier treated separately",
256  "DETECTOR": "One kernel per detector",
257  }
258  )
259  brighterFatterKernelFile = pexConfig.Field(
260  dtype=str,
261  default='',
262  doc="Kernel file used for the brighter fatter correction"
263  )
264  brighterFatterMaxIter = pexConfig.Field(
265  dtype=int,
266  default=10,
267  doc="Maximum number of iterations for the brighter fatter correction"
268  )
269  brighterFatterThreshold = pexConfig.Field(
270  dtype=float,
271  default=1000,
272  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
273  " absolute value of the difference between the current corrected image and the one"
274  " from the previous iteration summed over all the pixels."
275  )
276  brighterFatterApplyGain = pexConfig.Field(
277  dtype=bool,
278  default=True,
279  doc="Should the gain be applied when applying the brighter fatter correction?"
280  )
281  datasetType = pexConfig.Field(
282  dtype=str,
283  doc="Dataset type for input data; users will typically leave this alone, "
284  "but camera-specific ISR tasks will override it",
285  default="raw",
286  )
287  fallbackFilterName = pexConfig.Field(dtype=str,
288  doc="Fallback default filter name for calibrations", optional=True)
289  doAttachTransmissionCurve = pexConfig.Field(
290  dtype=bool,
291  default=False,
292  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
293  )
294  doUseOpticsTransmission = pexConfig.Field(
295  dtype=bool,
296  default=True,
297  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
298  )
299  doUseFilterTransmission = pexConfig.Field(
300  dtype=bool,
301  default=True,
302  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
303  )
304  doUseSensorTransmission = pexConfig.Field(
305  dtype=bool,
306  default=True,
307  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
308  )
309  doUseAtmosphereTransmission = pexConfig.Field(
310  dtype=bool,
311  default=True,
312  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
313  )
314  doEmpiricalReadNoise = pexConfig.Field(
315  dtype=bool,
316  default=False,
317  doc="Calculate empirical read noise instead of value from AmpInfo data?"
318  )
319 
320 
326 
327 
328 class IsrTask(pipeBase.CmdLineTask):
329  """!
330  @anchor IsrTask_
331 
332  @brief Apply common instrument signature correction algorithms to a raw frame.
333 
334  @section ip_isr_isr_Contents Contents
335 
336  - @ref ip_isr_isr_Purpose
337  - @ref ip_isr_isr_Initialize
338  - @ref ip_isr_isr_IO
339  - @ref ip_isr_isr_Config
340  - @ref ip_isr_isr_Debug
341 
342 
343  @section ip_isr_isr_Purpose Description
344 
345  The process for correcting imaging data is very similar from camera to camera.
346  This task provides a vanilla implementation of doing these corrections, including
347  the ability to turn certain corrections off if they are not needed.
348  The inputs to the primary method, run, are a raw exposure to be corrected and the
349  calibration data products. The raw input is a single chip sized mosaic of all amps
350  including overscans and other non-science pixels.
351  The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask
352  and takes as input only a daf.persistence.butlerSubset.ButlerDataRef.
353  This task may not meet all needs and it is expected that it will be subclassed for
354  specific applications.
355 
356  @section ip_isr_isr_Initialize Task initialization
357 
358  @copydoc \_\_init\_\_
359 
360  @section ip_isr_isr_IO Inputs/Outputs to the run method
361 
362  @copydoc run
363 
364  @section ip_isr_isr_Config Configuration parameters
365 
366  See @ref IsrTaskConfig
367 
368  @section ip_isr_isr_Debug Debug variables
369 
370  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
371  flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a
372  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
373  Using lsstDebug to control debugging output</a> for more about @b debug.py files.
374 
375  The available variables in IsrTask are:
376  <DL>
377  <DT> @c display
378  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
379  <DL>
380  <DT> postISRCCD
381  <DD> display exposure after ISR has been applied
382  </DL>
383  </DL>
384 
385  For example, put something like
386  @code{.py}
387  import lsstDebug
388  def DebugInfo(name):
389  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
390  if name == "lsst.ip.isrFunctions.isrTask":
391  di.display = {'postISRCCD':2}
392  return di
393  lsstDebug.Info = DebugInfo
394  @endcode
395  into your debug.py file and run the commandline task with the @c --debug flag.
396 
397  <HR>
398  """
399  ConfigClass = IsrTaskConfig
400  _DefaultName = "isr"
401 
402  def __init__(self, *args, **kwargs):
403  '''!Constructor for IsrTask
404  @param[in] *args a list of positional arguments passed on to the Task constructor
405  @param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor
406  Call the lsst.pipe.base.task.Task.__init__ method
407  Then setup the assembly and fringe correction subtasks
408  '''
409  pipeBase.Task.__init__(self, *args, **kwargs)
410  self.makeSubtask("assembleCcd")
411  self.makeSubtask("fringe")
412  self.makeSubtask("crosstalk")
413 
414  def readIsrData(self, dataRef, rawExposure):
415  """!Retrieve necessary frames for instrument signature removal
416  @param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef
417  of the detector data to be processed
418  @param[in] rawExposure a reference raw exposure that will later be
419  corrected with the retrieved calibration data;
420  should not be modified in this method.
421  @return a pipeBase.Struct with fields containing kwargs expected by run()
422  - bias: exposure of bias frame
423  - dark: exposure of dark frame
424  - flat: exposure of flat field
425  - defects: list of detects
426  - fringeStruct: a pipeBase.Struct with field fringes containing
427  exposure of fringe frame or list of fringe exposure
428  """
429  ccd = rawExposure.getDetector()
430 
431  biasExposure = self.getIsrExposure(dataRef, self.config.biasDataProductName) \
432  if self.config.doBias else None
433  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
434  linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None
435  darkExposure = self.getIsrExposure(dataRef, self.config.darkDataProductName) \
436  if self.config.doDark else None
437  flatExposure = self.getIsrExposure(dataRef, self.config.flatDataProductName) \
438  if self.config.doFlat else None
439  brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None
440  defectList = dataRef.get("defects") if self.config.doDefect else None
441 
442  if self.config.doCrosstalk:
443  crosstalkSources = self.crosstalk.prepCrosstalk(dataRef)
444  else:
445  crosstalkSources = None
446 
447  if self.config.doFringe and self.fringe.checkFilter(rawExposure):
448  fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd
449  if self.config.doAssembleIsrExposures else None)
450  else:
451  fringeStruct = pipeBase.Struct(fringes=None)
452 
453  if self.config.doAttachTransmissionCurve:
454  opticsTransmission = (dataRef.get("transmission_optics")
455  if self.config.doUseOpticsTransmission else None)
456  filterTransmission = (dataRef.get("transmission_filter")
457  if self.config.doUseFilterTransmission else None)
458  sensorTransmission = (dataRef.get("transmission_sensor")
459  if self.config.doUseSensorTransmission else None)
460  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
461  if self.config.doUseAtmosphereTransmission else None)
462  else:
463  opticsTransmission = None
464  filterTransmission = None
465  sensorTransmission = None
466  atmosphereTransmission = None
467 
468  # Struct should include only kwargs to run()
469  return pipeBase.Struct(bias=biasExposure,
470  linearizer=linearizer,
471  dark=darkExposure,
472  flat=flatExposure,
473  defects=defectList,
474  fringes=fringeStruct,
475  bfKernel=brighterFatterKernel,
476  opticsTransmission=opticsTransmission,
477  filterTransmission=filterTransmission,
478  sensorTransmission=sensorTransmission,
479  atmosphereTransmission=atmosphereTransmission,
480  crosstalkSources=crosstalkSources,
481  )
482 
483  @pipeBase.timeMethod
484  def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
485  fringes=None, bfKernel=None, camera=None,
486  opticsTransmission=None, filterTransmission=None,
487  sensorTransmission=None, atmosphereTransmission=None,
488  crosstalkSources=None):
489  """!Perform instrument signature removal on an exposure
490 
491  Steps include:
492  - Detect saturation, apply overscan correction, bias, dark and flat
493  - Perform CCD assembly
494  - Interpolate over defects, saturated pixels and all NaNs
495 
496  @param[in] ccdExposure lsst.afw.image.exposure of detector data
497  @param[in] bias exposure of bias frame
498  @param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase
499  @param[in] dark exposure of dark frame
500  @param[in] flat exposure of flatfield
501  @param[in] defects list of detects
502  @param[in] fringes a pipeBase.Struct with field fringes containing
503  exposure of fringe frame or list of fringe exposure
504  @param[in] bfKernel kernel for brighter-fatter correction, an
505  lsst.cp.pipe.makeBrighterFatterKernel.BrighterFatterKernel object
506  @param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera;
507  used by addDistortionModel
508  @param[in] opticsTransmission a TransmissionCurve for the optics
509  @param[in] filterTransmission a TransmissionCurve for the filter
510  @param[in] sensorTransmission a TransmissionCurve for the sensor
511  @param[in] atmosphereTransmission a TransmissionCurve for the atmosphere
512  @param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk
513 
514  @return a pipeBase.Struct with field:
515  - exposure
516  """
517  # parseAndRun expects to be able to call run() with a dataRef; see DM-6640
518  if isinstance(ccdExposure, ButlerDataRef):
519  return self.runDataRef(ccdExposure)
520 
521  ccd = ccdExposure.getDetector()
522 
523  # Validate Input
524  if self.config.doBias and bias is None:
525  raise RuntimeError("Must supply a bias exposure if config.doBias True")
526  if self.doLinearize(ccd) and linearizer is None:
527  raise RuntimeError("Must supply a linearizer if config.doBias True")
528  if self.config.doDark and dark is None:
529  raise RuntimeError("Must supply a dark exposure if config.doDark True")
530  if self.config.doFlat and flat is None:
531  raise RuntimeError("Must supply a flat exposure if config.doFlat True")
532  if self.config.doBrighterFatter and bfKernel is None:
533  raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
534  if fringes is None:
535  fringes = pipeBase.Struct(fringes=None)
536  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
537  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
538  if self.config.doDefect and defects is None:
539  raise RuntimeError("Must supply defects if config.doDefect True")
540  if self.config.doAddDistortionModel and camera is None:
541  raise RuntimeError("Must supply camera if config.doAddDistortionModel True")
542 
543  ccdExposure = self.convertIntToFloat(ccdExposure)
544 
545  if not ccd:
546  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
547  ccd = [FakeAmp(ccdExposure, self.config)]
548 
549  overscans = []
550  for amp in ccd:
551  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
552  if ccdExposure.getBBox().contains(amp.getBBox()):
553  self.saturationDetection(ccdExposure, amp)
554  self.suspectDetection(ccdExposure, amp)
555  overscanResults = self.overscanCorrection(ccdExposure, amp)
556  overscans.append(overscanResults.overscanImage if overscanResults is not None else None)
557  else:
558  overscans.append(None)
559 
560  if self.config.doCrosstalk:
561  self.crosstalk.run(ccdExposure, crosstalkSources)
562 
563  if self.config.doAssembleCcd:
564  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
565  if self.config.expectWcs and not ccdExposure.getWcs():
566  self.log.warn("No WCS found in input exposure")
567 
568  if self.config.doBias:
569  self.biasCorrection(ccdExposure, bias)
570 
571  if self.doLinearize(ccd):
572  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
573 
574  assert len(ccd) == len(overscans)
575  for amp, overscanImage in zip(ccd, overscans):
576  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
577  if ccdExposure.getBBox().contains(amp.getBBox()):
578  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
579  self.updateVariance(ampExposure, amp, overscanImage)
580 
581  interpolationDone = False
582 
583  if self.config.doBrighterFatter:
584 
585  # We need to apply flats and darks before we can interpolate, and we
586  # need to interpolate before we do B-F, but we do B-F without the
587  # flats and darks applied so we can work in units of electrons or holes.
588  # This context manager applies and then removes the darks and flats.
589  with self.flatContext(ccdExposure, flat, dark):
590  if self.config.doDefect:
591  self.maskAndInterpDefect(ccdExposure, defects)
592  if self.config.doSaturationInterpolation:
593  self.saturationInterpolation(ccdExposure)
594  self.maskAndInterpNan(ccdExposure)
595  interpolationDone = True
596 
597  if self.config.brighterFatterLevel == 'DETECTOR':
598  kernelElement = bfKernel.kernel[ccdExposure.getDetector().getId()]
599  else:
600  # TODO: DM-15631 for implementing this
601  raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented")
602  self.brighterFatterCorrection(ccdExposure, kernelElement,
603  self.config.brighterFatterMaxIter,
604  self.config.brighterFatterThreshold,
605  self.config.brighterFatterApplyGain,
606  )
607 
608  if self.config.doDark:
609  self.darkCorrection(ccdExposure, dark)
610 
611  if self.config.doFringe and not self.config.fringeAfterFlat:
612  self.fringe.run(ccdExposure, **fringes.getDict())
613 
614  if self.config.doFlat:
615  self.flatCorrection(ccdExposure, flat)
616 
617  if not interpolationDone:
618  if self.config.doDefect:
619  self.maskAndInterpDefect(ccdExposure, defects)
620  if self.config.doSaturationInterpolation:
621  self.saturationInterpolation(ccdExposure)
622  if not interpolationDone or self.config.doNanInterpAfterFlat:
623  self.maskAndInterpNan(ccdExposure)
624 
625  if self.config.doFringe and self.config.fringeAfterFlat:
626  self.fringe.run(ccdExposure, **fringes.getDict())
627 
628  exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime()
629  ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime)
630 
631  if self.config.doAddDistortionModel:
632  self.addDistortionModel(exposure=ccdExposure, camera=camera)
633 
634  if self.config.doAttachTransmissionCurve:
635  self.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
636  filterTransmission=filterTransmission,
637  sensorTransmission=sensorTransmission,
638  atmosphereTransmission=atmosphereTransmission)
639 
640  frame = getDebugFrame(self._display, "postISRCCD")
641  if frame:
642  getDisplay(frame).mtv(ccdExposure)
643 
644  return pipeBase.Struct(
645  exposure=ccdExposure,
646  )
647 
648  @pipeBase.timeMethod
649  def runDataRef(self, sensorRef):
650  """Perform instrument signature removal on a ButlerDataRef of a Sensor
651 
652  - Read in necessary detrending/isr/calibration data
653  - Process raw exposure in run()
654  - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
655 
656  Parameters
657  ----------
658  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
659  DataRef of the detector data to be processed
660 
661  Returns
662  -------
663  result : `pipeBase.Struct`
664  Struct contains field "exposure," which is the exposure after application of ISR
665  """
666  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
667  ccdExposure = sensorRef.get('raw')
668  camera = sensorRef.get("camera")
669  if camera is None and self.config.doAddDistortionModel:
670  raise RuntimeError("config.doAddDistortionModel is True "
671  "but could not get a camera from the butler")
672  isrData = self.readIsrData(sensorRef, ccdExposure)
673 
674  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
675 
676  if self.config.doWrite:
677  sensorRef.put(result.exposure, "postISRCCD")
678 
679  return result
680 
681  def convertIntToFloat(self, exposure):
682  """Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0
683  """
684  if isinstance(exposure, afwImage.ExposureF):
685  # Nothing to be done
686  return exposure
687  if not hasattr(exposure, "convertF"):
688  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
689 
690  newexposure = exposure.convertF()
691  maskedImage = newexposure.getMaskedImage()
692  varArray = maskedImage.getVariance().getArray()
693  varArray[:, :] = 1
694  maskArray = maskedImage.getMask().getArray()
695  maskArray[:, :] = 0
696  return newexposure
697 
698  def biasCorrection(self, exposure, biasExposure):
699  """!Apply bias correction in place
700 
701  @param[in,out] exposure exposure to process
702  @param[in] biasExposure bias exposure of same size as exposure
703  """
704  isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage())
705 
706  def darkCorrection(self, exposure, darkExposure, invert=False):
707  """!Apply dark correction in place
708 
709  @param[in,out] exposure exposure to process
710  @param[in] darkExposure dark exposure of same size as exposure
711  @param[in] invert if True, remove the dark from an already-corrected image
712  """
713  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
714  if math.isnan(expScale):
715  raise RuntimeError("Exposure darktime is NAN")
716  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
717  if math.isnan(darkScale):
718  raise RuntimeError("Dark calib darktime is NAN")
719  isrFunctions.darkCorrection(
720  maskedImage=exposure.getMaskedImage(),
721  darkMaskedImage=darkExposure.getMaskedImage(),
722  expScale=expScale,
723  darkScale=darkScale,
724  invert=invert
725  )
726 
727  def doLinearize(self, detector):
728  """!Is linearization wanted for this detector?
729 
730  Checks config.doLinearize and the linearity type of the first amplifier.
731 
732  @param[in] detector detector information (an lsst.afw.cameraGeom.Detector)
733  """
734  return self.config.doLinearize and \
735  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
736 
737  def updateVariance(self, ampExposure, amp, overscanImage=None):
738  """Set the variance plane using the amplifier gain and read noise
739 
740  The read noise is calculated from the ``overscanImage`` if the
741  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
742  the value from the amplifier data is used.
743 
744  Parameters
745  ----------
746  ampExposure : `lsst.afw.image.Exposure`
747  Exposure to process.
748  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
749  Amplifier detector data.
750  overscanImage : `lsst.afw.image.MaskedImage`, optional.
751  Image of overscan, required only for empirical read noise.
752  """
753  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
754  gain = amp.getGain()
755  if not math.isnan(gain):
756  if gain <= 0:
757  patchedGain = 1.0
758  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
759  (amp.getName(), gain, patchedGain))
760  gain = patchedGain
761 
762  if self.config.doEmpiricalReadNoise and overscanImage is not None:
763  stats = afwMath.StatisticsControl()
764  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
765  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
766  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
767  else:
768  readNoise = amp.getReadNoise()
769 
770  isrFunctions.updateVariance(
771  maskedImage=ampExposure.getMaskedImage(),
772  gain=gain,
773  readNoise=readNoise,
774  )
775 
776  def flatCorrection(self, exposure, flatExposure, invert=False):
777  """!Apply flat correction in place
778 
779  @param[in,out] exposure exposure to process
780  @param[in] flatExposure flatfield exposure same size as exposure
781  @param[in] invert if True, unflatten an already-flattened image instead.
782  """
783  isrFunctions.flatCorrection(
784  maskedImage=exposure.getMaskedImage(),
785  flatMaskedImage=flatExposure.getMaskedImage(),
786  scalingType=self.config.flatScalingType,
787  userScale=self.config.flatUserScale,
788  invert=invert
789  )
790 
791  def getIsrExposure(self, dataRef, datasetType, immediate=True):
792  """!Retrieve a calibration dataset for removing instrument signature
793 
794  @param[in] dataRef data reference for exposure
795  @param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat')
796  @param[in] immediate if True, disable butler proxies to enable error
797  handling within this routine
798  @return exposure
799  """
800  try:
801  exp = dataRef.get(datasetType, immediate=immediate)
802  except Exception as exc1:
803  if not self.config.fallbackFilterName:
804  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
805  try:
806  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
807  except Exception as exc2:
808  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
809  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
810  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
811 
812  if self.config.doAssembleIsrExposures:
813  exp = self.assembleCcd.assembleCcd(exp)
814  return exp
815 
816  def saturationDetection(self, exposure, amp):
817  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place
818 
819  @param[in,out] exposure exposure to process; only the amp DataSec is processed
820  @param[in] amp amplifier device data
821  """
822  if not math.isnan(amp.getSaturation()):
823  maskedImage = exposure.getMaskedImage()
824  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
825  isrFunctions.makeThresholdMask(
826  maskedImage=dataView,
827  threshold=amp.getSaturation(),
828  growFootprints=0,
829  maskName=self.config.saturatedMaskName,
830  )
831 
832  def saturationInterpolation(self, ccdExposure):
833  """!Interpolate over saturated pixels, in place
834 
835  @param[in,out] ccdExposure exposure to process
836 
837  @warning:
838  - Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask.
839  - Call this after CCD assembly, since saturated regions may cross amplifier boundaries
840  """
841  isrFunctions.interpolateFromMask(
842  maskedImage=ccdExposure.getMaskedImage(),
843  fwhm=self.config.fwhm,
844  growFootprints=self.config.growSaturationFootprintSize,
845  maskName=self.config.saturatedMaskName,
846  )
847 
848  def suspectDetection(self, exposure, amp):
849  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place
850 
851  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
852  This is intended to indicate pixels that may be affected by unknown systematics;
853  for example if non-linearity corrections above a certain level are unstable
854  then that would be a useful value for suspectLevel. A value of `nan` indicates
855  that no such level exists and no pixels are to be masked as suspicious.
856 
857  @param[in,out] exposure exposure to process; only the amp DataSec is processed
858  @param[in] amp amplifier device data
859  """
860  suspectLevel = amp.getSuspectLevel()
861  if math.isnan(suspectLevel):
862  return
863 
864  maskedImage = exposure.getMaskedImage()
865  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
866  isrFunctions.makeThresholdMask(
867  maskedImage=dataView,
868  threshold=suspectLevel,
869  growFootprints=0,
870  maskName=self.config.suspectMaskName,
871  )
872 
873  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
874  """!Mask defects using mask plane "BAD" and interpolate over them, in place
875 
876  @param[in,out] ccdExposure exposure to process
877  @param[in] defectBaseList a list of defects to mask and interpolate
878 
879  @warning: call this after CCD assembly, since defects may cross amplifier boundaries
880  """
881  maskedImage = ccdExposure.getMaskedImage()
882  defectList = []
883  for d in defectBaseList:
884  bbox = d.getBBox()
885  nd = measAlg.Defect(bbox)
886  defectList.append(nd)
887  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
888  isrFunctions.interpolateDefectList(
889  maskedImage=maskedImage,
890  defectList=defectList,
891  fwhm=self.config.fwhm,
892  )
893 
894  def maskAndInterpNan(self, exposure):
895  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place
896 
897  We mask and interpolate over all NaNs, including those
898  that are masked with other bits (because those may or may
899  not be interpolated over later, and we want to remove all
900  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
901  is used to preserve the historical name.
902 
903  @param[in,out] exposure exposure to process
904  """
905  maskedImage = exposure.getMaskedImage()
906 
907  # Find and mask NaNs
908  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
909  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
910  numNans = maskNans(maskedImage, maskVal)
911  self.metadata.set("NUMNANS", numNans)
912 
913  # Interpolate over these previously-unmasked NaNs
914  if numNans > 0:
915  self.log.warn("There were %i unmasked NaNs", numNans)
916  nanDefectList = isrFunctions.getDefectListFromMask(
917  maskedImage=maskedImage,
918  maskName='UNMASKEDNAN',
919  )
920  isrFunctions.interpolateDefectList(
921  maskedImage=exposure.getMaskedImage(),
922  defectList=nanDefectList,
923  fwhm=self.config.fwhm,
924  )
925 
926  def overscanCorrection(self, exposure, amp):
927  """Apply overscan correction, in-place
928 
929  Parameters
930  ----------
931  exposure : `lsst.afw.image.Exposure`
932  Exposure to process; must include both data and bias regions.
933  amp : `lsst.afw.table.AmpInfoRecord`
934  Amplifier device data.
935 
936  Results
937  -------
938  result : `lsst.pipe.base.Struct` or `NoneType`
939  `None` if there is no overscan; otherwise, this is a
940  result struct with components:
941 
942  - ``imageFit``: Value(s) removed from image (scalar or
943  `lsst.afw.image.Image`).
944  - ``overscanFit``: Value(s) removed from overscan (scalar or
945  `lsst.afw.image.Image`).
946  - ``overscanImage``: Image of the overscan, post-subtraction
947  (`lsst.afw.image.Image`).
948  """
949  if not amp.getHasRawInfo():
950  raise RuntimeError("This method must be executed on an amp with raw information.")
951 
952  if amp.getRawHorizontalOverscanBBox().isEmpty():
953  self.log.info("No Overscan region. Not performing Overscan Correction.")
954  return None
955 
956  oscanBBox = amp.getRawHorizontalOverscanBBox()
957 
958  # afw.cameraGeom.assembleImage.makeUpdatedDetector doesn't update readoutCorner; DM-15559
959  x0, x1 = oscanBBox.getBeginX(), oscanBBox.getEndX()
960 
961  prescanBBox = amp.getRawPrescanBBox()
962  if oscanBBox.getBeginX() > prescanBBox.getBeginX(): # amp is at the right
963  x0 += self.config.overscanNumLeadingColumnsToSkip
964  x1 -= self.config.overscanNumTrailingColumnsToSkip
965  else:
966  x0 += self.config.overscanNumTrailingColumnsToSkip
967  x1 -= self.config.overscanNumLeadingColumnsToSkip
968 
969  oscanBBox = afwGeom.BoxI(afwGeom.PointI(x0, oscanBBox.getBeginY()),
970  afwGeom.PointI(x1 - 1, oscanBBox.getEndY() - 1))
971 
972  maskedImage = exposure.maskedImage
973  dataView = maskedImage[amp.getRawDataBBox()]
974  overscanImage = maskedImage[oscanBBox]
975 
976  results = isrFunctions.overscanCorrection(
977  ampMaskedImage=dataView,
978  overscanImage=overscanImage,
979  fitType=self.config.overscanFitType,
980  order=self.config.overscanOrder,
981  collapseRej=self.config.overscanRej,
982  )
983  results.overscanImage = overscanImage
984  return results
985 
986  def addDistortionModel(self, exposure, camera):
987  """!Update the WCS in exposure with a distortion model based on camera geometry
988 
989  Add a model for optical distortion based on geometry found in `camera`
990  and the `exposure`'s detector. The raw input exposure is assumed
991  have a TAN WCS that has no compensation for optical distortion.
992  Two other possibilities are:
993  - The raw input exposure already has a model for optical distortion,
994  as is the case for raw DECam data.
995  In that case you should set config.doAddDistortionModel False.
996  - The raw input exposure has a model for distortion, but it has known
997  deficiencies severe enough to be worth fixing (e.g. because they
998  cause problems for fitting a better WCS). In that case you should
999  override this method with a version suitable for your raw data.
1000 
1001  @param[in,out] exposure exposure to process; must include a Detector and a WCS;
1002  the WCS of the exposure is modified in place
1003  @param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera
1004  """
1005  self.log.info("Adding a distortion model to the WCS")
1006  wcs = exposure.getWcs()
1007  if wcs is None:
1008  raise RuntimeError("exposure has no WCS")
1009  if camera is None:
1010  raise RuntimeError("camera is None")
1011  detector = exposure.getDetector()
1012  if detector is None:
1013  raise RuntimeError("exposure has no Detector")
1014  pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
1015  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
1016  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
1017  exposure.setWcs(distortedWcs)
1018 
1019  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
1020  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners
1021 
1022  @param[in,out] ccdExposure exposure to process
1023  @param[in] fpPolygon Polygon in focal plane coordinates
1024  """
1025  # Get ccd corners in focal plane coordinates
1026  ccd = ccdExposure.getDetector()
1027  fpCorners = ccd.getCorners(FOCAL_PLANE)
1028  ccdPolygon = Polygon(fpCorners)
1029 
1030  # Get intersection of ccd corners with fpPolygon
1031  intersect = ccdPolygon.intersectionSingle(fpPolygon)
1032 
1033  # Transform back to pixel positions and build new polygon
1034  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
1035  validPolygon = Polygon(ccdPoints)
1036  ccdExposure.getInfo().setValidPolygon(validPolygon)
1037 
1038  def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain):
1039  """Apply brighter fatter correction in place for the image
1040 
1041  This correction takes a kernel that has been derived from flat field images to
1042  redistribute the charge. The gradient of the kernel is the deflection
1043  field due to the accumulated charge.
1044 
1045  Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x)
1046  using the following equation:
1047 
1048  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
1049 
1050  To evaluate the derivative term we expand it as follows:
1051 
1052  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
1053 
1054  Because we use the measured counts instead of the incident counts we apply the correction
1055  iteratively to reconstruct the original counts and the correction. We stop iterating when the
1056  summed difference between the current corrected image and the one from the previous iteration
1057  is below the threshold. We do not require convergence because the number of iterations is
1058  too large a computational cost. How we define the threshold still needs to be evaluated, the
1059  current default was shown to work reasonably well on a small set of images. For more information
1060  on the method see DocuShare Document-19407.
1061 
1062  The edges as defined by the kernel are not corrected because they have spurious values
1063  due to the convolution.
1064  """
1065  self.log.info("Applying brighter fatter correction")
1066 
1067  image = exposure.getMaskedImage().getImage()
1068 
1069  # The image needs to be units of electrons/holes
1070  with self.gainContext(exposure, image, applyGain):
1071 
1072  kLx = numpy.shape(kernel)[0]
1073  kLy = numpy.shape(kernel)[1]
1074  kernelImage = afwImage.ImageD(kLx, kLy)
1075  kernelImage.getArray()[:, :] = kernel
1076  tempImage = image.clone()
1077 
1078  nanIndex = numpy.isnan(tempImage.getArray())
1079  tempImage.getArray()[nanIndex] = 0.
1080 
1081  outImage = afwImage.ImageF(image.getDimensions())
1082  corr = numpy.zeros_like(image.getArray())
1083  prev_image = numpy.zeros_like(image.getArray())
1084  convCntrl = afwMath.ConvolutionControl(False, True, 1)
1085  fixedKernel = afwMath.FixedKernel(kernelImage)
1086 
1087  # Define boundary by convolution region. The region that the correction will be
1088  # calculated for is one fewer in each dimension because of the second derivative terms.
1089  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
1090  startX = kLx//2
1091  endX = -kLx//2
1092  startY = kLy//2
1093  endY = -kLy//2
1094 
1095  for iteration in range(maxIter):
1096 
1097  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
1098  tmpArray = tempImage.getArray()
1099  outArray = outImage.getArray()
1100 
1101  with numpy.errstate(invalid="ignore", over="ignore"):
1102  # First derivative term
1103  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
1104  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
1105  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
1106 
1107  # Second derivative term
1108  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
1109  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
1110  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
1111 
1112  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
1113 
1114  tmpArray[:, :] = image.getArray()[:, :]
1115  tmpArray[nanIndex] = 0.
1116  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
1117 
1118  if iteration > 0:
1119  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
1120 
1121  if diff < threshold:
1122  break
1123  prev_image[:, :] = tmpArray[:, :]
1124 
1125  if iteration == maxIter - 1:
1126  self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff)
1127 
1128  self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
1129  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
1130  corr[startY + 1:endY - 1, startX + 1:endX - 1]
1131 
1132  def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None,
1133  sensorTransmission=None, atmosphereTransmission=None):
1134  """Attach a TransmissionCurve to an Exposure, given separate curves for
1135  different components.
1136 
1137  Parameters
1138  ----------
1139  exposure : `lsst.afw.image.Exposure`
1140  Exposure object to modify by attaching the product of all given
1141  ``TransmissionCurves`` in post-assembly trimmed detector
1142  coordinates. Must have a valid ``Detector`` attached that matches
1143  the detector associated with sensorTransmission.
1144  opticsTransmission : `lsst.afw.image.TransmissionCurve`
1145  A ``TransmissionCurve`` that represents the throughput of the
1146  optics, to be evaluated in focal-plane coordinates.
1147  filterTransmission : `lsst.afw.image.TransmissionCurve`
1148  A ``TransmissionCurve`` that represents the throughput of the
1149  filter itself, to be evaluated in focal-plane coordinates.
1150  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1151  A ``TransmissionCurve`` that represents the throughput of the
1152  sensor itself, to be evaluated in post-assembly trimmed detector
1153  coordinates.
1154  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1155  A ``TransmissionCurve`` that represents the throughput of the
1156  atmosphere, assumed to be spatially constant.
1157 
1158  All ``TransmissionCurve`` arguments are optional; if none are provided,
1159  the attached ``TransmissionCurve`` will have unit transmission
1160  everywhere.
1161 
1162  Returns
1163  -------
1164  combined : ``lsst.afw.image.TransmissionCurve``
1165  The TransmissionCurve attached to the exposure.
1166  """
1167  return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission,
1168  filterTransmission=filterTransmission,
1169  sensorTransmission=sensorTransmission,
1170  atmosphereTransmission=atmosphereTransmission)
1171 
1172  @contextmanager
1173  def gainContext(self, exp, image, apply):
1174  """Context manager that applies and removes gain
1175  """
1176  if apply:
1177  ccd = exp.getDetector()
1178  for amp in ccd:
1179  sim = image.Factory(image, amp.getBBox())
1180  sim *= amp.getGain()
1181 
1182  try:
1183  yield exp
1184  finally:
1185  if apply:
1186  ccd = exp.getDetector()
1187  for amp in ccd:
1188  sim = image.Factory(image, amp.getBBox())
1189  sim /= amp.getGain()
1190 
1191  @contextmanager
1192  def flatContext(self, exp, flat, dark=None):
1193  """Context manager that applies and removes flats and darks,
1194  if the task is configured to apply them.
1195  """
1196  if self.config.doDark and dark is not None:
1197  self.darkCorrection(exp, dark)
1198  if self.config.doFlat:
1199  self.flatCorrection(exp, flat)
1200  try:
1201  yield exp
1202  finally:
1203  if self.config.doFlat:
1204  self.flatCorrection(exp, flat, invert=True)
1205  if self.config.doDark and dark is not None:
1206  self.darkCorrection(exp, dark, invert=True)
1207 
1208 
1209 class FakeAmp(object):
1210  """A Detector-like object that supports returning gain and saturation level"""
1211 
1212  def __init__(self, exposure, config):
1213  self._bbox = exposure.getBBox(afwImage.LOCAL)
1214  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
1215  self._gain = config.gain
1216  self._readNoise = config.readNoise
1217  self._saturation = config.saturation
1218 
1219  def getBBox(self):
1220  return self._bbox
1221 
1222  def getRawBBox(self):
1223  return self._bbox
1224 
1225  def getHasRawInfo(self):
1226  return True # but see getRawHorizontalOverscanBBox()
1227 
1229  return self._RawHorizontalOverscanBBox
1230 
1231  def getGain(self):
1232  return self._gain
1233 
1234  def getReadNoise(self):
1235  return self._readNoise
1236 
1237  def getSaturation(self):
1238  return self._saturation
1239 
1240  def getSuspectLevel(self):
1241  return float("NaN")
1242 
1243 
1244 class RunIsrConfig(pexConfig.Config):
1245  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
1246 
1247 
1253 
1254 
1255 class RunIsrTask(pipeBase.CmdLineTask):
1256  """Task to wrap the default IsrTask to allow it to be retargeted.
1257 
1258  The standard IsrTask can be called directly from a command line
1259  program, but doing so removes the ability of the task to be
1260  retargeted. As most cameras override some set of the IsrTask
1261  methods, this would remove those data-specific methods in the
1262  output post-ISR images. This wrapping class fixes the issue,
1263  allowing identical post-ISR images to be generated by both the
1264  processCcd and isrTask code.
1265  """
1266  ConfigClass = RunIsrConfig
1267  _DefaultName = "runIsr"
1268 
1269  def __init__(self, *args, **kwargs):
1270  super().__init__(*args, **kwargs)
1271  self.makeSubtask("isr")
1272 
1273  def runDataRef(self, dataRef):
1274  """
1275  Parameters
1276  ----------
1277  dataRef : `lsst.daf.persistence.ButlerDataRef`
1278  data reference of the detector data to be processed
1279 
1280  Returns
1281  -------
1282  result : `pipeBase.Struct`
1283  Result struct with component:
1284 
1285  - exposure : `lsst.afw.image.Exposure`
1286  Post-ISR processed exposure.
1287  """
1288  return self.isr.runDataRef(dataRef)
def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain)
Definition: isrTask.py:1038
def runDataRef(self, sensorRef)
Definition: isrTask.py:649
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, crosstalkSources=None)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:488
def gainContext(self, exp, image, apply)
Definition: isrTask.py:1173
def __init__(self, args, kwargs)
Constructor for IsrTask.
Definition: isrTask.py:402
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:414
def runDataRef(self, dataRef)
Definition: isrTask.py:1273
def __init__(self, args, kwargs)
Definition: isrTask.py:1269
def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Definition: isrTask.py:1133
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:894
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:832
Apply common instrument signature correction algorithms to a raw frame.
Definition: isrTask.py:328
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:1228
def convertIntToFloat(self, exposure)
Definition: isrTask.py:681
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:776
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:791
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:706
def doLinearize(self, detector)
Is linearization wanted for this detector?
Definition: isrTask.py:727
def addDistortionModel(self, exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
Definition: isrTask.py:986
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:1019
def biasCorrection(self, exposure, biasExposure)
Apply bias correction in place.
Definition: isrTask.py:698
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:1192
def overscanCorrection(self, exposure, amp)
Definition: isrTask.py:926
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:737
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:848
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:873
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:816
def __init__(self, exposure, config)
Definition: isrTask.py:1212