lsst.ip.isr  19.0.0-23-g4cc2bcb
overscan.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import numpy as np
23 import lsst.afw.math as afwMath
24 import lsst.afw.image as afwImage
25 import lsst.pipe.base as pipeBase
26 import lsst.pex.config as pexConfig
27 
28 __all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"]
29 
30 
31 class OverscanCorrectionTaskConfig(pexConfig.Config):
32  """Overscan correction options.
33  """
34  fitType = pexConfig.ChoiceField(
35  dtype=str,
36  doc="The method for fitting the overscan bias level.",
37  default='MEDIAN',
38  allowed={
39  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
40  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
41  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
42  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
43  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
44  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
45  "MEAN": "Correct using the mean of the overscan region",
46  "MEANCLIP": "Correct using a clipped mean of the overscan region",
47  "MEDIAN": "Correct using the median of the overscan region",
48  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
49  },
50  )
51  order = pexConfig.Field(
52  dtype=int,
53  doc=("Order of polynomial to fit if overscan fit type is a polynomial, " +
54  "or number of spline knots if overscan fit type is a spline."),
55  default=1,
56  )
57  numSigmaClip = pexConfig.Field(
58  dtype=float,
59  doc="Rejection threshold (sigma) for collapsing overscan before fit",
60  default=3.0,
61  )
62  overscanIsInt = pexConfig.Field(
63  dtype=bool,
64  doc="Treat overscan as an integer image for purposes of fitType=MEDIAN" +
65  " and fitType=MEDIAN_PER_ROW.",
66  default=True,
67  )
68 
69 
70 class OverscanCorrectionTask(pipeBase.Task):
71  """Correction task for overscan.
72 
73  This class contains a number of utilities that are easier to
74  understand and use when they are not embedded in nested if/else
75  loops.
76 
77  Parameters
78  ----------
79  statControl : `lsst.afw.math.StatisticsControl`, optional
80  Statistics control object.
81  """
82  ConfigClass = OverscanCorrectionTaskConfig
83  _DefaultName = "overscan"
84 
85  def __init__(self, statControl=None, **kwargs):
86  super().__init__(**kwargs)
87  if statControl:
88  self.statControl = statControl
89  else:
90  self.statControl = afwMath.StatisticsControl()
91  self.statControl.setNumSigmaClip(self.config.numSigmaClip)
92 
93  def run(self, ampImage, overscanImage):
94  """Measure and remove an overscan from an amplifier image.
95 
96  Parameters
97  ----------
98  ampImage : `lsst.afw.image.Image`
99  Image data that will have the overscan removed.
100  overscanImage : `lsst.afw.image.Image`
101  Overscan data that the overscan is measured from.
102 
103  Returns
104  -------
105  overscanResults : `lsst.pipe.base.Struct`
106  Result struct with components:
107 
108  ``imageFit``
109  Value or fit subtracted from the amplifier image data
110  (scalar or `lsst.afw.image.Image`).
111  ``overscanFit``
112  Value or fit subtracted from the overscan image data
113  (scalar or `lsst.afw.image.Image`).
114  ``overscanImage``
115  Image of the overscan region with the overscan
116  correction applied (`lsst.afw.image.Image`). This
117  quantity is used to estimate the amplifier read noise
118  empirically.
119 
120  Raises
121  ------
122  RuntimeError
123  Raised if an invalid overscan type is set.
124 
125  """
126  if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'):
127  overscanResult = self.measureConstantOverscan(overscanImage)
128  overscanValue = overscanResult.overscanValue
129  offImage = overscanValue
130  overscanModel = overscanValue
131  maskSuspect = None
132  elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG',
133  'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
134  overscanResult = self.measureVectorOverscan(overscanImage)
135  overscanValue = overscanResult.overscanValue
136  maskArray = overscanResult.maskArray
137  isTransposed = overscanResult.isTransposed
138 
139  offImage = afwImage.ImageF(ampImage.getDimensions())
140  offArray = offImage.getArray()
141  overscanModel = afwImage.ImageF(overscanImage.getDimensions())
142  overscanArray = overscanModel.getArray()
143 
144  if hasattr(ampImage, 'getMask'):
145  maskSuspect = afwImage.Mask(ampImage.getDimensions())
146  else:
147  maskSuspect = None
148 
149  if isTransposed:
150  offArray[:, :] = overscanValue[np.newaxis, :]
151  overscanArray[:, :] = overscanValue[np.newaxis, :]
152  if maskSuspect:
153  maskSuspect.getArray()[:, maskArray] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
154  else:
155  offArray[:, :] = overscanValue[:, np.newaxis]
156  overscanArray[:, :] = overscanValue[:, np.newaxis]
157  if maskSuspect:
158  maskSuspect.getArray()[maskArray, :] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
159  else:
160  raise RuntimeError('%s : %s an invalid overscan type' %
161  ("overscanCorrection", self.config.fitType))
162 
163  self.debugView(overscanImage, overscanValue)
164 
165  ampImage -= offImage
166  if maskSuspect:
167  ampImage.getMask().getArray()[:, :] |= maskSuspect.getArray()[:, :]
168  overscanImage -= overscanModel
169  return pipeBase.Struct(imageFit=offImage,
170  overscanFit=overscanModel,
171  overscanImage=overscanImage,
172  edgeMask=maskSuspect)
173 
174  @staticmethod
175  def integerConvert(image):
176  """Return an integer version of the input image.
177 
178  Parameters
179  ----------
180  image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
181  Image to convert to integers.
182 
183  Returns
184  -------
185  outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
186  The integer converted image.
187 
188  Raises
189  ------
190  RuntimeError
191  Raised if the input image could not be converted.
192  """
193  if hasattr(image, "image"):
194  # Is a maskedImage:
195  imageI = image.image.convertI()
196  outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
197  elif hasattr(image, "convertI"):
198  # Is an Image:
199  outI = image.convertI()
200  elif hasattr(image, "astype"):
201  # Is a numpy array:
202  outI = image.astype(int)
203  else:
204  raise RuntimeError("Could not convert this to integers: %s %s %s",
205  image, type(image), dir(image))
206  return outI
207 
208  # Constant methods
209  def measureConstantOverscan(self, image):
210  """Measure a constant overscan value.
211 
212  Parameters
213  ----------
214  image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
215  Image data to measure the overscan from.
216 
217  Returns
218  -------
219  results : `lsst.pipe.base.Struct`
220  Overscan result with entries:
221  - ``overscanValue``: Overscan value to subtract (`float`)
222  - ``maskArray``: Placeholder for a mask array (`list`)
223  - ``isTransposed``: Orientation of the overscan (`bool`)
224  """
225  if self.config.fitType == 'MEDIAN':
226  calcImage = self.integerConvert(image)
227  else:
228  calcImage = image
229 
230  fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
231  overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControl).getValue()
232 
233  return pipeBase.Struct(overscanValue=overscanValue,
234  maskArray=None,
235  isTransposed=False)
236 
237  # Vector correction utilities
238  @staticmethod
239  def getImageArray(image):
240  """Extract the numpy array from the input image.
241 
242  Parameters
243  ----------
244  image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
245  Image data to pull array from.
246 
247  calcImage : `numpy.ndarray`
248  Image data array for numpy operating.
249  """
250  if hasattr(image, "getImage"):
251  calcImage = image.getImage().getArray()
252  else:
253  calcImage = image.getArray()
254  return calcImage
255 
256  @staticmethod
257  def transpose(imageArray):
258  """Transpose input numpy array if necessary.
259 
260  Parameters
261  ----------
262  imageArray : `numpy.ndarray`
263  Image data to transpose.
264 
265  Returns
266  -------
267  imageArray : `numpy.ndarray`
268  Transposed image data.
269  isTransposed : `bool`
270  Indicates whether the input data was transposed.
271  """
272  if np.argmin(imageArray.shape) == 0:
273  return np.transpose(imageArray), True
274  else:
275  return imageArray, False
276 
277  def maskOutliers(self, imageArray):
278  """Mask outliers in a row of overscan data from a robust sigma
279  clipping procedure.
280 
281  Parameters
282  ----------
283  imageArray : `numpy.ndarray`
284  Image to filter along numpy axis=1.
285 
286  Returns
287  -------
288  maskedArray : `numpy.ma.masked_array`
289  Masked image marking outliers.
290  """
291  lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
292  axisMedians = median
293  axisStdev = 0.74*(uq - lq) # robust stdev
294 
295  diff = np.abs(imageArray - axisMedians[:, np.newaxis])
296  return np.ma.masked_where(diff > self.statControl.getNumSigmaClip() *
297  axisStdev[:, np.newaxis], imageArray)
298 
299  @staticmethod
300  def collapseArray(maskedArray):
301  """Collapse overscan array (and mask) to a 1-D vector of values.
302 
303  Parameters
304  ----------
305  maskedArray : `numpy.ma.masked_array`
306  Masked array of input overscan data.
307 
308  Returns
309  -------
310  collapsed : `numpy.ma.masked_array`
311  Single dimensional overscan data, combined with the mean.
312  """
313  collapsed = np.mean(maskedArray, axis=1)
314  if collapsed.mask.sum() > 0:
315  collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
316  return collapsed
317 
318  def collapseArrayMedian(self, maskedArray):
319  """Collapse overscan array (and mask) to a 1-D vector of using the
320  correct integer median of row-values.
321 
322  Parameters
323  ----------
324  maskedArray : `numpy.ma.masked_array`
325  Masked array of input overscan data.
326 
327  Returns
328  -------
329  collapsed : `numpy.ma.masked_array`
330  Single dimensional overscan data, combined with the afwMath median.
331  """
332  integerMI = self.integerConvert(maskedArray)
333 
334  collapsed = []
335  fitType = afwMath.stringToStatisticsProperty('MEDIAN')
336  for row in integerMI:
337  rowMedian = afwMath.makeStatistics(row, fitType, self.statControl).getValue()
338  collapsed.append(rowMedian)
339 
340  return np.array(collapsed)
341 
342  def splineFit(self, indices, collapsed, numBins):
343  """Wrapper function to match spline fit API to polynomial fit API.
344 
345  Parameters
346  ----------
347  indices : `numpy.ndarray`
348  Locations to evaluate the spline.
349  collapsed : `numpy.ndarray`
350  Collapsed overscan values corresponding to the spline
351  evaluation points.
352  numBins : `int`
353  Number of bins to use in constructing the spline.
354 
355  Returns
356  -------
357  interp : `lsst.afw.math.Interpolate`
358  Interpolation object for later evaluation.
359  """
360  if not np.ma.is_masked(collapsed):
361  collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
362 
363  numPerBin, binEdges = np.histogram(indices, bins=numBins,
364  weights=1 - collapsed.mask.astype(int))
365  with np.errstate(invalid="ignore"):
366  values = np.histogram(indices, bins=numBins,
367  weights=collapsed.data*~collapsed.mask)[0]/numPerBin
368  binCenters = np.histogram(indices, bins=numBins,
369  weights=indices*~collapsed.mask)[0]/numPerBin
370  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
371  values.astype(float)[numPerBin > 0],
372  afwMath.stringToInterpStyle(self.config.fitType))
373  return interp
374 
375  @staticmethod
376  def splineEval(indices, interp):
377  """Wrapper function to match spline evaluation API to polynomial fit API.
378 
379  Parameters
380  ----------
381  indices : `numpy.ndarray`
382  Locations to evaluate the spline.
383  interp : `lsst.afw.math.interpolate`
384  Interpolation object to use.
385 
386  Returns
387  -------
388  values : `numpy.ndarray`
389  Evaluated spline values at each index.
390  """
391 
392  return interp.interpolate(indices.astype(float))
393 
394  @staticmethod
395  def maskExtrapolated(collapsed):
396  """Create mask if edges are extrapolated.
397 
398  Parameters
399  ----------
400  collapsed : `numpy.ma.masked_array`
401  Masked array to check the edges of.
402 
403  Returns
404  -------
405  maskArray : `numpy.ndarray`
406  Boolean numpy array of pixels to mask.
407  """
408  maskArray = np.full_like(collapsed, False, dtype=bool)
409  if np.ma.is_masked(collapsed):
410  num = len(collapsed)
411  for low in range(num):
412  if not collapsed.mask[low]:
413  break
414  if low > 0:
415  maskArray[:low] = True
416  for high in range(1, num):
417  if not collapsed.mask[-high]:
418  break
419  if high > 1:
420  maskArray[-high:] = True
421  return maskArray
422 
423  def measureVectorOverscan(self, image):
424  """Calculate the 1-d vector overscan from the input overscan image.
425 
426  Parameters
427  ----------
428  image : `lsst.afw.image.MaskedImage`
429  Image containing the overscan data.
430 
431  Returns
432  -------
433  results : `lsst.pipe.base.Struct`
434  Overscan result with entries:
435  - ``overscanValue``: Overscan value to subtract (`float`)
436  - ``maskArray`` : `list` [ `bool` ]
437  List of rows that should be masked as ``SUSPECT`` when the
438  overscan solution is applied.
439  - ``isTransposed`` : `bool`
440  Indicates if the overscan data was transposed during
441  calcuation, noting along which axis the overscan should be
442  subtracted.
443  """
444  calcImage = self.getImageArray(image)
445 
446  # operate on numpy-arrays from here
447  calcImage, isTransposed = self.transpose(calcImage)
448  masked = self.maskOutliers(calcImage)
449 
450  if self.config.fitType == 'MEDIAN_PER_ROW':
451  overscanVector = self.collapseArrayMedian(masked)
452  maskArray = self.maskExtrapolated(overscanVector)
453  else:
454  collapsed = self.collapseArray(masked)
455 
456  num = len(collapsed)
457  indices = 2.0*np.arange(num)/float(num) - 1.0
458 
459  poly = np.polynomial
460  fitter, evaler = {
461  'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
462  'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
463  'LEG': (poly.legendre.legfit, poly.legendre.legval),
464  'NATURAL_SPLINE': (self.splineFit, self.splineEval),
465  'CUBIC_SPLINE': (self.splineFit, self.splineEval),
466  'AKIMA_SPLINE': (self.splineFit, self.splineEval)
467  }[self.config.fitType]
468 
469  coeffs = fitter(indices, collapsed, self.config.order)
470  overscanVector = evaler(indices, coeffs)
471  maskArray = self.maskExtrapolated(collapsed)
472  return pipeBase.Struct(overscanValue=np.array(overscanVector),
473  maskArray=maskArray,
474  isTransposed=isTransposed)
475 
476  def debugView(self, image, model):
477  """Debug display for the final overscan solution.
478 
479  Parameters
480  ----------
481  image : `lsst.afw.image.Image`
482  Input image the overscan solution was determined from.
483  model : `numpy.ndarray` or `float`
484  Overscan model determined for the image.
485  """
486  import lsstDebug
487  if not lsstDebug.Info(__name__).display:
488  return
489 
490  calcImage = self.getImageArray(image)
491  calcImage, isTransposed = self.transpose(calcImage)
492  masked = self.maskOutliers(calcImage)
493  collapsed = self.collapseArray(masked)
494 
495  num = len(collapsed)
496  indices = 2.0 * np.arange(num)/float(num) - 1.0
497 
498  if np.ma.is_masked(collapsed):
499  collapsedMask = collapsed.mask
500  else:
501  collapsedMask = np.array(num*[np.ma.nomask])
502 
503  import matplotlib.pyplot as plot
504  figure = plot.figure(1)
505  figure.clear()
506  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
507  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
508  if collapsedMask.sum() > 0:
509  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
510  if isinstance(model, np.ndarray):
511  plotModel = model
512  else:
513  plotModel = np.zeros_like(indices)
514  plotModel += model
515  axes.plot(indices, plotModel, 'r-')
516  plot.xlabel("centered/scaled position along overscan region")
517  plot.ylabel("pixel value/fit value")
518  figure.show()
519  prompt = "Press Enter or c to continue [chp]..."
520  while True:
521  ans = input(prompt).lower()
522  if ans in ("", " ", "c",):
523  break
524  elif ans in ("p", ):
525  import pdb
526  pdb.set_trace()
527  elif ans in ("h", ):
528  print("[h]elp [c]ontinue [p]db")
529  plot.close()
lsst::ip::isr.overscan.OverscanCorrectionTask
Definition: overscan.py:70
lsst::afw::image
lsst::ip::isr.overscan.OverscanCorrectionTaskConfig
Definition: overscan.py:31
lsst::ip::isr.overscan.OverscanCorrectionTask.transpose
def transpose(imageArray)
Definition: overscan.py:257
lsst::ip::isr.overscan.OverscanCorrectionTask.collapseArray
def collapseArray(maskedArray)
Definition: overscan.py:300
lsst::ip::isr.overscan.OverscanCorrectionTask.integerConvert
def integerConvert(image)
Definition: overscan.py:175
lsst::ip::isr.overscan.OverscanCorrectionTask.maskExtrapolated
def maskExtrapolated(collapsed)
Definition: overscan.py:395
lsst::ip::isr.overscan.OverscanCorrectionTask.collapseArrayMedian
def collapseArrayMedian(self, maskedArray)
Definition: overscan.py:318
lsst::ip::isr.overscan.OverscanCorrectionTask.getImageArray
def getImageArray(image)
Definition: overscan.py:239
lsst::ip::isr.overscan.OverscanCorrectionTask.__init__
def __init__(self, statControl=None, **kwargs)
Definition: overscan.py:85
lsstDebug::Info
lsst::ip::isr.overscan.OverscanCorrectionTask.maskOutliers
def maskOutliers(self, imageArray)
Definition: overscan.py:277
lsst::ip::isr.overscan.OverscanCorrectionTask.statControl
statControl
Definition: overscan.py:88
lsst::ip::isr.overscan.OverscanCorrectionTask.measureVectorOverscan
def measureVectorOverscan(self, image)
Definition: overscan.py:423
lsst::ip::isr.overscan.OverscanCorrectionTask.splineFit
def splineFit(self, indices, collapsed, numBins)
Definition: overscan.py:342
lsst::ip::isr.overscan.OverscanCorrectionTask.run
def run(self, ampImage, overscanImage)
Definition: overscan.py:93
lsst::afw::math
lsst::ip::isr.overscan.OverscanCorrectionTask.splineEval
def splineEval(indices, interp)
Definition: overscan.py:376
lsst::ip::isr.overscan.OverscanCorrectionTask.measureConstantOverscan
def measureConstantOverscan(self, image)
Definition: overscan.py:209
lsst::ip::isr.overscan.OverscanCorrectionTask.debugView
def debugView(self, image, model)
Definition: overscan.py:476
lsst::pipe::base