Coverage for python/lsst/ip/isr/overscan.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23import lsst.afw.math as afwMath
24import lsst.afw.image as afwImage
25import lsst.pipe.base as pipeBase
26import lsst.pex.config as pexConfig
28__all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"]
31class OverscanCorrectionTaskConfig(pexConfig.Config):
32 """Overscan correction options.
33 """
34 fitType = pexConfig.ChoiceField(
35 dtype=str,
36 doc="The method for fitting the overscan bias level.",
37 default='MEDIAN',
38 allowed={
39 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
40 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
41 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
42 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
43 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
44 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
45 "MEAN": "Correct using the mean of the overscan region",
46 "MEANCLIP": "Correct using a clipped mean of the overscan region",
47 "MEDIAN": "Correct using the median of the overscan region",
48 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
49 },
50 )
51 order = pexConfig.Field(
52 dtype=int,
53 doc=("Order of polynomial to fit if overscan fit type is a polynomial, " +
54 "or number of spline knots if overscan fit type is a spline."),
55 default=1,
56 )
57 numSigmaClip = pexConfig.Field(
58 dtype=float,
59 doc="Rejection threshold (sigma) for collapsing overscan before fit",
60 default=3.0,
61 )
62 overscanIsInt = pexConfig.Field(
63 dtype=bool,
64 doc="Treat overscan as an integer image for purposes of fitType=MEDIAN" +
65 " and fitType=MEDIAN_PER_ROW.",
66 default=True,
67 )
70class OverscanCorrectionTask(pipeBase.Task):
71 """Correction task for overscan.
73 This class contains a number of utilities that are easier to
74 understand and use when they are not embedded in nested if/else
75 loops.
77 Parameters
78 ----------
79 statControl : `lsst.afw.math.StatisticsControl`, optional
80 Statistics control object.
81 """
82 ConfigClass = OverscanCorrectionTaskConfig
83 _DefaultName = "overscan"
85 def __init__(self, statControl=None, **kwargs):
86 super().__init__(**kwargs)
87 if statControl:
88 self.statControl = statControl
89 else:
90 self.statControl = afwMath.StatisticsControl()
91 self.statControl.setNumSigmaClip(self.config.numSigmaClip)
93 def run(self, ampImage, overscanImage):
94 """Measure and remove an overscan from an amplifier image.
96 Parameters
97 ----------
98 ampImage : `lsst.afw.image.Image`
99 Image data that will have the overscan removed.
100 overscanImage : `lsst.afw.image.Image`
101 Overscan data that the overscan is measured from.
103 Returns
104 -------
105 overscanResults : `lsst.pipe.base.Struct`
106 Result struct with components:
108 ``imageFit``
109 Value or fit subtracted from the amplifier image data
110 (scalar or `lsst.afw.image.Image`).
111 ``overscanFit``
112 Value or fit subtracted from the overscan image data
113 (scalar or `lsst.afw.image.Image`).
114 ``overscanImage``
115 Image of the overscan region with the overscan
116 correction applied (`lsst.afw.image.Image`). This
117 quantity is used to estimate the amplifier read noise
118 empirically.
120 Raises
121 ------
122 RuntimeError
123 Raised if an invalid overscan type is set.
125 """
126 if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'):
127 overscanResult = self.measureConstantOverscan(overscanImage)
128 overscanValue = overscanResult.overscanValue
129 offImage = overscanValue
130 overscanModel = overscanValue
131 maskSuspect = None
132 elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG',
133 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
134 overscanResult = self.measureVectorOverscan(overscanImage)
135 overscanValue = overscanResult.overscanValue
136 maskArray = overscanResult.maskArray
137 isTransposed = overscanResult.isTransposed
139 offImage = afwImage.ImageF(ampImage.getDimensions())
140 offArray = offImage.getArray()
141 overscanModel = afwImage.ImageF(overscanImage.getDimensions())
142 overscanArray = overscanModel.getArray()
144 if hasattr(ampImage, 'getMask'):
145 maskSuspect = afwImage.Mask(ampImage.getDimensions())
146 else:
147 maskSuspect = None
149 if isTransposed:
150 offArray[:, :] = overscanValue[np.newaxis, :]
151 overscanArray[:, :] = overscanValue[np.newaxis, :]
152 if maskSuspect:
153 maskSuspect.getArray()[:, maskArray] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
154 else:
155 offArray[:, :] = overscanValue[:, np.newaxis]
156 overscanArray[:, :] = overscanValue[:, np.newaxis]
157 if maskSuspect:
158 maskSuspect.getArray()[maskArray, :] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
159 else:
160 raise RuntimeError('%s : %s an invalid overscan type' %
161 ("overscanCorrection", self.config.fitType))
163 self.debugView(overscanImage, overscanValue)
165 ampImage -= offImage
166 if maskSuspect:
167 ampImage.getMask().getArray()[:, :] |= maskSuspect.getArray()[:, :]
168 overscanImage -= overscanModel
169 return pipeBase.Struct(imageFit=offImage,
170 overscanFit=overscanModel,
171 overscanImage=overscanImage,
172 edgeMask=maskSuspect)
174 @staticmethod
175 def integerConvert(image):
176 """Return an integer version of the input image.
178 Parameters
179 ----------
180 image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
181 Image to convert to integers.
183 Returns
184 -------
185 outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
186 The integer converted image.
188 Raises
189 ------
190 RuntimeError
191 Raised if the input image could not be converted.
192 """
193 if hasattr(image, "image"):
194 # Is a maskedImage:
195 imageI = image.image.convertI()
196 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
197 elif hasattr(image, "convertI"):
198 # Is an Image:
199 outI = image.convertI()
200 elif hasattr(image, "astype"):
201 # Is a numpy array:
202 outI = image.astype(int)
203 else:
204 raise RuntimeError("Could not convert this to integers: %s %s %s",
205 image, type(image), dir(image))
206 return outI
208 # Constant methods
209 def measureConstantOverscan(self, image):
210 """Measure a constant overscan value.
212 Parameters
213 ----------
214 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
215 Image data to measure the overscan from.
217 Returns
218 -------
219 results : `lsst.pipe.base.Struct`
220 Overscan result with entries:
221 - ``overscanValue``: Overscan value to subtract (`float`)
222 - ``maskArray``: Placeholder for a mask array (`list`)
223 - ``isTransposed``: Orientation of the overscan (`bool`)
224 """
225 if self.config.fitType == 'MEDIAN':
226 calcImage = self.integerConvert(image)
227 else:
228 calcImage = image
230 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
231 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControl).getValue()
233 return pipeBase.Struct(overscanValue=overscanValue,
234 maskArray=None,
235 isTransposed=False)
237 # Vector correction utilities
238 @staticmethod
239 def getImageArray(image):
240 """Extract the numpy array from the input image.
242 Parameters
243 ----------
244 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
245 Image data to pull array from.
247 calcImage : `numpy.ndarray`
248 Image data array for numpy operating.
249 """
250 if hasattr(image, "getImage"):
251 calcImage = image.getImage().getArray()
252 else:
253 calcImage = image.getArray()
254 return calcImage
256 @staticmethod
257 def transpose(imageArray):
258 """Transpose input numpy array if necessary.
260 Parameters
261 ----------
262 imageArray : `numpy.ndarray`
263 Image data to transpose.
265 Returns
266 -------
267 imageArray : `numpy.ndarray`
268 Transposed image data.
269 isTransposed : `bool`
270 Indicates whether the input data was transposed.
271 """
272 if np.argmin(imageArray.shape) == 0:
273 return np.transpose(imageArray), True
274 else:
275 return imageArray, False
277 def maskOutliers(self, imageArray):
278 """Mask outliers in a row of overscan data from a robust sigma
279 clipping procedure.
281 Parameters
282 ----------
283 imageArray : `numpy.ndarray`
284 Image to filter along numpy axis=1.
286 Returns
287 -------
288 maskedArray : `numpy.ma.masked_array`
289 Masked image marking outliers.
290 """
291 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
292 axisMedians = median
293 axisStdev = 0.74*(uq - lq) # robust stdev
295 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
296 return np.ma.masked_where(diff > self.statControl.getNumSigmaClip() *
297 axisStdev[:, np.newaxis], imageArray)
299 @staticmethod
300 def collapseArray(maskedArray):
301 """Collapse overscan array (and mask) to a 1-D vector of values.
303 Parameters
304 ----------
305 maskedArray : `numpy.ma.masked_array`
306 Masked array of input overscan data.
308 Returns
309 -------
310 collapsed : `numpy.ma.masked_array`
311 Single dimensional overscan data, combined with the mean.
312 """
313 collapsed = np.mean(maskedArray, axis=1)
314 if collapsed.mask.sum() > 0:
315 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
316 return collapsed
318 def collapseArrayMedian(self, maskedArray):
319 """Collapse overscan array (and mask) to a 1-D vector of using the
320 correct integer median of row-values.
322 Parameters
323 ----------
324 maskedArray : `numpy.ma.masked_array`
325 Masked array of input overscan data.
327 Returns
328 -------
329 collapsed : `numpy.ma.masked_array`
330 Single dimensional overscan data, combined with the afwMath median.
331 """
332 integerMI = self.integerConvert(maskedArray)
334 collapsed = []
335 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
336 for row in integerMI:
337 rowMedian = afwMath.makeStatistics(row, fitType, self.statControl).getValue()
338 collapsed.append(rowMedian)
340 return np.array(collapsed)
342 def splineFit(self, indices, collapsed, numBins):
343 """Wrapper function to match spline fit API to polynomial fit API.
345 Parameters
346 ----------
347 indices : `numpy.ndarray`
348 Locations to evaluate the spline.
349 collapsed : `numpy.ndarray`
350 Collapsed overscan values corresponding to the spline
351 evaluation points.
352 numBins : `int`
353 Number of bins to use in constructing the spline.
355 Returns
356 -------
357 interp : `lsst.afw.math.Interpolate`
358 Interpolation object for later evaluation.
359 """
360 if not np.ma.is_masked(collapsed):
361 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
363 numPerBin, binEdges = np.histogram(indices, bins=numBins,
364 weights=1 - collapsed.mask.astype(int))
365 with np.errstate(invalid="ignore"):
366 values = np.histogram(indices, bins=numBins,
367 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
368 binCenters = np.histogram(indices, bins=numBins,
369 weights=indices*~collapsed.mask)[0]/numPerBin
370 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
371 values.astype(float)[numPerBin > 0],
372 afwMath.stringToInterpStyle(self.config.fitType))
373 return interp
375 @staticmethod
376 def splineEval(indices, interp):
377 """Wrapper function to match spline evaluation API to polynomial fit API.
379 Parameters
380 ----------
381 indices : `numpy.ndarray`
382 Locations to evaluate the spline.
383 interp : `lsst.afw.math.interpolate`
384 Interpolation object to use.
386 Returns
387 -------
388 values : `numpy.ndarray`
389 Evaluated spline values at each index.
390 """
392 return interp.interpolate(indices.astype(float))
394 @staticmethod
395 def maskExtrapolated(collapsed):
396 """Create mask if edges are extrapolated.
398 Parameters
399 ----------
400 collapsed : `numpy.ma.masked_array`
401 Masked array to check the edges of.
403 Returns
404 -------
405 maskArray : `numpy.ndarray`
406 Boolean numpy array of pixels to mask.
407 """
408 maskArray = np.full_like(collapsed, False, dtype=bool)
409 if np.ma.is_masked(collapsed):
410 num = len(collapsed)
411 for low in range(num):
412 if not collapsed.mask[low]:
413 break
414 if low > 0:
415 maskArray[:low] = True
416 for high in range(1, num):
417 if not collapsed.mask[-high]:
418 break
419 if high > 1:
420 maskArray[-high:] = True
421 return maskArray
423 def measureVectorOverscan(self, image):
424 """Calculate the 1-d vector overscan from the input overscan image.
426 Parameters
427 ----------
428 image : `lsst.afw.image.MaskedImage`
429 Image containing the overscan data.
431 Returns
432 -------
433 results : `lsst.pipe.base.Struct`
434 Overscan result with entries:
435 - ``overscanValue``: Overscan value to subtract (`float`)
436 - ``maskArray`` : `list` [ `bool` ]
437 List of rows that should be masked as ``SUSPECT`` when the
438 overscan solution is applied.
439 - ``isTransposed`` : `bool`
440 Indicates if the overscan data was transposed during
441 calcuation, noting along which axis the overscan should be
442 subtracted.
443 """
444 calcImage = self.getImageArray(image)
446 # operate on numpy-arrays from here
447 calcImage, isTransposed = self.transpose(calcImage)
448 masked = self.maskOutliers(calcImage)
450 if self.config.fitType == 'MEDIAN_PER_ROW':
451 overscanVector = self.collapseArrayMedian(masked)
452 maskArray = self.maskExtrapolated(overscanVector)
453 else:
454 collapsed = self.collapseArray(masked)
456 num = len(collapsed)
457 indices = 2.0*np.arange(num)/float(num) - 1.0
459 poly = np.polynomial
460 fitter, evaler = {
461 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
462 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
463 'LEG': (poly.legendre.legfit, poly.legendre.legval),
464 'NATURAL_SPLINE': (self.splineFit, self.splineEval),
465 'CUBIC_SPLINE': (self.splineFit, self.splineEval),
466 'AKIMA_SPLINE': (self.splineFit, self.splineEval)
467 }[self.config.fitType]
469 coeffs = fitter(indices, collapsed, self.config.order)
470 overscanVector = evaler(indices, coeffs)
471 maskArray = self.maskExtrapolated(collapsed)
472 return pipeBase.Struct(overscanValue=np.array(overscanVector),
473 maskArray=maskArray,
474 isTransposed=isTransposed)
476 def debugView(self, image, model):
477 """Debug display for the final overscan solution.
479 Parameters
480 ----------
481 image : `lsst.afw.image.Image`
482 Input image the overscan solution was determined from.
483 model : `numpy.ndarray` or `float`
484 Overscan model determined for the image.
485 """
486 import lsstDebug
487 if not lsstDebug.Info(__name__).display:
488 return
490 calcImage = self.getImageArray(image)
491 calcImage, isTransposed = self.transpose(calcImage)
492 masked = self.maskOutliers(calcImage)
493 collapsed = self.collapseArray(masked)
495 num = len(collapsed)
496 indices = 2.0 * np.arange(num)/float(num) - 1.0
498 if np.ma.is_masked(collapsed):
499 collapsedMask = collapsed.mask
500 else:
501 collapsedMask = np.array(num*[np.ma.nomask])
503 import matplotlib.pyplot as plot
504 figure = plot.figure(1)
505 figure.clear()
506 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
507 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
508 if collapsedMask.sum() > 0:
509 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
510 if isinstance(model, np.ndarray):
511 plotModel = model
512 else:
513 plotModel = np.zeros_like(indices)
514 plotModel += model
515 axes.plot(indices, plotModel, 'r-')
516 plot.xlabel("centered/scaled position along overscan region")
517 plot.ylabel("pixel value/fit value")
518 figure.show()
519 prompt = "Press Enter or c to continue [chp]..."
520 while True:
521 ans = input(prompt).lower()
522 if ans in ("", " ", "c",):
523 break
524 elif ans in ("p", ):
525 import pdb
526 pdb.set_trace()
527 elif ans in ("h", ):
528 print("[h]elp [c]ontinue [p]db")
529 plot.close()