Coverage for python/lsst/ip/isr/overscan.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23import lsst.afw.math as afwMath
24import lsst.afw.image as afwImage
25import lsst.pipe.base as pipeBase
26import lsst.pex.config as pexConfig
28__all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"]
31class OverscanCorrectionTaskConfig(pexConfig.Config):
32 """Overscan correction options.
33 """
34 fitType = pexConfig.ChoiceField(
35 dtype=str,
36 doc="The method for fitting the overscan bias level.",
37 default='MEDIAN',
38 allowed={
39 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
40 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
41 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
42 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
43 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
44 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
45 "MEAN": "Correct using the mean of the overscan region",
46 "MEANCLIP": "Correct using a clipped mean of the overscan region",
47 "MEDIAN": "Correct using the median of the overscan region",
48 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
49 },
50 )
51 order = pexConfig.Field(
52 dtype=int,
53 doc=("Order of polynomial to fit if overscan fit type is a polynomial, "
54 "or number of spline knots if overscan fit type is a spline."),
55 default=1,
56 )
57 numSigmaClip = pexConfig.Field(
58 dtype=float,
59 doc="Rejection threshold (sigma) for collapsing overscan before fit",
60 default=3.0,
61 )
62 maskPlanes = pexConfig.ListField(
63 dtype=str,
64 doc="Mask planes to reject when measuring overscan",
65 default=['SAT'],
66 )
67 overscanIsInt = pexConfig.Field(
68 dtype=bool,
69 doc="Treat overscan as an integer image for purposes of fitType=MEDIAN"
70 " and fitType=MEDIAN_PER_ROW.",
71 default=True,
72 )
75class OverscanCorrectionTask(pipeBase.Task):
76 """Correction task for overscan.
78 This class contains a number of utilities that are easier to
79 understand and use when they are not embedded in nested if/else
80 loops.
82 Parameters
83 ----------
84 statControl : `lsst.afw.math.StatisticsControl`, optional
85 Statistics control object.
86 """
87 ConfigClass = OverscanCorrectionTaskConfig
88 _DefaultName = "overscan"
90 def __init__(self, statControl=None, **kwargs):
91 super().__init__(**kwargs)
92 if statControl:
93 self.statControl = statControl
94 else:
95 self.statControl = afwMath.StatisticsControl()
96 self.statControl.setNumSigmaClip(self.config.numSigmaClip)
97 self.statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes))
99 def run(self, ampImage, overscanImage):
100 """Measure and remove an overscan from an amplifier image.
102 Parameters
103 ----------
104 ampImage : `lsst.afw.image.Image`
105 Image data that will have the overscan removed.
106 overscanImage : `lsst.afw.image.Image`
107 Overscan data that the overscan is measured from.
109 Returns
110 -------
111 overscanResults : `lsst.pipe.base.Struct`
112 Result struct with components:
114 ``imageFit``
115 Value or fit subtracted from the amplifier image data
116 (scalar or `lsst.afw.image.Image`).
117 ``overscanFit``
118 Value or fit subtracted from the overscan image data
119 (scalar or `lsst.afw.image.Image`).
120 ``overscanImage``
121 Image of the overscan region with the overscan
122 correction applied (`lsst.afw.image.Image`). This
123 quantity is used to estimate the amplifier read noise
124 empirically.
126 Raises
127 ------
128 RuntimeError
129 Raised if an invalid overscan type is set.
131 """
132 if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'):
133 overscanResult = self.measureConstantOverscan(overscanImage)
134 overscanValue = overscanResult.overscanValue
135 offImage = overscanValue
136 overscanModel = overscanValue
137 maskSuspect = None
138 elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG',
139 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
140 overscanResult = self.measureVectorOverscan(overscanImage)
141 overscanValue = overscanResult.overscanValue
142 maskArray = overscanResult.maskArray
143 isTransposed = overscanResult.isTransposed
145 offImage = afwImage.ImageF(ampImage.getDimensions())
146 offArray = offImage.getArray()
147 overscanModel = afwImage.ImageF(overscanImage.getDimensions())
148 overscanArray = overscanModel.getArray()
150 if hasattr(ampImage, 'getMask'):
151 maskSuspect = afwImage.Mask(ampImage.getDimensions())
152 else:
153 maskSuspect = None
155 if isTransposed:
156 offArray[:, :] = overscanValue[np.newaxis, :]
157 overscanArray[:, :] = overscanValue[np.newaxis, :]
158 if maskSuspect:
159 maskSuspect.getArray()[:, maskArray] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
160 else:
161 offArray[:, :] = overscanValue[:, np.newaxis]
162 overscanArray[:, :] = overscanValue[:, np.newaxis]
163 if maskSuspect:
164 maskSuspect.getArray()[maskArray, :] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
165 else:
166 raise RuntimeError('%s : %s an invalid overscan type' %
167 ("overscanCorrection", self.config.fitType))
169 self.debugView(overscanImage, overscanValue)
171 ampImage -= offImage
172 if maskSuspect:
173 ampImage.getMask().getArray()[:, :] |= maskSuspect.getArray()[:, :]
174 overscanImage -= overscanModel
175 return pipeBase.Struct(imageFit=offImage,
176 overscanFit=overscanModel,
177 overscanImage=overscanImage,
178 edgeMask=maskSuspect)
180 @staticmethod
181 def integerConvert(image):
182 """Return an integer version of the input image.
184 Parameters
185 ----------
186 image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
187 Image to convert to integers.
189 Returns
190 -------
191 outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
192 The integer converted image.
194 Raises
195 ------
196 RuntimeError
197 Raised if the input image could not be converted.
198 """
199 if hasattr(image, "image"):
200 # Is a maskedImage:
201 imageI = image.image.convertI()
202 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
203 elif hasattr(image, "convertI"):
204 # Is an Image:
205 outI = image.convertI()
206 elif hasattr(image, "astype"):
207 # Is a numpy array:
208 outI = image.astype(int)
209 else:
210 raise RuntimeError("Could not convert this to integers: %s %s %s",
211 image, type(image), dir(image))
212 return outI
214 # Constant methods
215 def measureConstantOverscan(self, image):
216 """Measure a constant overscan value.
218 Parameters
219 ----------
220 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
221 Image data to measure the overscan from.
223 Returns
224 -------
225 results : `lsst.pipe.base.Struct`
226 Overscan result with entries:
227 - ``overscanValue``: Overscan value to subtract (`float`)
228 - ``maskArray``: Placeholder for a mask array (`list`)
229 - ``isTransposed``: Orientation of the overscan (`bool`)
230 """
231 if self.config.fitType == 'MEDIAN':
232 calcImage = self.integerConvert(image)
233 else:
234 calcImage = image
236 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
237 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControl).getValue()
239 return pipeBase.Struct(overscanValue=overscanValue,
240 maskArray=None,
241 isTransposed=False)
243 # Vector correction utilities
244 def getImageArray(self, image):
245 """Extract the numpy array from the input image.
247 Parameters
248 ----------
249 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
250 Image data to pull array from.
252 calcImage : `numpy.ndarray`
253 Image data array for numpy operating.
254 """
255 if hasattr(image, "getImage"):
256 calcImage = image.getImage().getArray()
257 calcImage = np.ma.masked_where(image.getMask().getArray() & self.statControl.getAndMask(),
258 calcImage)
259 else:
260 calcImage = image.getArray()
261 return calcImage
263 @staticmethod
264 def transpose(imageArray):
265 """Transpose input numpy array if necessary.
267 Parameters
268 ----------
269 imageArray : `numpy.ndarray`
270 Image data to transpose.
272 Returns
273 -------
274 imageArray : `numpy.ndarray`
275 Transposed image data.
276 isTransposed : `bool`
277 Indicates whether the input data was transposed.
278 """
279 if np.argmin(imageArray.shape) == 0:
280 return np.transpose(imageArray), True
281 else:
282 return imageArray, False
284 def maskOutliers(self, imageArray):
285 """Mask outliers in a row of overscan data from a robust sigma
286 clipping procedure.
288 Parameters
289 ----------
290 imageArray : `numpy.ndarray`
291 Image to filter along numpy axis=1.
293 Returns
294 -------
295 maskedArray : `numpy.ma.masked_array`
296 Masked image marking outliers.
297 """
298 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
299 axisMedians = median
300 axisStdev = 0.74*(uq - lq) # robust stdev
302 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
303 return np.ma.masked_where(diff > self.statControl.getNumSigmaClip()
304 * axisStdev[:, np.newaxis], imageArray)
306 @staticmethod
307 def collapseArray(maskedArray):
308 """Collapse overscan array (and mask) to a 1-D vector of values.
310 Parameters
311 ----------
312 maskedArray : `numpy.ma.masked_array`
313 Masked array of input overscan data.
315 Returns
316 -------
317 collapsed : `numpy.ma.masked_array`
318 Single dimensional overscan data, combined with the mean.
319 """
320 collapsed = np.mean(maskedArray, axis=1)
321 if collapsed.mask.sum() > 0:
322 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
323 return collapsed
325 def collapseArrayMedian(self, maskedArray):
326 """Collapse overscan array (and mask) to a 1-D vector of using the
327 correct integer median of row-values.
329 Parameters
330 ----------
331 maskedArray : `numpy.ma.masked_array`
332 Masked array of input overscan data.
334 Returns
335 -------
336 collapsed : `numpy.ma.masked_array`
337 Single dimensional overscan data, combined with the afwMath median.
338 """
339 integerMI = self.integerConvert(maskedArray)
341 collapsed = []
342 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
343 for row in integerMI:
344 rowMedian = afwMath.makeStatistics(row, fitType, self.statControl).getValue()
345 collapsed.append(rowMedian)
347 return np.array(collapsed)
349 def splineFit(self, indices, collapsed, numBins):
350 """Wrapper function to match spline fit API to polynomial fit API.
352 Parameters
353 ----------
354 indices : `numpy.ndarray`
355 Locations to evaluate the spline.
356 collapsed : `numpy.ndarray`
357 Collapsed overscan values corresponding to the spline
358 evaluation points.
359 numBins : `int`
360 Number of bins to use in constructing the spline.
362 Returns
363 -------
364 interp : `lsst.afw.math.Interpolate`
365 Interpolation object for later evaluation.
366 """
367 if not np.ma.is_masked(collapsed):
368 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
370 numPerBin, binEdges = np.histogram(indices, bins=numBins,
371 weights=1 - collapsed.mask.astype(int))
372 with np.errstate(invalid="ignore"):
373 values = np.histogram(indices, bins=numBins,
374 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
375 binCenters = np.histogram(indices, bins=numBins,
376 weights=indices*~collapsed.mask)[0]/numPerBin
377 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
378 values.astype(float)[numPerBin > 0],
379 afwMath.stringToInterpStyle(self.config.fitType))
380 return interp
382 @staticmethod
383 def splineEval(indices, interp):
384 """Wrapper function to match spline evaluation API to polynomial fit API.
386 Parameters
387 ----------
388 indices : `numpy.ndarray`
389 Locations to evaluate the spline.
390 interp : `lsst.afw.math.interpolate`
391 Interpolation object to use.
393 Returns
394 -------
395 values : `numpy.ndarray`
396 Evaluated spline values at each index.
397 """
399 return interp.interpolate(indices.astype(float))
401 @staticmethod
402 def maskExtrapolated(collapsed):
403 """Create mask if edges are extrapolated.
405 Parameters
406 ----------
407 collapsed : `numpy.ma.masked_array`
408 Masked array to check the edges of.
410 Returns
411 -------
412 maskArray : `numpy.ndarray`
413 Boolean numpy array of pixels to mask.
414 """
415 maskArray = np.full_like(collapsed, False, dtype=bool)
416 if np.ma.is_masked(collapsed):
417 num = len(collapsed)
418 for low in range(num):
419 if not collapsed.mask[low]:
420 break
421 if low > 0:
422 maskArray[:low] = True
423 for high in range(1, num):
424 if not collapsed.mask[-high]:
425 break
426 if high > 1:
427 maskArray[-high:] = True
428 return maskArray
430 def measureVectorOverscan(self, image):
431 """Calculate the 1-d vector overscan from the input overscan image.
433 Parameters
434 ----------
435 image : `lsst.afw.image.MaskedImage`
436 Image containing the overscan data.
438 Returns
439 -------
440 results : `lsst.pipe.base.Struct`
441 Overscan result with entries:
442 - ``overscanValue``: Overscan value to subtract (`float`)
443 - ``maskArray`` : `list` [ `bool` ]
444 List of rows that should be masked as ``SUSPECT`` when the
445 overscan solution is applied.
446 - ``isTransposed`` : `bool`
447 Indicates if the overscan data was transposed during
448 calcuation, noting along which axis the overscan should be
449 subtracted.
450 """
451 calcImage = self.getImageArray(image)
453 # operate on numpy-arrays from here
454 calcImage, isTransposed = self.transpose(calcImage)
455 masked = self.maskOutliers(calcImage)
457 if self.config.fitType == 'MEDIAN_PER_ROW':
458 overscanVector = self.collapseArrayMedian(masked)
459 maskArray = self.maskExtrapolated(overscanVector)
460 else:
461 collapsed = self.collapseArray(masked)
463 num = len(collapsed)
464 indices = 2.0*np.arange(num)/float(num) - 1.0
466 poly = np.polynomial
467 fitter, evaler = {
468 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
469 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
470 'LEG': (poly.legendre.legfit, poly.legendre.legval),
471 'NATURAL_SPLINE': (self.splineFit, self.splineEval),
472 'CUBIC_SPLINE': (self.splineFit, self.splineEval),
473 'AKIMA_SPLINE': (self.splineFit, self.splineEval)
474 }[self.config.fitType]
476 coeffs = fitter(indices, collapsed, self.config.order)
477 overscanVector = evaler(indices, coeffs)
478 maskArray = self.maskExtrapolated(collapsed)
479 return pipeBase.Struct(overscanValue=np.array(overscanVector),
480 maskArray=maskArray,
481 isTransposed=isTransposed)
483 def debugView(self, image, model):
484 """Debug display for the final overscan solution.
486 Parameters
487 ----------
488 image : `lsst.afw.image.Image`
489 Input image the overscan solution was determined from.
490 model : `numpy.ndarray` or `float`
491 Overscan model determined for the image.
492 """
493 import lsstDebug
494 if not lsstDebug.Info(__name__).display:
495 return
497 calcImage = self.getImageArray(image)
498 calcImage, isTransposed = self.transpose(calcImage)
499 masked = self.maskOutliers(calcImage)
500 collapsed = self.collapseArray(masked)
502 num = len(collapsed)
503 indices = 2.0 * np.arange(num)/float(num) - 1.0
505 if np.ma.is_masked(collapsed):
506 collapsedMask = collapsed.mask
507 else:
508 collapsedMask = np.array(num*[np.ma.nomask])
510 import matplotlib.pyplot as plot
511 figure = plot.figure(1)
512 figure.clear()
513 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
514 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
515 if collapsedMask.sum() > 0:
516 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
517 if isinstance(model, np.ndarray):
518 plotModel = model
519 else:
520 plotModel = np.zeros_like(indices)
521 plotModel += model
522 axes.plot(indices, plotModel, 'r-')
523 plot.xlabel("centered/scaled position along overscan region")
524 plot.ylabel("pixel value/fit value")
525 figure.show()
526 prompt = "Press Enter or c to continue [chp]..."
527 while True:
528 ans = input(prompt).lower()
529 if ans in ("", " ", "c",):
530 break
531 elif ans in ("p", ):
532 import pdb
533 pdb.set_trace()
534 elif ans in ("h", ):
535 print("[h]elp [c]ontinue [p]db")
536 plot.close()