lsst.ip.isr g9cb75138f3+6db9579539
overscan.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import numpy as np
23import lsst.afw.math as afwMath
24import lsst.afw.image as afwImage
25import lsst.pipe.base as pipeBase
26import lsst.pex.config as pexConfig
27
28__all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"]
29
30
31class OverscanCorrectionTaskConfig(pexConfig.Config):
32 """Overscan correction options.
33 """
34 fitType = pexConfig.ChoiceField(
35 dtype=str,
36 doc="The method for fitting the overscan bias level.",
37 default='MEDIAN',
38 allowed={
39 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
40 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
41 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
42 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
43 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
44 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
45 "MEAN": "Correct using the mean of the overscan region",
46 "MEANCLIP": "Correct using a clipped mean of the overscan region",
47 "MEDIAN": "Correct using the median of the overscan region",
48 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
49 },
50 )
51 order = pexConfig.Field(
52 dtype=int,
53 doc=("Order of polynomial to fit if overscan fit type is a polynomial, "
54 "or number of spline knots if overscan fit type is a spline."),
55 default=1,
56 )
57 numSigmaClip = pexConfig.Field(
58 dtype=float,
59 doc="Rejection threshold (sigma) for collapsing overscan before fit",
60 default=3.0,
61 )
62 maskPlanes = pexConfig.ListField(
63 dtype=str,
64 doc="Mask planes to reject when measuring overscan",
65 default=['SAT'],
66 )
67 overscanIsInt = pexConfig.Field(
68 dtype=bool,
69 doc="Treat overscan as an integer image for purposes of fitType=MEDIAN"
70 " and fitType=MEDIAN_PER_ROW.",
71 default=True,
72 )
73
74
75class OverscanCorrectionTask(pipeBase.Task):
76 """Correction task for overscan.
77
78 This class contains a number of utilities that are easier to
79 understand and use when they are not embedded in nested if/else
80 loops.
81
82 Parameters
83 ----------
84 statControl : `lsst.afw.math.StatisticsControl`, optional
85 Statistics control object.
86 """
87 ConfigClass = OverscanCorrectionTaskConfig
88 _DefaultName = "overscan"
89
90 def __init__(self, statControl=None, **kwargs):
91 super().__init__(**kwargs)
92 self.allowDebugallowDebug = True
93
94 if statControl:
95 self.statControlstatControl = statControl
96 else:
97 self.statControlstatControl = afwMath.StatisticsControl()
98 self.statControlstatControl.setNumSigmaClip(self.config.numSigmaClip)
99 self.statControlstatControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes))
100
101 def run(self, ampImage, overscanImage, amp=None):
102 """Measure and remove an overscan from an amplifier image.
103
104 Parameters
105 ----------
106 ampImage : `lsst.afw.image.Image`
107 Image data that will have the overscan removed.
108 overscanImage : `lsst.afw.image.Image`
109 Overscan data that the overscan is measured from.
110 amp : `lsst.afw.cameraGeom.Amplifier`, optional
111 Amplifier to use for debugging purposes.
112
113 Returns
114 -------
115 overscanResults : `lsst.pipe.base.Struct`
116 Result struct with components:
117
118 ``imageFit``
119 Value or fit subtracted from the amplifier image data
120 (scalar or `lsst.afw.image.Image`).
121 ``overscanFit``
122 Value or fit subtracted from the overscan image data
123 (scalar or `lsst.afw.image.Image`).
124 ``overscanImage``
125 Image of the overscan region with the overscan
126 correction applied (`lsst.afw.image.Image`). This
127 quantity is used to estimate the amplifier read noise
128 empirically.
129
130 Raises
131 ------
132 RuntimeError
133 Raised if an invalid overscan type is set.
134
135 """
136 if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'):
137 overscanResult = self.measureConstantOverscanmeasureConstantOverscan(overscanImage)
138 overscanValue = overscanResult.overscanValue
139 offImage = overscanValue
140 overscanModel = overscanValue
141 maskSuspect = None
142 elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG',
143 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
144 overscanResult = self.measureVectorOverscanmeasureVectorOverscan(overscanImage)
145 overscanValue = overscanResult.overscanValue
146 maskArray = overscanResult.maskArray
147 isTransposed = overscanResult.isTransposed
148
149 offImage = afwImage.ImageF(ampImage.getDimensions())
150 offArray = offImage.getArray()
151 overscanModel = afwImage.ImageF(overscanImage.getDimensions())
152 overscanArray = overscanModel.getArray()
153
154 if hasattr(ampImage, 'getMask'):
155 maskSuspect = afwImage.Mask(ampImage.getDimensions())
156 else:
157 maskSuspect = None
158
159 if isTransposed:
160 offArray[:, :] = overscanValue[np.newaxis, :]
161 overscanArray[:, :] = overscanValue[np.newaxis, :]
162 if maskSuspect:
163 maskSuspect.getArray()[:, maskArray] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
164 else:
165 offArray[:, :] = overscanValue[:, np.newaxis]
166 overscanArray[:, :] = overscanValue[:, np.newaxis]
167 if maskSuspect:
168 maskSuspect.getArray()[maskArray, :] |= ampImage.getMask().getPlaneBitMask("SUSPECT")
169 else:
170 raise RuntimeError('%s : %s an invalid overscan type' %
171 ("overscanCorrection", self.config.fitType))
172
173 self.debugViewdebugView(overscanImage, overscanValue, amp)
174
175 ampImage -= offImage
176 if maskSuspect:
177 ampImage.getMask().getArray()[:, :] |= maskSuspect.getArray()[:, :]
178 overscanImage -= overscanModel
179 return pipeBase.Struct(imageFit=offImage,
180 overscanFit=overscanModel,
181 overscanImage=overscanImage,
182 edgeMask=maskSuspect)
183
184 @staticmethod
185 def integerConvert(image):
186 """Return an integer version of the input image.
187
188 Parameters
189 ----------
190 image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
191 Image to convert to integers.
192
193 Returns
194 -------
195 outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
196 The integer converted image.
197
198 Raises
199 ------
200 RuntimeError
201 Raised if the input image could not be converted.
202 """
203 if hasattr(image, "image"):
204 # Is a maskedImage:
205 imageI = image.image.convertI()
206 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
207 elif hasattr(image, "convertI"):
208 # Is an Image:
209 outI = image.convertI()
210 elif hasattr(image, "astype"):
211 # Is a numpy array:
212 outI = image.astype(int)
213 else:
214 raise RuntimeError("Could not convert this to integers: %s %s %s",
215 image, type(image), dir(image))
216 return outI
217
218 # Constant methods
219 def measureConstantOverscan(self, image):
220 """Measure a constant overscan value.
221
222 Parameters
223 ----------
225 Image data to measure the overscan from.
226
227 Returns
228 -------
229 results : `lsst.pipe.base.Struct`
230 Overscan result with entries:
231 - ``overscanValue``: Overscan value to subtract (`float`)
232 - ``maskArray``: Placeholder for a mask array (`list`)
233 - ``isTransposed``: Orientation of the overscan (`bool`)
234 """
235 if self.config.fitType == 'MEDIAN':
236 calcImage = self.integerConvertintegerConvert(image)
237 else:
238 calcImage = image
239
240 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
241 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControlstatControl).getValue()
242
243 return pipeBase.Struct(overscanValue=overscanValue,
244 maskArray=None,
245 isTransposed=False)
246
247 # Vector correction utilities
248 def getImageArray(self, image):
249 """Extract the numpy array from the input image.
250
251 Parameters
252 ----------
254 Image data to pull array from.
255
256 calcImage : `numpy.ndarray`
257 Image data array for numpy operating.
258 """
259 if hasattr(image, "getImage"):
260 calcImage = image.getImage().getArray()
261 calcImage = np.ma.masked_where(image.getMask().getArray() & self.statControlstatControl.getAndMask(),
262 calcImage)
263 else:
264 calcImage = image.getArray()
265 return calcImage
266
267 @staticmethod
268 def transpose(imageArray):
269 """Transpose input numpy array if necessary.
270
271 Parameters
272 ----------
273 imageArray : `numpy.ndarray`
274 Image data to transpose.
275
276 Returns
277 -------
278 imageArray : `numpy.ndarray`
279 Transposed image data.
280 isTransposed : `bool`
281 Indicates whether the input data was transposed.
282 """
283 if np.argmin(imageArray.shape) == 0:
284 return np.transpose(imageArray), True
285 else:
286 return imageArray, False
287
288 def maskOutliers(self, imageArray):
289 """Mask outliers in a row of overscan data from a robust sigma
290 clipping procedure.
291
292 Parameters
293 ----------
294 imageArray : `numpy.ndarray`
295 Image to filter along numpy axis=1.
296
297 Returns
298 -------
299 maskedArray : `numpy.ma.masked_array`
300 Masked image marking outliers.
301 """
302 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
303 axisMedians = median
304 axisStdev = 0.74*(uq - lq) # robust stdev
305
306 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
307 return np.ma.masked_where(diff > self.statControlstatControl.getNumSigmaClip()
308 * axisStdev[:, np.newaxis], imageArray)
309
310 @staticmethod
311 def collapseArray(maskedArray):
312 """Collapse overscan array (and mask) to a 1-D vector of values.
313
314 Parameters
315 ----------
316 maskedArray : `numpy.ma.masked_array`
317 Masked array of input overscan data.
318
319 Returns
320 -------
321 collapsed : `numpy.ma.masked_array`
322 Single dimensional overscan data, combined with the mean.
323 """
324 collapsed = np.mean(maskedArray, axis=1)
325 if collapsed.mask.sum() > 0:
326 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
327 return collapsed
328
329 def collapseArrayMedian(self, maskedArray):
330 """Collapse overscan array (and mask) to a 1-D vector of using the
331 correct integer median of row-values.
332
333 Parameters
334 ----------
335 maskedArray : `numpy.ma.masked_array`
336 Masked array of input overscan data.
337
338 Returns
339 -------
340 collapsed : `numpy.ma.masked_array`
341 Single dimensional overscan data, combined with the afwMath median.
342 """
343 integerMI = self.integerConvertintegerConvert(maskedArray)
344
345 collapsed = []
346 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
347 for row in integerMI:
348 newRow = row.compressed()
349 if len(newRow) > 0:
350 rowMedian = afwMath.makeStatistics(newRow, fitType, self.statControlstatControl).getValue()
351 else:
352 rowMedian = np.nan
353 collapsed.append(rowMedian)
354
355 return np.array(collapsed)
356
357 def splineFit(self, indices, collapsed, numBins):
358 """Wrapper function to match spline fit API to polynomial fit API.
359
360 Parameters
361 ----------
362 indices : `numpy.ndarray`
363 Locations to evaluate the spline.
364 collapsed : `numpy.ndarray`
365 Collapsed overscan values corresponding to the spline
366 evaluation points.
367 numBins : `int`
368 Number of bins to use in constructing the spline.
369
370 Returns
371 -------
373 Interpolation object for later evaluation.
374 """
375 if not np.ma.is_masked(collapsed):
376 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
377
378 numPerBin, binEdges = np.histogram(indices, bins=numBins,
379 weights=1 - collapsed.mask.astype(int))
380 with np.errstate(invalid="ignore"):
381 values = np.histogram(indices, bins=numBins,
382 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
383 binCenters = np.histogram(indices, bins=numBins,
384 weights=indices*~collapsed.mask)[0]/numPerBin
385 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
386 values.astype(float)[numPerBin > 0],
387 afwMath.stringToInterpStyle(self.config.fitType))
388 return interp
389
390 @staticmethod
391 def splineEval(indices, interp):
392 """Wrapper function to match spline evaluation API to polynomial fit
393 API.
394
395 Parameters
396 ----------
397 indices : `numpy.ndarray`
398 Locations to evaluate the spline.
399 interp : `lsst.afw.math.interpolate`
400 Interpolation object to use.
401
402 Returns
403 -------
404 values : `numpy.ndarray`
405 Evaluated spline values at each index.
406 """
407
408 return interp.interpolate(indices.astype(float))
409
410 @staticmethod
411 def maskExtrapolated(collapsed):
412 """Create mask if edges are extrapolated.
413
414 Parameters
415 ----------
416 collapsed : `numpy.ma.masked_array`
417 Masked array to check the edges of.
418
419 Returns
420 -------
421 maskArray : `numpy.ndarray`
422 Boolean numpy array of pixels to mask.
423 """
424 maskArray = np.full_like(collapsed, False, dtype=bool)
425 if np.ma.is_masked(collapsed):
426 num = len(collapsed)
427 for low in range(num):
428 if not collapsed.mask[low]:
429 break
430 if low > 0:
431 maskArray[:low] = True
432 for high in range(1, num):
433 if not collapsed.mask[-high]:
434 break
435 if high > 1:
436 maskArray[-high:] = True
437 return maskArray
438
439 def measureVectorOverscan(self, image):
440 """Calculate the 1-d vector overscan from the input overscan image.
441
442 Parameters
443 ----------
445 Image containing the overscan data.
446
447 Returns
448 -------
449 results : `lsst.pipe.base.Struct`
450 Overscan result with entries:
451 - ``overscanValue``: Overscan value to subtract (`float`)
452 - ``maskArray`` : `list` [ `bool` ]
453 List of rows that should be masked as ``SUSPECT`` when the
454 overscan solution is applied.
455 - ``isTransposed`` : `bool`
456 Indicates if the overscan data was transposed during
457 calcuation, noting along which axis the overscan should be
458 subtracted.
459 """
460 calcImage = self.getImageArraygetImageArray(image)
461
462 # operate on numpy-arrays from here
463 calcImage, isTransposed = self.transposetranspose(calcImage)
464 masked = self.maskOutliersmaskOutliers(calcImage)
465
466 if self.config.fitType == 'MEDIAN_PER_ROW':
467 overscanVector = self.collapseArrayMediancollapseArrayMedian(masked)
468 maskArray = self.maskExtrapolatedmaskExtrapolated(overscanVector)
469 else:
470 collapsed = self.collapseArraycollapseArray(masked)
471
472 num = len(collapsed)
473 indices = 2.0*np.arange(num)/float(num) - 1.0
474
475 poly = np.polynomial
476 fitter, evaler = {
477 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
478 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
479 'LEG': (poly.legendre.legfit, poly.legendre.legval),
480 'NATURAL_SPLINE': (self.splineFitsplineFit, self.splineEvalsplineEval),
481 'CUBIC_SPLINE': (self.splineFitsplineFit, self.splineEvalsplineEval),
482 'AKIMA_SPLINE': (self.splineFitsplineFit, self.splineEvalsplineEval)
483 }[self.config.fitType]
484
485 coeffs = fitter(indices, collapsed, self.config.order)
486 overscanVector = evaler(indices, coeffs)
487 maskArray = self.maskExtrapolatedmaskExtrapolated(collapsed)
488 return pipeBase.Struct(overscanValue=np.array(overscanVector),
489 maskArray=maskArray,
490 isTransposed=isTransposed)
491
492 def debugView(self, image, model, amp=None):
493 """Debug display for the final overscan solution.
494
495 Parameters
496 ----------
497 image : `lsst.afw.image.Image`
498 Input image the overscan solution was determined from.
499 model : `numpy.ndarray` or `float`
500 Overscan model determined for the image.
501 amp : `lsst.afw.cameraGeom.Amplifier`, optional
502 Amplifier to extract diagnostic information.
503 """
504 import lsstDebug
505 if not lsstDebug.Info(__name__).display:
506 return
507 if not self.allowDebugallowDebug:
508 return
509
510 calcImage = self.getImageArraygetImageArray(image)
511 calcImage, isTransposed = self.transposetranspose(calcImage)
512 masked = self.maskOutliersmaskOutliers(calcImage)
513 collapsed = self.collapseArraycollapseArray(masked)
514
515 num = len(collapsed)
516 indices = 2.0 * np.arange(num)/float(num) - 1.0
517
518 if np.ma.is_masked(collapsed):
519 collapsedMask = collapsed.mask
520 else:
521 collapsedMask = np.array(num*[np.ma.nomask])
522
523 import matplotlib.pyplot as plot
524 figure = plot.figure(1)
525 figure.clear()
526 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
527 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
528 if collapsedMask.sum() > 0:
529 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
530 if isinstance(model, np.ndarray):
531 plotModel = model
532 else:
533 plotModel = np.zeros_like(indices)
534 plotModel += model
535 axes.plot(indices, plotModel, 'r-')
536 plot.xlabel("centered/scaled position along overscan region")
537 plot.ylabel("pixel value/fit value")
538 if amp:
539 plot.title(f"{amp.getName()} DataX: "
540 f"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]"
541 f"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:"
542 f"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}")
543 else:
544 plot.title("No amp supplied.")
545 figure.show()
546 prompt = "Press Enter or c to continue [chp]..."
547 while True:
548 ans = input(prompt).lower()
549 if ans in ("", " ", "c",):
550 break
551 elif ans in ("p", ):
552 import pdb
553 pdb.set_trace()
554 elif ans in ('x', ):
555 self.allowDebugallowDebug = False
556 break
557 elif ans in ("h", ):
558 print("[h]elp [c]ontinue [p]db e[x]itDebug")
559 plot.close()
def collapseArrayMedian(self, maskedArray)
Definition: overscan.py:329
def __init__(self, statControl=None, **kwargs)
Definition: overscan.py:90
def debugView(self, image, model, amp=None)
Definition: overscan.py:492
def run(self, ampImage, overscanImage, amp=None)
Definition: overscan.py:101
def splineFit(self, indices, collapsed, numBins)
Definition: overscan.py:357