22__all__ = [
"OverscanCorrectionTaskConfig",
"OverscanCorrectionTask"]
28import lsst.pipe.base
as pipeBase
31from .isr
import fitOverscanImage
35 """Overscan correction options.
37 fitType = pexConfig.ChoiceField(
39 doc="The method for fitting the overscan bias level.",
42 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
43 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
44 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
45 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
46 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
47 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
48 "MEAN":
"Correct using the mean of the overscan region",
49 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
50 "MEDIAN":
"Correct using the median of the overscan region",
51 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
54 order = pexConfig.Field(
56 doc=(
"Order of polynomial to fit if overscan fit type is a polynomial, "
57 "or number of spline knots if overscan fit type is a spline."),
60 numSigmaClip = pexConfig.Field(
62 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
65 maskPlanes = pexConfig.ListField(
67 doc=
"Mask planes to reject when measuring overscan",
68 default=[
'BAD',
'SAT'],
70 overscanIsInt = pexConfig.Field(
72 doc=
"Treat overscan as an integer image for purposes of fitType=MEDIAN"
73 " and fitType=MEDIAN_PER_ROW.",
77 doParallelOverscan = pexConfig.Field(
79 doc=
"Correct using parallel overscan after serial overscan correction?",
83 leadingColumnsToSkip = pexConfig.Field(
85 doc=
"Number of leading columns to skip in serial overscan correction.",
88 trailingColumnsToSkip = pexConfig.Field(
90 doc=
"Number of trailing columns to skip in serial overscan correction.",
93 leadingRowsToSkip = pexConfig.Field(
95 doc=
"Number of leading rows to skip in parallel overscan correction.",
98 trailingRowsToSkip = pexConfig.Field(
100 doc=
"Number of trailing rows to skip in parallel overscan correction.",
104 maxDeviation = pexConfig.Field(
106 doc=
"Maximum deviation from median (in ADU) to mask in overscan correction.",
107 default=1000.0, check=
lambda x: x > 0,
112 """Correction task for overscan.
114 This class contains a number of utilities that are easier to
115 understand
and use when they are
not embedded
in nested
if/
else
121 Statistics control object.
123 ConfigClass = OverscanCorrectionTaskConfig
124 _DefaultName = "overscan"
134 self.
statControl.setNumSigmaClip(self.config.numSigmaClip)
135 self.
statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes))
137 def run(self, exposure, amp, isTransposed=False):
138 """Measure and remove an overscan from an amplifier image.
143 Image data that will have the overscan corrections applied.
145 Amplifier to use for debugging purposes.
146 isTransposed : `bool`, optional
147 Is the image transposed, such that serial
and parallel
148 overscan regions are reversed? Default
is False.
152 overscanResults : `lsst.pipe.base.Struct`
153 Result struct
with components:
156 Value
or fit subtracted
from the amplifier image data
159 Value
or fit subtracted
from the overscan image data
162 Image of the overscan region
with the overscan
164 quantity
is used to estimate the amplifier read noise
170 Raised
if an invalid overscan type
is set.
173 serialOverscanBBox = amp.getRawSerialOverscanBBox()
174 imageBBox = amp.getRawDataBBox()
176 if self.config.doParallelOverscan:
179 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
180 imageBBox = imageBBox.expandedTo(parallelOverscanBBox)
183 imageBBox.getMinY()),
185 imageBBox.getHeight()))
187 imageBBox, serialOverscanBBox, isTransposed=isTransposed)
188 overscanMean = serialResults.overscanMean
189 overscanSigma = serialResults.overscanSigma
190 residualMean = serialResults.overscanMeanResidual
191 residualSigma = serialResults.overscanSigmaResidual
194 if self.config.doParallelOverscan:
197 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
198 imageBBox = amp.getRawDataBBox()
201 imageBBox, parallelOverscanBBox,
202 isTransposed=
not isTransposed)
204 overscanMean = (overscanMean, parallelResults.overscanMean)
205 overscanSigma = (overscanSigma, parallelResults.overscanSigma)
206 residualMean = (residualMean, parallelResults.overscanMeanResidual)
207 residualSigma = (residualSigma, parallelResults.overscanSigmaResidual)
209 return pipeBase.Struct(imageFit=serialResults.ampOverscanModel,
210 overscanFit=serialResults.overscanOverscanModel,
211 overscanImage=serialResults.overscanImage,
213 overscanMean=overscanMean,
214 overscanSigma=overscanSigma,
215 residualMean=residualMean,
216 residualSigma=residualSigma)
221 overscanBox = self.trimOverscan(exposure, amp, overscanBBox,
222 self.config.leadingColumnsToSkip,
223 self.config.trailingColumnsToSkip,
224 transpose=isTransposed)
225 overscanImage = exposure[overscanBox].getMaskedImage()
226 overscanArray = overscanImage.image.array
229 maskVal = overscanImage.mask.getPlaneBitMask(self.config.maskPlanes)
230 overscanMask = ~((overscanImage.mask.array & maskVal) == 0)
232 median = np.ma.median(np.ma.masked_where(overscanMask, overscanArray))
233 bad = np.where(np.abs(overscanArray - median) > self.config.maxDeviation)
234 overscanMask[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
238 overscanResults = self.
fitOverscan(overscanImage, isTransposed=isTransposed)
241 ampImage = exposure[imageBBox]
243 ampImage.image.array,
244 transpose=isTransposed)
245 ampImage.image.array -= ampOverscanModel
249 overscanImage = exposure[overscanBBox]
252 overscanImage.image.array)
253 overscanImage.image.array -= overscanOverscanModel
255 self.
debugView(overscanImage, overscanResults.overscanValue, amp)
258 stats = afwMath.makeStatistics(overscanImage.getMaskedImage(),
259 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
260 residualMean = stats.getValue(afwMath.MEDIAN)
261 residualSigma = stats.getValue(afwMath.STDEVCLIP)
263 return pipeBase.Struct(ampOverscanModel=ampOverscanModel,
264 overscanOverscanModel=overscanOverscanModel,
265 overscanImage=overscanImage,
266 overscanValue=overscanResults.overscanValue,
268 overscanMean=overscanResults.overscanMean,
269 overscanSigma=overscanResults.overscanSigma,
270 overscanMeanResidual=residualMean,
271 overscanSigmaResidual=residualSigma
275 """Broadcast 0 or 1 dimension fit to appropriate shape.
279 overscanValue : `numpy.ndarray`, (Nrows, ) or scalar
280 Overscan fit to broadcast.
281 imageArray : `numpy.ndarray`, (Nrows, Ncols)
282 Image array that we want to match.
283 transpose : `bool`, optional
284 Switch order to broadcast along the other axis.
288 overscanModel : `numpy.ndarray`, (Nrows, Ncols)
or scalar
289 Expanded overscan fit.
294 Raised
if no axis has the appropriate dimension.
296 if isinstance(overscanValue, np.ndarray):
297 overscanModel = np.zeros_like(imageArray)
299 if transpose
is False:
300 if imageArray.shape[0] == overscanValue.shape[0]:
301 overscanModel[:, :] = overscanValue[:, np.newaxis]
302 elif imageArray.shape[1] == overscanValue.shape[0]:
303 overscanModel[:, :] = overscanValue[np.newaxis, :]
304 elif imageArray.shape[0] == overscanValue.shape[1]:
305 overscanModel[:, :] = overscanValue[np.newaxis, :]
307 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
308 f
"match {imageArray.shape}")
310 if imageArray.shape[1] == overscanValue.shape[0]:
311 overscanModel[:, :] = overscanValue[np.newaxis, :]
312 elif imageArray.shape[0] == overscanValue.shape[0]:
313 overscanModel[:, :] = overscanValue[:, np.newaxis]
314 elif imageArray.shape[1] == overscanValue.shape[1]:
315 overscanModel[:, :] = overscanValue[:, np.newaxis]
317 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
318 f
"match {imageArray.shape}")
320 overscanModel = overscanValue
324 def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False):
325 """Trim overscan region to remove edges.
330 Exposure containing data.
332 Amplifier containing geometry information.
334 Bounding box of the overscan region.
336 Number of leading (towards data region) rows/columns to skip.
338 Number of trailing (away from data region) rows/columns to skip.
339 transpose : `bool`, optional
340 Operate on the transposed array.
344 overscanArray : `numpy.array`, (N, M)
346 overscanMask : `numpy.array`, (N, M)
349 dx0, dy0, dx1, dy1 = (0, 0, 0, 0)
350 dataBBox = amp.getRawDataBBox()
352 if dataBBox.getBeginY() < bbox.getBeginY():
359 if dataBBox.getBeginX() < bbox.getBeginX():
368 bbox.getHeight() - dy0 + dy1))
372 if self.config.fitType
in (
'MEAN',
'MEANCLIP',
'MEDIAN'):
375 overscanValue = overscanResult.overscanValue
376 overscanMean = overscanValue
378 elif self.config.fitType
in (
'MEDIAN_PER_ROW',
'POLY',
'CHEB',
'LEG',
379 'NATURAL_SPLINE',
'CUBIC_SPLINE',
'AKIMA_SPLINE'):
382 overscanValue = overscanResult.overscanValue
384 stats = afwMath.makeStatistics(overscanResult.overscanValue,
385 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
386 overscanMean = stats.getValue(afwMath.MEDIAN)
387 overscanSigma = stats.getValue(afwMath.STDEVCLIP)
389 raise ValueError(
'%s : %s an invalid overscan type' %
390 (
"overscanCorrection", self.config.fitType))
392 return pipeBase.Struct(overscanValue=overscanValue,
393 overscanMean=overscanMean,
394 overscanSigma=overscanSigma,
399 """Return an integer version of the input image.
404 Image to convert to integers.
409 The integer converted image.
414 Raised
if the input image could
not be converted.
416 if hasattr(image,
"image"):
418 imageI = image.image.convertI()
419 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
420 elif hasattr(image,
"convertI"):
422 outI = image.convertI()
423 elif hasattr(image,
"astype"):
425 outI = image.astype(int)
427 raise RuntimeError(
"Could not convert this to integers: %s %s %s",
428 image, type(image), dir(image))
433 """Measure a constant overscan value.
438 Image data to measure the overscan
from.
442 results : `lsst.pipe.base.Struct`
443 Overscan result
with entries:
444 - ``overscanValue``: Overscan value to subtract (`float`)
445 - ``isTransposed``: Orientation of the overscan (`bool`)
447 if self.config.fitType ==
'MEDIAN':
451 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
452 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.
statControl).getValue()
454 return pipeBase.Struct(overscanValue=overscanValue,
459 """Extract the numpy array from the input image.
464 Image data to pull array
from.
466 calcImage : `numpy.ndarray`
467 Image data array
for numpy operating.
469 if hasattr(image,
"getImage"):
470 calcImage = image.getImage().getArray()
471 calcImage = np.ma.masked_where(image.getMask().getArray() & self.
statControl.getAndMask(),
474 calcImage = image.getArray()
478 """Mask outliers in a row of overscan data
from a robust sigma
483 imageArray : `numpy.ndarray`
484 Image to filter along numpy axis=1.
488 maskedArray : `numpy.ma.masked_array`
489 Masked image marking outliers.
491 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
493 axisStdev = 0.74*(uq - lq)
495 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
496 return np.ma.masked_where(diff > self.
statControl.getNumSigmaClip()
497 * axisStdev[:, np.newaxis], imageArray)
501 """Collapse overscan array (and mask) to a 1-D vector of values.
505 maskedArray : `numpy.ma.masked_array`
506 Masked array of input overscan data.
510 collapsed : `numpy.ma.masked_array`
511 Single dimensional overscan data, combined with the mean.
513 collapsed = np.mean(maskedArray, axis=1)
514 if collapsed.mask.sum() > 0:
515 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
519 """Collapse overscan array (and mask) to a 1-D vector of using the
520 correct integer median of row-values.
524 maskedArray : `numpy.ma.masked_array`
525 Masked array of input overscan data.
529 collapsed : `numpy.ma.masked_array`
530 Single dimensional overscan data, combined with the afwMath median.
535 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
536 for row
in integerMI:
537 newRow = row.compressed()
539 rowMedian = afwMath.makeStatistics(newRow, fitType, self.
statControl).getValue()
542 collapsed.append(rowMedian)
544 return np.array(collapsed)
547 """Wrapper function to match spline fit API to polynomial fit API.
551 indices : `numpy.ndarray`
552 Locations to evaluate the spline.
553 collapsed : `numpy.ndarray`
554 Collapsed overscan values corresponding to the spline
557 Number of bins to use in constructing the spline.
562 Interpolation object
for later evaluation.
564 if not np.ma.is_masked(collapsed):
565 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
567 numPerBin, binEdges = np.histogram(indices, bins=numBins,
568 weights=1 - collapsed.mask.astype(int))
569 with np.errstate(invalid=
"ignore"):
570 values = np.histogram(indices, bins=numBins,
571 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
572 binCenters = np.histogram(indices, bins=numBins,
573 weights=indices*~collapsed.mask)[0]/numPerBin
575 if len(binCenters[numPerBin > 0]) < 5:
576 self.log.warn(
"Cannot do spline fitting for overscan: %s valid points.",
577 len(binCenters[numPerBin > 0]))
581 if len(values[numPerBin > 0]) != 0:
582 return float(values[numPerBin > 0][0])
586 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
587 values.astype(float)[numPerBin > 0],
588 afwMath.stringToInterpStyle(self.config.fitType))
593 """Wrapper function to match spline evaluation API to polynomial fit
598 indices : `numpy.ndarray`
599 Locations to evaluate the spline.
600 interp : `lsst.afw.math.interpolate`
601 Interpolation object to use.
605 values : `numpy.ndarray`
606 Evaluated spline values at each index.
609 return interp.interpolate(indices.astype(float))
613 """Create mask if edges are extrapolated.
617 collapsed : `numpy.ma.masked_array`
618 Masked array to check the edges of.
622 maskArray : `numpy.ndarray`
623 Boolean numpy array of pixels to mask.
625 maskArray = np.full_like(collapsed, False, dtype=bool)
626 if np.ma.is_masked(collapsed):
628 for low
in range(num):
629 if not collapsed.mask[low]:
632 maskArray[:low] =
True
633 for high
in range(1, num):
634 if not collapsed.mask[-high]:
637 maskArray[-high:] =
True
641 """Calculate the 1-d vector overscan from the input overscan image.
646 Image containing the overscan data.
647 isTransposed : `bool`
648 If true, the image has been transposed.
652 results : `lsst.pipe.base.Struct`
653 Overscan result with entries:
656 Overscan value to subtract (`float`)
658 List of rows that should be masked
as ``SUSPECT`` when the
659 overscan solution
is applied. (`list` [ `bool` ])
661 Indicates
if the overscan data was transposed during
662 calcuation, noting along which axis the overscan should be
669 calcImage = np.transpose(calcImage)
672 if self.config.fitType ==
'MEDIAN_PER_ROW':
673 mi = afwImage.MaskedImageI(image.getBBox())
674 masked = masked.astype(int)
676 masked = masked.transpose()
678 mi.image.array[:, :] = masked.data[:, :]
679 if bool(masked.mask.shape):
680 mi.mask.array[:, :] = masked.mask[:, :]
688 indices = 2.0*np.arange(num)/float(num) - 1.0
692 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
693 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
694 'LEG': (poly.legendre.legfit, poly.legendre.legval),
698 }[self.config.fitType]
702 coeffs = fitter(indices, collapsed, self.config.order)
704 if isinstance(coeffs, float):
705 self.log.warn(
"Using fallback value %f due to fitter failure. Amplifier will be masked.",
707 overscanVector = np.full_like(indices, coeffs)
708 maskArray = np.full_like(collapsed,
True, dtype=bool)
711 overscanVector = evaler(indices, coeffs)
714 return pipeBase.Struct(overscanValue=np.array(overscanVector),
716 isTransposed=isTransposed)
719 """Debug display for the final overscan solution.
724 Input image the overscan solution was determined from.
725 model : `numpy.ndarray`
or `float`
726 Overscan model determined
for the image.
728 Amplifier to extract diagnostic information.
738 calcImage = np.transpose(calcImage)
743 indices = 2.0 * np.arange(num)/float(num) - 1.0
745 if np.ma.is_masked(collapsed):
746 collapsedMask = collapsed.mask
748 collapsedMask = np.array(num*[np.ma.nomask])
750 import matplotlib.pyplot
as plot
751 figure = plot.figure(1)
753 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
754 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask],
'k+')
755 if collapsedMask.sum() > 0:
756 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask],
'b+')
757 if isinstance(model, np.ndarray):
760 plotModel = np.zeros_like(indices)
762 axes.plot(indices, plotModel,
'r-')
763 plot.xlabel(
"centered/scaled position along overscan region")
764 plot.ylabel(
"pixel value/fit value")
766 plot.title(f
"{amp.getName()} DataX: "
767 f
"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]"
768 f
"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:"
769 f
"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}")
771 plot.title(
"No amp supplied.")
773 prompt =
"Press Enter or c to continue [chp]..."
775 ans = input(prompt).lower()
776 if ans
in (
"",
" ",
"c",):
785 print(
"[h]elp [c]ontinue [p]db e[x]itDebug")
def collapseArrayMedian(self, maskedArray)
def maskExtrapolated(collapsed)
def fitOverscan(self, overscanImage, isTransposed=False)
def measureVectorOverscan(self, image, isTransposed=False)
def __init__(self, statControl=None, **kwargs)
def collapseArray(maskedArray)
def maskOutliers(self, imageArray)
def debugView(self, image, model, amp=None)
def getImageArray(self, image)
def integerConvert(image)
def splineFit(self, indices, collapsed, numBins)
def broadcastFitToImage(self, overscanValue, imageArray, transpose=False)
def correctOverscan(self, exposure, amp, imageBBox, overscanBBox, isTransposed=True)
def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False)
def run(self, exposure, amp, isTransposed=False)
def splineEval(indices, interp)
def measureConstantOverscan(self, image)
std::vector< double > fitOverscanImage(lsst::afw::image::MaskedImage< ImagePixelT > const &overscan, std::vector< std::string > badPixelMask, bool isTransposed)