22__all__ = [
"OverscanCorrectionTaskConfig",
"OverscanCorrectionTask"]
28import lsst.pipe.base
as pipeBase
31from .isr
import fitOverscanImage
32from .isrFunctions
import makeThresholdMask, countMaskedPixels
36 """Overscan correction options.
38 fitType = pexConfig.ChoiceField(
40 doc="The method for fitting the overscan bias level.",
43 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
44 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
45 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
46 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
47 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
48 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
49 "MEAN":
"Correct using the mean of the overscan region",
50 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
51 "MEDIAN":
"Correct using the median of the overscan region",
52 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
55 order = pexConfig.Field(
57 doc=(
"Order of polynomial to fit if overscan fit type is a polynomial, "
58 "or number of spline knots if overscan fit type is a spline."),
61 numSigmaClip = pexConfig.Field(
63 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
66 maskPlanes = pexConfig.ListField(
68 doc=
"Mask planes to reject when measuring overscan",
69 default=[
'BAD',
'SAT'],
71 overscanIsInt = pexConfig.Field(
73 doc=
"Treat overscan as an integer image for purposes of fitType=MEDIAN"
74 " and fitType=MEDIAN_PER_ROW.",
78 doParallelOverscan = pexConfig.Field(
80 doc=
"Correct using parallel overscan after serial overscan correction?",
83 parallelOverscanMaskThreshold = pexConfig.RangeField(
85 doc=
"Minimum fraction of pixels in parallel overscan region necessary "
86 "for parallel overcan correction.",
94 leadingColumnsToSkip = pexConfig.Field(
96 doc=
"Number of leading columns to skip in serial overscan correction.",
99 trailingColumnsToSkip = pexConfig.Field(
101 doc=
"Number of trailing columns to skip in serial overscan correction.",
104 leadingRowsToSkip = pexConfig.Field(
106 doc=
"Number of leading rows to skip in parallel overscan correction.",
109 trailingRowsToSkip = pexConfig.Field(
111 doc=
"Number of trailing rows to skip in parallel overscan correction.",
115 maxDeviation = pexConfig.Field(
117 doc=
"Maximum deviation from median (in ADU) to mask in overscan correction.",
118 default=1000.0, check=
lambda x: x > 0,
123 """Correction task for overscan.
125 This class contains a number of utilities that are easier to
126 understand
and use when they are
not embedded
in nested
if/
else
132 Statistics control object.
134 ConfigClass = OverscanCorrectionTaskConfig
135 _DefaultName = "overscan"
145 self.
statControl.setNumSigmaClip(self.config.numSigmaClip)
146 self.
statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes))
148 def run(self, exposure, amp, isTransposed=False):
149 """Measure and remove an overscan from an amplifier image.
154 Image data that will have the overscan corrections applied.
156 Amplifier to use for debugging purposes.
157 isTransposed : `bool`, optional
158 Is the image transposed, such that serial
and parallel
159 overscan regions are reversed? Default
is False.
163 overscanResults : `lsst.pipe.base.Struct`
164 Result struct
with components:
167 Value
or fit subtracted
from the amplifier image data
170 Value
or fit subtracted
from the overscan image data
173 Image of the overscan region
with the overscan
175 quantity
is used to estimate the amplifier read noise
181 Raised
if an invalid overscan type
is set.
184 serialOverscanBBox = amp.getRawSerialOverscanBBox()
185 imageBBox = amp.getRawDataBBox()
187 if self.config.doParallelOverscan:
190 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
191 imageBBox = imageBBox.expandedTo(parallelOverscanBBox)
194 imageBBox.getMinY()),
196 imageBBox.getHeight()))
198 imageBBox, serialOverscanBBox, isTransposed=isTransposed)
199 overscanMean = serialResults.overscanMean
200 overscanSigma = serialResults.overscanSigma
201 residualMean = serialResults.overscanMeanResidual
202 residualSigma = serialResults.overscanSigmaResidual
205 if self.config.doParallelOverscan:
208 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
209 imageBBox = amp.getRawDataBBox()
211 maskIm = exposure.getMaskedImage()
212 maskIm = maskIm.Factory(maskIm, parallelOverscanBBox)
215 xSize, ySize = parallelOverscanBBox.getDimensions()
216 if maskPix > xSize*ySize*self.config.parallelOverscanMaskThreshold:
217 self.log.warning(
'Fraction of masked pixels for parallel overscan calculation larger'
218 ' than %f of total pixels (i.e. %f masked pixels) on amp %s.',
219 self.config.parallelOverscanMaskThreshold, maskPix, amp.getName())
220 self.log.warning(
'Not doing parallel overscan correction.')
223 imageBBox, parallelOverscanBBox,
224 isTransposed=
not isTransposed)
226 overscanMean = (overscanMean, parallelResults.overscanMean)
227 overscanSigma = (overscanSigma, parallelResults.overscanSigma)
228 residualMean = (residualMean, parallelResults.overscanMeanResidual)
229 residualSigma = (residualSigma, parallelResults.overscanSigmaResidual)
231 return pipeBase.Struct(imageFit=serialResults.ampOverscanModel,
232 overscanFit=serialResults.overscanOverscanModel,
233 overscanImage=serialResults.overscanImage,
235 overscanMean=overscanMean,
236 overscanSigma=overscanSigma,
237 residualMean=residualMean,
238 residualSigma=residualSigma)
243 overscanBox = self.trimOverscan(exposure, amp, overscanBBox,
244 self.config.leadingColumnsToSkip,
245 self.config.trailingColumnsToSkip,
246 transpose=isTransposed)
247 overscanImage = exposure[overscanBox].getMaskedImage()
248 overscanArray = overscanImage.image.array
251 maskVal = overscanImage.mask.getPlaneBitMask(self.config.maskPlanes)
252 overscanMask = ~((overscanImage.mask.array & maskVal) == 0)
254 median = np.ma.median(np.ma.masked_where(overscanMask, overscanArray))
255 bad = np.where(np.abs(overscanArray - median) > self.config.maxDeviation)
256 overscanMask[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
260 overscanResults = self.
fitOverscan(overscanImage, isTransposed=isTransposed)
263 ampImage = exposure[imageBBox]
265 ampImage.image.array,
266 transpose=isTransposed)
267 ampImage.image.array -= ampOverscanModel
271 overscanImage = exposure[overscanBBox]
274 overscanImage.image.array)
275 overscanImage.image.array -= overscanOverscanModel
277 self.
debugView(overscanImage, overscanResults.overscanValue, amp)
280 stats = afwMath.makeStatistics(overscanImage.getMaskedImage(),
281 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
282 residualMean = stats.getValue(afwMath.MEDIAN)
283 residualSigma = stats.getValue(afwMath.STDEVCLIP)
285 return pipeBase.Struct(ampOverscanModel=ampOverscanModel,
286 overscanOverscanModel=overscanOverscanModel,
287 overscanImage=overscanImage,
288 overscanValue=overscanResults.overscanValue,
290 overscanMean=overscanResults.overscanMean,
291 overscanSigma=overscanResults.overscanSigma,
292 overscanMeanResidual=residualMean,
293 overscanSigmaResidual=residualSigma
297 """Broadcast 0 or 1 dimension fit to appropriate shape.
301 overscanValue : `numpy.ndarray`, (Nrows, ) or scalar
302 Overscan fit to broadcast.
303 imageArray : `numpy.ndarray`, (Nrows, Ncols)
304 Image array that we want to match.
305 transpose : `bool`, optional
306 Switch order to broadcast along the other axis.
310 overscanModel : `numpy.ndarray`, (Nrows, Ncols)
or scalar
311 Expanded overscan fit.
316 Raised
if no axis has the appropriate dimension.
318 if isinstance(overscanValue, np.ndarray):
319 overscanModel = np.zeros_like(imageArray)
321 if transpose
is False:
322 if imageArray.shape[0] == overscanValue.shape[0]:
323 overscanModel[:, :] = overscanValue[:, np.newaxis]
324 elif imageArray.shape[1] == overscanValue.shape[0]:
325 overscanModel[:, :] = overscanValue[np.newaxis, :]
326 elif imageArray.shape[0] == overscanValue.shape[1]:
327 overscanModel[:, :] = overscanValue[np.newaxis, :]
329 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
330 f
"match {imageArray.shape}")
332 if imageArray.shape[1] == overscanValue.shape[0]:
333 overscanModel[:, :] = overscanValue[np.newaxis, :]
334 elif imageArray.shape[0] == overscanValue.shape[0]:
335 overscanModel[:, :] = overscanValue[:, np.newaxis]
336 elif imageArray.shape[1] == overscanValue.shape[1]:
337 overscanModel[:, :] = overscanValue[:, np.newaxis]
339 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
340 f
"match {imageArray.shape}")
342 overscanModel = overscanValue
346 def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False):
347 """Trim overscan region to remove edges.
352 Exposure containing data.
354 Amplifier containing geometry information.
356 Bounding box of the overscan region.
358 Number of leading (towards data region) rows/columns to skip.
360 Number of trailing (away from data region) rows/columns to skip.
361 transpose : `bool`, optional
362 Operate on the transposed array.
366 overscanArray : `numpy.array`, (N, M)
368 overscanMask : `numpy.array`, (N, M)
371 dx0, dy0, dx1, dy1 = (0, 0, 0, 0)
372 dataBBox = amp.getRawDataBBox()
374 if dataBBox.getBeginY() < bbox.getBeginY():
381 if dataBBox.getBeginX() < bbox.getBeginX():
390 bbox.getHeight() - dy0 + dy1))
394 if self.config.fitType
in (
'MEAN',
'MEANCLIP',
'MEDIAN'):
397 overscanValue = overscanResult.overscanValue
398 overscanMean = overscanValue
400 elif self.config.fitType
in (
'MEDIAN_PER_ROW',
'POLY',
'CHEB',
'LEG',
401 'NATURAL_SPLINE',
'CUBIC_SPLINE',
'AKIMA_SPLINE'):
404 overscanValue = overscanResult.overscanValue
406 stats = afwMath.makeStatistics(overscanResult.overscanValue,
407 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
408 overscanMean = stats.getValue(afwMath.MEDIAN)
409 overscanSigma = stats.getValue(afwMath.STDEVCLIP)
411 raise ValueError(
'%s : %s an invalid overscan type' %
412 (
"overscanCorrection", self.config.fitType))
414 return pipeBase.Struct(overscanValue=overscanValue,
415 overscanMean=overscanMean,
416 overscanSigma=overscanSigma,
421 """Return an integer version of the input image.
426 Image to convert to integers.
431 The integer converted image.
436 Raised
if the input image could
not be converted.
438 if hasattr(image,
"image"):
440 imageI = image.image.convertI()
441 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
442 elif hasattr(image,
"convertI"):
444 outI = image.convertI()
445 elif hasattr(image,
"astype"):
447 outI = image.astype(int)
449 raise RuntimeError(
"Could not convert this to integers: %s %s %s",
450 image, type(image), dir(image))
455 """Measure a constant overscan value.
460 Image data to measure the overscan
from.
464 results : `lsst.pipe.base.Struct`
465 Overscan result
with entries:
466 - ``overscanValue``: Overscan value to subtract (`float`)
467 - ``isTransposed``: Orientation of the overscan (`bool`)
469 if self.config.fitType ==
'MEDIAN':
473 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
474 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.
statControl).getValue()
476 return pipeBase.Struct(overscanValue=overscanValue,
481 """Extract the numpy array from the input image.
486 Image data to pull array
from.
488 calcImage : `numpy.ndarray`
489 Image data array
for numpy operating.
491 if hasattr(image,
"getImage"):
492 calcImage = image.getImage().getArray()
493 calcImage = np.ma.masked_where(image.getMask().getArray() & self.
statControl.getAndMask(),
496 calcImage = image.getArray()
500 """Mask outliers in a row of overscan data
from a robust sigma
505 imageArray : `numpy.ndarray`
506 Image to filter along numpy axis=1.
510 maskedArray : `numpy.ma.masked_array`
511 Masked image marking outliers.
513 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
515 axisStdev = 0.74*(uq - lq)
517 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
518 return np.ma.masked_where(diff > self.
statControl.getNumSigmaClip()
519 * axisStdev[:, np.newaxis], imageArray)
523 """Collapse overscan array (and mask) to a 1-D vector of values.
527 maskedArray : `numpy.ma.masked_array`
528 Masked array of input overscan data.
532 collapsed : `numpy.ma.masked_array`
533 Single dimensional overscan data, combined with the mean.
535 collapsed = np.mean(maskedArray, axis=1)
536 if collapsed.mask.sum() > 0:
537 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
541 """Collapse overscan array (and mask) to a 1-D vector of using the
542 correct integer median of row-values.
546 maskedArray : `numpy.ma.masked_array`
547 Masked array of input overscan data.
551 collapsed : `numpy.ma.masked_array`
552 Single dimensional overscan data, combined with the afwMath median.
557 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
558 for row
in integerMI:
559 newRow = row.compressed()
561 rowMedian = afwMath.makeStatistics(newRow, fitType, self.
statControl).getValue()
564 collapsed.append(rowMedian)
566 return np.array(collapsed)
569 """Wrapper function to match spline fit API to polynomial fit API.
573 indices : `numpy.ndarray`
574 Locations to evaluate the spline.
575 collapsed : `numpy.ndarray`
576 Collapsed overscan values corresponding to the spline
579 Number of bins to use in constructing the spline.
584 Interpolation object
for later evaluation.
586 if not np.ma.is_masked(collapsed):
587 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
589 numPerBin, binEdges = np.histogram(indices, bins=numBins,
590 weights=1 - collapsed.mask.astype(int))
591 with np.errstate(invalid=
"ignore"):
592 values = np.histogram(indices, bins=numBins,
593 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
594 binCenters = np.histogram(indices, bins=numBins,
595 weights=indices*~collapsed.mask)[0]/numPerBin
597 if len(binCenters[numPerBin > 0]) < 5:
598 self.log.warn(
"Cannot do spline fitting for overscan: %s valid points.",
599 len(binCenters[numPerBin > 0]))
603 if len(values[numPerBin > 0]) != 0:
604 return float(values[numPerBin > 0][0])
608 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
609 values.astype(float)[numPerBin > 0],
610 afwMath.stringToInterpStyle(self.config.fitType))
615 """Wrapper function to match spline evaluation API to polynomial fit
620 indices : `numpy.ndarray`
621 Locations to evaluate the spline.
622 interp : `lsst.afw.math.interpolate`
623 Interpolation object to use.
627 values : `numpy.ndarray`
628 Evaluated spline values at each index.
631 return interp.interpolate(indices.astype(float))
635 """Create mask if edges are extrapolated.
639 collapsed : `numpy.ma.masked_array`
640 Masked array to check the edges of.
644 maskArray : `numpy.ndarray`
645 Boolean numpy array of pixels to mask.
647 maskArray = np.full_like(collapsed, False, dtype=bool)
648 if np.ma.is_masked(collapsed):
650 for low
in range(num):
651 if not collapsed.mask[low]:
654 maskArray[:low] =
True
655 for high
in range(1, num):
656 if not collapsed.mask[-high]:
659 maskArray[-high:] =
True
663 """Calculate the 1-d vector overscan from the input overscan image.
668 Image containing the overscan data.
669 isTransposed : `bool`
670 If true, the image has been transposed.
674 results : `lsst.pipe.base.Struct`
675 Overscan result with entries:
678 Overscan value to subtract (`float`)
680 List of rows that should be masked
as ``SUSPECT`` when the
681 overscan solution
is applied. (`list` [ `bool` ])
683 Indicates
if the overscan data was transposed during
684 calcuation, noting along which axis the overscan should be
691 calcImage = np.transpose(calcImage)
694 if self.config.fitType ==
'MEDIAN_PER_ROW':
695 mi = afwImage.MaskedImageI(image.getBBox())
696 masked = masked.astype(int)
698 masked = masked.transpose()
700 mi.image.array[:, :] = masked.data[:, :]
701 if bool(masked.mask.shape):
702 mi.mask.array[:, :] = masked.mask[:, :]
710 indices = 2.0*np.arange(num)/float(num) - 1.0
714 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
715 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
716 'LEG': (poly.legendre.legfit, poly.legendre.legval),
720 }[self.config.fitType]
724 coeffs = fitter(indices, collapsed, self.config.order)
726 if isinstance(coeffs, float):
727 self.log.warn(
"Using fallback value %f due to fitter failure. Amplifier will be masked.",
729 overscanVector = np.full_like(indices, coeffs)
730 maskArray = np.full_like(collapsed,
True, dtype=bool)
733 overscanVector = evaler(indices, coeffs)
736 return pipeBase.Struct(overscanValue=np.array(overscanVector),
738 isTransposed=isTransposed)
741 """Debug display for the final overscan solution.
746 Input image the overscan solution was determined from.
747 model : `numpy.ndarray`
or `float`
748 Overscan model determined
for the image.
750 Amplifier to extract diagnostic information.
760 calcImage = np.transpose(calcImage)
765 indices = 2.0 * np.arange(num)/float(num) - 1.0
767 if np.ma.is_masked(collapsed):
768 collapsedMask = collapsed.mask
770 collapsedMask = np.array(num*[np.ma.nomask])
772 import matplotlib.pyplot
as plot
773 figure = plot.figure(1)
775 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
776 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask],
'k+')
777 if collapsedMask.sum() > 0:
778 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask],
'b+')
779 if isinstance(model, np.ndarray):
782 plotModel = np.zeros_like(indices)
784 axes.plot(indices, plotModel,
'r-')
785 plot.xlabel(
"centered/scaled position along overscan region")
786 plot.ylabel(
"pixel value/fit value")
788 plot.title(f
"{amp.getName()} DataX: "
789 f
"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]"
790 f
"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:"
791 f
"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}")
793 plot.title(
"No amp supplied.")
795 prompt =
"Press Enter or c to continue [chp]..."
797 ans = input(prompt).lower()
798 if ans
in (
"",
" ",
"c",):
807 print(
"[h]elp [c]ontinue [p]db e[x]itDebug")
def collapseArrayMedian(self, maskedArray)
def maskExtrapolated(collapsed)
def fitOverscan(self, overscanImage, isTransposed=False)
def measureVectorOverscan(self, image, isTransposed=False)
def __init__(self, statControl=None, **kwargs)
def collapseArray(maskedArray)
def maskOutliers(self, imageArray)
def debugView(self, image, model, amp=None)
def getImageArray(self, image)
def integerConvert(image)
def splineFit(self, indices, collapsed, numBins)
def broadcastFitToImage(self, overscanValue, imageArray, transpose=False)
def correctOverscan(self, exposure, amp, imageBBox, overscanBBox, isTransposed=True)
def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False)
def run(self, exposure, amp, isTransposed=False)
def splineEval(indices, interp)
def measureConstantOverscan(self, image)
def countMaskedPixels(maskedIm, maskPlane)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
std::vector< double > fitOverscanImage(lsst::afw::image::MaskedImage< ImagePixelT > const &overscan, std::vector< std::string > badPixelMask, bool isTransposed)