22__all__ = [
"OverscanCorrectionTaskConfig",
"OverscanCorrectionTask"]
28import lsst.pipe.base
as pipeBase
31from .isr
import fitOverscanImage
32from .isrFunctions
import makeThresholdMask, countMaskedPixels
36 """Overscan correction options.
38 fitType = pexConfig.ChoiceField(
40 doc="The method for fitting the overscan bias level.",
43 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
44 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
45 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
46 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
47 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
48 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
49 "MEAN":
"Correct using the mean of the overscan region",
50 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
51 "MEDIAN":
"Correct using the median of the overscan region",
52 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
55 order = pexConfig.Field(
57 doc=(
"Order of polynomial to fit if overscan fit type is a polynomial, "
58 "or number of spline knots if overscan fit type is a spline."),
61 numSigmaClip = pexConfig.Field(
63 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
66 maskPlanes = pexConfig.ListField(
68 doc=
"Mask planes to reject when measuring overscan",
69 default=[
'BAD',
'SAT'],
71 overscanIsInt = pexConfig.Field(
73 doc=
"Treat overscan as an integer image for purposes of fitType=MEDIAN"
74 " and fitType=MEDIAN_PER_ROW.",
78 doParallelOverscan = pexConfig.Field(
80 doc=
"Correct using parallel overscan after serial overscan correction?",
83 parallelOverscanMaskThreshold = pexConfig.RangeField(
85 doc=
"Minimum fraction of pixels in parallel overscan region necessary "
86 "for parallel overcan correction.",
94 leadingColumnsToSkip = pexConfig.Field(
96 doc=
"Number of leading columns to skip in serial overscan correction.",
99 trailingColumnsToSkip = pexConfig.Field(
101 doc=
"Number of trailing columns to skip in serial overscan correction.",
104 leadingRowsToSkip = pexConfig.Field(
106 doc=
"Number of leading rows to skip in parallel overscan correction.",
109 trailingRowsToSkip = pexConfig.Field(
111 doc=
"Number of trailing rows to skip in parallel overscan correction.",
115 maxDeviation = pexConfig.Field(
117 doc=
"Maximum deviation from median (in ADU) to mask in overscan correction.",
118 default=1000.0, check=
lambda x: x > 0,
123 """Correction task for overscan.
125 This class contains a number of utilities that are easier to
126 understand
and use when they are
not embedded
in nested
if/
else
132 Statistics control object.
134 ConfigClass = OverscanCorrectionTaskConfig
135 _DefaultName = "overscan"
145 self.
statControl.setNumSigmaClip(self.config.numSigmaClip)
146 self.
statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes))
148 def run(self, exposure, amp, isTransposed=False):
149 """Measure and remove an overscan from an amplifier image.
154 Image data that will have the overscan corrections applied.
156 Amplifier to use for debugging purposes.
157 isTransposed : `bool`, optional
158 Is the image transposed, such that serial
and parallel
159 overscan regions are reversed? Default
is False.
163 overscanResults : `lsst.pipe.base.Struct`
164 Result struct
with components:
167 Value
or fit subtracted
from the amplifier image data
170 Value
or fit subtracted
from the serial overscan image
173 Image of the serial overscan region
with the serial
174 overscan correction applied
176 estimate the amplifier read noise empirically.
177 ``parallelOverscanFit``
178 Value
or fit subtracted
from the parallel overscan
180 ``parallelOverscanImage``
181 Image of the parallel overscan region
with the
182 parallel overscan correction applied
188 Raised
if an invalid overscan type
is set.
191 serialOverscanBBox = amp.getRawSerialOverscanBBox()
192 imageBBox = amp.getRawDataBBox()
194 if self.config.doParallelOverscan:
197 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
198 imageBBox = imageBBox.expandedTo(parallelOverscanBBox)
201 imageBBox.getMinY()),
203 imageBBox.getHeight()))
205 imageBBox, serialOverscanBBox, isTransposed=isTransposed)
206 overscanMean = serialResults.overscanMean
207 overscanSigma = serialResults.overscanSigma
208 residualMean = serialResults.overscanMeanResidual
209 residualSigma = serialResults.overscanSigmaResidual
212 parallelResults =
None
213 if self.config.doParallelOverscan:
216 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
217 imageBBox = amp.getRawDataBBox()
219 maskIm = exposure.getMaskedImage()
220 maskIm = maskIm.Factory(maskIm, parallelOverscanBBox)
223 xSize, ySize = parallelOverscanBBox.getDimensions()
224 if maskPix > xSize*ySize*self.config.parallelOverscanMaskThreshold:
225 self.log.warning(
'Fraction of masked pixels for parallel overscan calculation larger'
226 ' than %f of total pixels (i.e. %f masked pixels) on amp %s.',
227 self.config.parallelOverscanMaskThreshold, maskPix, amp.getName())
228 self.log.warning(
'Not doing parallel overscan correction.')
231 imageBBox, parallelOverscanBBox,
232 isTransposed=
not isTransposed)
234 overscanMean = (overscanMean, parallelResults.overscanMean)
235 overscanSigma = (overscanSigma, parallelResults.overscanSigma)
236 residualMean = (residualMean, parallelResults.overscanMeanResidual)
237 residualSigma = (residualSigma, parallelResults.overscanSigmaResidual)
238 parallelOverscanFit = parallelResults.overscanOverscanModel
if parallelResults
else None
239 parallelOverscanImage = parallelResults.overscanImage
if parallelResults
else None
241 return pipeBase.Struct(imageFit=serialResults.ampOverscanModel,
242 overscanFit=serialResults.overscanOverscanModel,
243 overscanImage=serialResults.overscanImage,
245 parallelOverscanFit=parallelOverscanFit,
246 parallelOverscanImage=parallelOverscanImage,
247 overscanMean=overscanMean,
248 overscanSigma=overscanSigma,
249 residualMean=residualMean,
250 residualSigma=residualSigma)
255 overscanBox = self.trimOverscan(exposure, amp, overscanBBox,
256 self.config.leadingColumnsToSkip,
257 self.config.trailingColumnsToSkip,
258 transpose=isTransposed)
259 overscanImage = exposure[overscanBox].getMaskedImage()
260 overscanArray = overscanImage.image.array
263 maskVal = overscanImage.mask.getPlaneBitMask(self.config.maskPlanes)
264 overscanMask = ~((overscanImage.mask.array & maskVal) == 0)
266 median = np.ma.median(np.ma.masked_where(overscanMask, overscanArray))
267 bad = np.where(np.abs(overscanArray - median) > self.config.maxDeviation)
268 overscanMask[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
272 overscanResults = self.
fitOverscan(overscanImage, isTransposed=isTransposed)
275 ampImage = exposure[imageBBox]
277 ampImage.image.array,
278 transpose=isTransposed)
279 ampImage.image.array -= ampOverscanModel
283 overscanImage = exposure[overscanBBox]
286 overscanImage.image.array)
287 overscanImage.image.array -= overscanOverscanModel
289 self.
debugView(overscanImage, overscanResults.overscanValue, amp)
292 stats = afwMath.makeStatistics(overscanImage.getMaskedImage(),
293 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
294 residualMean = stats.getValue(afwMath.MEDIAN)
295 residualSigma = stats.getValue(afwMath.STDEVCLIP)
297 return pipeBase.Struct(ampOverscanModel=ampOverscanModel,
298 overscanOverscanModel=overscanOverscanModel,
299 overscanImage=overscanImage,
300 overscanValue=overscanResults.overscanValue,
302 overscanMean=overscanResults.overscanMean,
303 overscanSigma=overscanResults.overscanSigma,
304 overscanMeanResidual=residualMean,
305 overscanSigmaResidual=residualSigma
309 """Broadcast 0 or 1 dimension fit to appropriate shape.
313 overscanValue : `numpy.ndarray`, (Nrows, ) or scalar
314 Overscan fit to broadcast.
315 imageArray : `numpy.ndarray`, (Nrows, Ncols)
316 Image array that we want to match.
317 transpose : `bool`, optional
318 Switch order to broadcast along the other axis.
322 overscanModel : `numpy.ndarray`, (Nrows, Ncols)
or scalar
323 Expanded overscan fit.
328 Raised
if no axis has the appropriate dimension.
330 if isinstance(overscanValue, np.ndarray):
331 overscanModel = np.zeros_like(imageArray)
333 if transpose
is False:
334 if imageArray.shape[0] == overscanValue.shape[0]:
335 overscanModel[:, :] = overscanValue[:, np.newaxis]
336 elif imageArray.shape[1] == overscanValue.shape[0]:
337 overscanModel[:, :] = overscanValue[np.newaxis, :]
338 elif imageArray.shape[0] == overscanValue.shape[1]:
339 overscanModel[:, :] = overscanValue[np.newaxis, :]
341 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
342 f
"match {imageArray.shape}")
344 if imageArray.shape[1] == overscanValue.shape[0]:
345 overscanModel[:, :] = overscanValue[np.newaxis, :]
346 elif imageArray.shape[0] == overscanValue.shape[0]:
347 overscanModel[:, :] = overscanValue[:, np.newaxis]
348 elif imageArray.shape[1] == overscanValue.shape[1]:
349 overscanModel[:, :] = overscanValue[:, np.newaxis]
351 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
352 f
"match {imageArray.shape}")
354 overscanModel = overscanValue
358 def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False):
359 """Trim overscan region to remove edges.
364 Exposure containing data.
366 Amplifier containing geometry information.
368 Bounding box of the overscan region.
370 Number of leading (towards data region) rows/columns to skip.
372 Number of trailing (away from data region) rows/columns to skip.
373 transpose : `bool`, optional
374 Operate on the transposed array.
378 overscanArray : `numpy.array`, (N, M)
380 overscanMask : `numpy.array`, (N, M)
383 dx0, dy0, dx1, dy1 = (0, 0, 0, 0)
384 dataBBox = amp.getRawDataBBox()
386 if dataBBox.getBeginY() < bbox.getBeginY():
393 if dataBBox.getBeginX() < bbox.getBeginX():
402 bbox.getHeight() - dy0 + dy1))
406 if self.config.fitType
in (
'MEAN',
'MEANCLIP',
'MEDIAN'):
409 overscanValue = overscanResult.overscanValue
410 overscanMean = overscanValue
412 elif self.config.fitType
in (
'MEDIAN_PER_ROW',
'POLY',
'CHEB',
'LEG',
413 'NATURAL_SPLINE',
'CUBIC_SPLINE',
'AKIMA_SPLINE'):
416 overscanValue = overscanResult.overscanValue
418 stats = afwMath.makeStatistics(overscanResult.overscanValue,
419 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
420 overscanMean = stats.getValue(afwMath.MEDIAN)
421 overscanSigma = stats.getValue(afwMath.STDEVCLIP)
423 raise ValueError(
'%s : %s an invalid overscan type' %
424 (
"overscanCorrection", self.config.fitType))
426 return pipeBase.Struct(overscanValue=overscanValue,
427 overscanMean=overscanMean,
428 overscanSigma=overscanSigma,
433 """Return an integer version of the input image.
438 Image to convert to integers.
443 The integer converted image.
448 Raised
if the input image could
not be converted.
450 if hasattr(image,
"image"):
452 imageI = image.image.convertI()
453 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
454 elif hasattr(image,
"convertI"):
456 outI = image.convertI()
457 elif hasattr(image,
"astype"):
459 outI = image.astype(int)
461 raise RuntimeError(
"Could not convert this to integers: %s %s %s",
462 image, type(image), dir(image))
467 """Measure a constant overscan value.
472 Image data to measure the overscan
from.
476 results : `lsst.pipe.base.Struct`
477 Overscan result
with entries:
478 - ``overscanValue``: Overscan value to subtract (`float`)
479 - ``isTransposed``: Orientation of the overscan (`bool`)
481 if self.config.fitType ==
'MEDIAN':
485 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
486 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.
statControl).getValue()
488 return pipeBase.Struct(overscanValue=overscanValue,
493 """Extract the numpy array from the input image.
498 Image data to pull array
from.
500 calcImage : `numpy.ndarray`
501 Image data array
for numpy operating.
503 if hasattr(image,
"getImage"):
504 calcImage = image.getImage().getArray()
505 calcImage = np.ma.masked_where(image.getMask().getArray() & self.
statControl.getAndMask(),
508 calcImage = image.getArray()
512 """Mask outliers in a row of overscan data
from a robust sigma
517 imageArray : `numpy.ndarray`
518 Image to filter along numpy axis=1.
522 maskedArray : `numpy.ma.masked_array`
523 Masked image marking outliers.
525 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
527 axisStdev = 0.74*(uq - lq)
529 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
530 return np.ma.masked_where(diff > self.
statControl.getNumSigmaClip()
531 * axisStdev[:, np.newaxis], imageArray)
535 """Collapse overscan array (and mask) to a 1-D vector of values.
539 maskedArray : `numpy.ma.masked_array`
540 Masked array of input overscan data.
544 collapsed : `numpy.ma.masked_array`
545 Single dimensional overscan data, combined with the mean.
547 collapsed = np.mean(maskedArray, axis=1)
548 if collapsed.mask.sum() > 0:
549 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
553 """Collapse overscan array (and mask) to a 1-D vector of using the
554 correct integer median of row-values.
558 maskedArray : `numpy.ma.masked_array`
559 Masked array of input overscan data.
563 collapsed : `numpy.ma.masked_array`
564 Single dimensional overscan data, combined with the afwMath median.
569 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
570 for row
in integerMI:
571 newRow = row.compressed()
573 rowMedian = afwMath.makeStatistics(newRow, fitType, self.
statControl).getValue()
576 collapsed.append(rowMedian)
578 return np.array(collapsed)
581 """Wrapper function to match spline fit API to polynomial fit API.
585 indices : `numpy.ndarray`
586 Locations to evaluate the spline.
587 collapsed : `numpy.ndarray`
588 Collapsed overscan values corresponding to the spline
591 Number of bins to use in constructing the spline.
596 Interpolation object
for later evaluation.
598 if not np.ma.is_masked(collapsed):
599 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
601 numPerBin, binEdges = np.histogram(indices, bins=numBins,
602 weights=1 - collapsed.mask.astype(int))
603 with np.errstate(invalid=
"ignore"):
604 values = np.histogram(indices, bins=numBins,
605 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
606 binCenters = np.histogram(indices, bins=numBins,
607 weights=indices*~collapsed.mask)[0]/numPerBin
609 if len(binCenters[numPerBin > 0]) < 5:
610 self.log.warn(
"Cannot do spline fitting for overscan: %s valid points.",
611 len(binCenters[numPerBin > 0]))
615 if len(values[numPerBin > 0]) != 0:
616 return float(values[numPerBin > 0][0])
620 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
621 values.astype(float)[numPerBin > 0],
622 afwMath.stringToInterpStyle(self.config.fitType))
627 """Wrapper function to match spline evaluation API to polynomial fit
632 indices : `numpy.ndarray`
633 Locations to evaluate the spline.
634 interp : `lsst.afw.math.interpolate`
635 Interpolation object to use.
639 values : `numpy.ndarray`
640 Evaluated spline values at each index.
643 return interp.interpolate(indices.astype(float))
647 """Create mask if edges are extrapolated.
651 collapsed : `numpy.ma.masked_array`
652 Masked array to check the edges of.
656 maskArray : `numpy.ndarray`
657 Boolean numpy array of pixels to mask.
659 maskArray = np.full_like(collapsed, False, dtype=bool)
660 if np.ma.is_masked(collapsed):
662 for low
in range(num):
663 if not collapsed.mask[low]:
666 maskArray[:low] =
True
667 for high
in range(1, num):
668 if not collapsed.mask[-high]:
671 maskArray[-high:] =
True
675 """Calculate the 1-d vector overscan from the input overscan image.
680 Image containing the overscan data.
681 isTransposed : `bool`
682 If true, the image has been transposed.
686 results : `lsst.pipe.base.Struct`
687 Overscan result with entries:
690 Overscan value to subtract (`float`)
692 List of rows that should be masked
as ``SUSPECT`` when the
693 overscan solution
is applied. (`list` [ `bool` ])
695 Indicates
if the overscan data was transposed during
696 calcuation, noting along which axis the overscan should be
703 calcImage = np.transpose(calcImage)
706 if self.config.fitType ==
'MEDIAN_PER_ROW':
707 mi = afwImage.MaskedImageI(image.getBBox())
708 masked = masked.astype(int)
710 masked = masked.transpose()
712 mi.image.array[:, :] = masked.data[:, :]
713 if bool(masked.mask.shape):
714 mi.mask.array[:, :] = masked.mask[:, :]
722 indices = 2.0*np.arange(num)/float(num) - 1.0
726 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
727 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
728 'LEG': (poly.legendre.legfit, poly.legendre.legval),
732 }[self.config.fitType]
736 coeffs = fitter(indices, collapsed, self.config.order)
738 if isinstance(coeffs, float):
739 self.log.warn(
"Using fallback value %f due to fitter failure. Amplifier will be masked.",
741 overscanVector = np.full_like(indices, coeffs)
742 maskArray = np.full_like(collapsed,
True, dtype=bool)
745 overscanVector = evaler(indices, coeffs)
748 return pipeBase.Struct(overscanValue=np.array(overscanVector),
750 isTransposed=isTransposed)
753 """Debug display for the final overscan solution.
758 Input image the overscan solution was determined from.
759 model : `numpy.ndarray`
or `float`
760 Overscan model determined
for the image.
762 Amplifier to extract diagnostic information.
772 calcImage = np.transpose(calcImage)
777 indices = 2.0 * np.arange(num)/float(num) - 1.0
779 if np.ma.is_masked(collapsed):
780 collapsedMask = collapsed.mask
782 collapsedMask = np.array(num*[np.ma.nomask])
784 import matplotlib.pyplot
as plot
785 figure = plot.figure(1)
787 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
788 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask],
'k+')
789 if collapsedMask.sum() > 0:
790 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask],
'b+')
791 if isinstance(model, np.ndarray):
794 plotModel = np.zeros_like(indices)
796 axes.plot(indices, plotModel,
'r-')
797 plot.xlabel(
"centered/scaled position along overscan region")
798 plot.ylabel(
"pixel value/fit value")
800 plot.title(f
"{amp.getName()} DataX: "
801 f
"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]"
802 f
"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:"
803 f
"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}")
805 plot.title(
"No amp supplied.")
807 prompt =
"Press Enter or c to continue [chp]..."
809 ans = input(prompt).lower()
810 if ans
in (
"",
" ",
"c",):
819 print(
"[h]elp [c]ontinue [p]db e[x]itDebug")
def collapseArrayMedian(self, maskedArray)
def maskExtrapolated(collapsed)
def fitOverscan(self, overscanImage, isTransposed=False)
def measureVectorOverscan(self, image, isTransposed=False)
def __init__(self, statControl=None, **kwargs)
def collapseArray(maskedArray)
def maskOutliers(self, imageArray)
def debugView(self, image, model, amp=None)
def getImageArray(self, image)
def integerConvert(image)
def splineFit(self, indices, collapsed, numBins)
def broadcastFitToImage(self, overscanValue, imageArray, transpose=False)
def correctOverscan(self, exposure, amp, imageBBox, overscanBBox, isTransposed=True)
def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False)
def run(self, exposure, amp, isTransposed=False)
def splineEval(indices, interp)
def measureConstantOverscan(self, image)
def countMaskedPixels(maskedIm, maskPlane)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
std::vector< double > fitOverscanImage(lsst::afw::image::MaskedImage< ImagePixelT > const &overscan, std::vector< std::string > badPixelMask, bool isTransposed)