22__all__ = [
"OverscanCorrectionTaskConfig",
"OverscanCorrectionTask"]
28import lsst.pipe.base
as pipeBase
31from .isr
import fitOverscanImage
32from .isrFunctions
import makeThresholdMask, countMaskedPixels
36 """Overscan correction options.
38 fitType = pexConfig.ChoiceField(
40 doc="The method for fitting the overscan bias level.",
43 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
44 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
45 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
46 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
47 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
48 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
49 "MEAN":
"Correct using the mean of the overscan region",
50 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
51 "MEDIAN":
"Correct using the median of the overscan region",
52 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
55 order = pexConfig.Field(
57 doc=(
"Order of polynomial to fit if overscan fit type is a polynomial, "
58 "or number of spline knots if overscan fit type is a spline."),
61 numSigmaClip = pexConfig.Field(
63 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
66 maskPlanes = pexConfig.ListField(
68 doc=
"Mask planes to reject when measuring overscan",
69 default=[
'BAD',
'SAT'],
71 overscanIsInt = pexConfig.Field(
73 doc=
"Treat overscan as an integer image for purposes of fitType=MEDIAN"
74 " and fitType=MEDIAN_PER_ROW.",
78 doParallelOverscan = pexConfig.Field(
80 doc=
"Correct using parallel overscan after serial overscan correction?",
83 parallelOverscanMaskThreshold = pexConfig.RangeField(
85 doc=
"Minimum fraction of pixels in parallel overscan region necessary "
86 "for parallel overcan correction.",
94 leadingColumnsToSkip = pexConfig.Field(
96 doc=
"Number of leading columns to skip in serial overscan correction.",
99 trailingColumnsToSkip = pexConfig.Field(
101 doc=
"Number of trailing columns to skip in serial overscan correction.",
104 leadingRowsToSkip = pexConfig.Field(
106 doc=
"Number of leading rows to skip in parallel overscan correction.",
109 trailingRowsToSkip = pexConfig.Field(
111 doc=
"Number of trailing rows to skip in parallel overscan correction.",
115 maxDeviation = pexConfig.Field(
117 doc=
"Maximum deviation from median (in ADU) to mask in overscan correction.",
118 default=1000.0, check=
lambda x: x > 0,
123 """Correction task for overscan.
125 This class contains a number of utilities that are easier to
126 understand
and use when they are
not embedded
in nested
if/
else
132 Statistics control object.
134 ConfigClass = OverscanCorrectionTaskConfig
135 _DefaultName = "overscan"
145 self.
statControl.setNumSigmaClip(self.config.numSigmaClip)
146 self.
statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes))
148 def run(self, exposure, amp, isTransposed=False):
149 """Measure and remove an overscan from an amplifier image.
154 Image data that will have the overscan corrections applied.
156 Amplifier to use for debugging purposes.
157 isTransposed : `bool`, optional
158 Is the image transposed, such that serial
and parallel
159 overscan regions are reversed? Default
is False.
163 overscanResults : `lsst.pipe.base.Struct`
164 Result struct
with components:
167 Value
or fit subtracted
from the amplifier image data
170 Value
or fit subtracted
from the serial overscan image
173 Image of the serial overscan region
with the serial
174 overscan correction applied
176 estimate the amplifier read noise empirically.
177 ``parallelOverscanFit``
178 Value
or fit subtracted
from the parallel overscan
180 ``parallelOverscanImage``
181 Image of the parallel overscan region
with the
182 parallel overscan correction applied
188 Raised
if an invalid overscan type
is set.
191 serialOverscanBBox = amp.getRawSerialOverscanBBox()
192 imageBBox = amp.getRawDataBBox()
194 if self.config.doParallelOverscan:
197 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
198 imageBBox = imageBBox.expandedTo(parallelOverscanBBox)
201 imageBBox.getMinY()),
203 imageBBox.getHeight()))
205 imageBBox, serialOverscanBBox, isTransposed=isTransposed)
206 overscanMean = serialResults.overscanMean
207 overscanMedian = serialResults.overscanMedian
208 overscanSigma = serialResults.overscanSigma
209 residualMean = serialResults.overscanMeanResidual
210 residualMedian = serialResults.overscanMedianResidual
211 residualSigma = serialResults.overscanSigmaResidual
214 parallelResults =
None
215 if self.config.doParallelOverscan:
218 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
219 imageBBox = amp.getRawDataBBox()
221 maskIm = exposure.getMaskedImage()
222 maskIm = maskIm.Factory(maskIm, parallelOverscanBBox)
230 thresholdLevel = self.config.numSigmaClip * serialResults.overscanSigmaResidual
231 makeThresholdMask(maskIm, threshold=thresholdLevel, growFootprints=0)
232 maskPix = countMaskedPixels(maskIm, self.config.maskPlanes)
233 xSize, ySize = parallelOverscanBBox.getDimensions()
234 if maskPix > xSize*ySize*self.config.parallelOverscanMaskThreshold:
235 self.log.warning(
'Fraction of masked pixels for parallel overscan calculation larger'
236 ' than %f of total pixels (i.e. %f masked pixels) on amp %s.',
237 self.config.parallelOverscanMaskThreshold, maskPix, amp.getName())
238 self.log.warning(
'Not doing parallel overscan correction.')
241 imageBBox, parallelOverscanBBox,
242 isTransposed=
not isTransposed)
244 overscanMean = (overscanMean, parallelResults.overscanMean)
245 overscanMedian = (overscanMedian, parallelResults.overscanMedian)
246 overscanSigma = (overscanSigma, parallelResults.overscanSigma)
247 residualMean = (residualMean, parallelResults.overscanMeanResidual)
248 residualMedian = (residualMedian, parallelResults.overscanMedianResidual)
249 residualSigma = (residualSigma, parallelResults.overscanSigmaResidual)
250 parallelOverscanFit = parallelResults.overscanOverscanModel
if parallelResults
else None
251 parallelOverscanImage = parallelResults.overscanImage
if parallelResults
else None
253 return pipeBase.Struct(imageFit=serialResults.ampOverscanModel,
254 overscanFit=serialResults.overscanOverscanModel,
255 overscanImage=serialResults.overscanImage,
257 parallelOverscanFit=parallelOverscanFit,
258 parallelOverscanImage=parallelOverscanImage,
259 overscanMean=overscanMean,
260 overscanMedian=overscanMedian,
261 overscanSigma=overscanSigma,
262 residualMean=residualMean,
263 residualMedian=residualMedian,
264 residualSigma=residualSigma)
269 overscanBox = self.trimOverscan(exposure, amp, overscanBBox,
270 self.config.leadingColumnsToSkip,
271 self.config.trailingColumnsToSkip,
272 transpose=isTransposed)
273 overscanImage = exposure[overscanBox].getMaskedImage()
274 overscanArray = overscanImage.image.array
277 maskVal = overscanImage.mask.getPlaneBitMask(self.config.maskPlanes)
278 overscanMask = ~((overscanImage.mask.array & maskVal) == 0)
280 median = np.ma.median(np.ma.masked_where(overscanMask, overscanArray))
281 bad = np.where(np.abs(overscanArray - median) > self.config.maxDeviation)
282 overscanMask[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
286 overscanResults = self.
fitOverscan(overscanImage, isTransposed=isTransposed)
289 ampImage = exposure[imageBBox]
291 ampImage.image.array,
292 transpose=isTransposed)
293 ampImage.image.array -= ampOverscanModel
297 overscanImage = exposure[overscanBBox]
300 overscanImage.image.array)
301 overscanImage.image.array -= overscanOverscanModel
303 self.
debugView(overscanImage, overscanResults.overscanValue, amp)
306 stats = afwMath.makeStatistics(overscanImage.getMaskedImage(),
307 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
308 residualMean = stats.getValue(afwMath.MEAN)
309 residualMedian = stats.getValue(afwMath.MEDIAN)
310 residualSigma = stats.getValue(afwMath.STDEVCLIP)
312 return pipeBase.Struct(ampOverscanModel=ampOverscanModel,
313 overscanOverscanModel=overscanOverscanModel,
314 overscanImage=overscanImage,
315 overscanValue=overscanResults.overscanValue,
317 overscanMean=overscanResults.overscanMean,
318 overscanMedian=overscanResults.overscanMedian,
319 overscanSigma=overscanResults.overscanSigma,
320 overscanMeanResidual=residualMean,
321 overscanMedianResidual=residualMedian,
322 overscanSigmaResidual=residualSigma
326 """Broadcast 0 or 1 dimension fit to appropriate shape.
330 overscanValue : `numpy.ndarray`, (Nrows, ) or scalar
331 Overscan fit to broadcast.
332 imageArray : `numpy.ndarray`, (Nrows, Ncols)
333 Image array that we want to match.
334 transpose : `bool`, optional
335 Switch order to broadcast along the other axis.
339 overscanModel : `numpy.ndarray`, (Nrows, Ncols)
or scalar
340 Expanded overscan fit.
345 Raised
if no axis has the appropriate dimension.
347 if isinstance(overscanValue, np.ndarray):
348 overscanModel = np.zeros_like(imageArray)
350 if transpose
is False:
351 if imageArray.shape[0] == overscanValue.shape[0]:
352 overscanModel[:, :] = overscanValue[:, np.newaxis]
353 elif imageArray.shape[1] == overscanValue.shape[0]:
354 overscanModel[:, :] = overscanValue[np.newaxis, :]
355 elif imageArray.shape[0] == overscanValue.shape[1]:
356 overscanModel[:, :] = overscanValue[np.newaxis, :]
358 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
359 f
"match {imageArray.shape}")
361 if imageArray.shape[1] == overscanValue.shape[0]:
362 overscanModel[:, :] = overscanValue[np.newaxis, :]
363 elif imageArray.shape[0] == overscanValue.shape[0]:
364 overscanModel[:, :] = overscanValue[:, np.newaxis]
365 elif imageArray.shape[1] == overscanValue.shape[1]:
366 overscanModel[:, :] = overscanValue[:, np.newaxis]
368 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
369 f
"match {imageArray.shape}")
371 overscanModel = overscanValue
375 def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False):
376 """Trim overscan region to remove edges.
381 Exposure containing data.
383 Amplifier containing geometry information.
385 Bounding box of the overscan region.
387 Number of leading (towards data region) rows/columns to skip.
389 Number of trailing (away from data region) rows/columns to skip.
390 transpose : `bool`, optional
391 Operate on the transposed array.
395 overscanArray : `numpy.array`, (N, M)
397 overscanMask : `numpy.array`, (N, M)
400 dx0, dy0, dx1, dy1 = (0, 0, 0, 0)
401 dataBBox = amp.getRawDataBBox()
403 if dataBBox.getBeginY() < bbox.getBeginY():
410 if dataBBox.getBeginX() < bbox.getBeginX():
419 bbox.getHeight() - dy0 + dy1))
423 if self.config.fitType
in (
'MEAN',
'MEANCLIP',
'MEDIAN'):
426 overscanValue = overscanResult.overscanValue
427 overscanMean = overscanValue
428 overscanMedian = overscanValue
430 elif self.config.fitType
in (
'MEDIAN_PER_ROW',
'POLY',
'CHEB',
'LEG',
431 'NATURAL_SPLINE',
'CUBIC_SPLINE',
'AKIMA_SPLINE'):
434 overscanValue = overscanResult.overscanValue
436 stats = afwMath.makeStatistics(overscanResult.overscanValue,
437 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP,
439 overscanMean = stats.getValue(afwMath.MEAN)
440 overscanMedian = stats.getValue(afwMath.MEDIAN)
441 overscanSigma = stats.getValue(afwMath.STDEVCLIP)
443 raise ValueError(
'%s : %s an invalid overscan type' %
444 (
"overscanCorrection", self.config.fitType))
446 return pipeBase.Struct(overscanValue=overscanValue,
447 overscanMean=overscanMean,
448 overscanMedian=overscanMedian,
449 overscanSigma=overscanSigma,
454 """Return an integer version of the input image.
459 Image to convert to integers.
464 The integer converted image.
469 Raised
if the input image could
not be converted.
471 if hasattr(image,
"image"):
473 imageI = image.image.convertI()
474 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
475 elif hasattr(image,
"convertI"):
477 outI = image.convertI()
478 elif hasattr(image,
"astype"):
480 outI = image.astype(int)
482 raise RuntimeError(
"Could not convert this to integers: %s %s %s",
483 image, type(image), dir(image))
488 """Measure a constant overscan value.
493 Image data to measure the overscan
from.
497 results : `lsst.pipe.base.Struct`
498 Overscan result
with entries:
499 - ``overscanValue``: Overscan value to subtract (`float`)
500 - ``isTransposed``: Orientation of the overscan (`bool`)
502 if self.config.fitType ==
'MEDIAN':
506 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
507 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.
statControl).getValue()
509 return pipeBase.Struct(overscanValue=overscanValue,
514 """Extract the numpy array from the input image.
519 Image data to pull array
from.
521 calcImage : `numpy.ndarray`
522 Image data array
for numpy operating.
524 if hasattr(image,
"getImage"):
525 calcImage = image.getImage().getArray()
526 calcImage = np.ma.masked_where(image.getMask().getArray() & self.
statControl.getAndMask(),
529 calcImage = image.getArray()
533 """Mask outliers in a row of overscan data
from a robust sigma
538 imageArray : `numpy.ndarray`
539 Image to filter along numpy axis=1.
543 maskedArray : `numpy.ma.masked_array`
544 Masked image marking outliers.
546 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
548 axisStdev = 0.74*(uq - lq)
550 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
551 return np.ma.masked_where(diff > self.
statControl.getNumSigmaClip()
552 * axisStdev[:, np.newaxis], imageArray)
556 """Collapse overscan array (and mask) to a 1-D vector of values.
560 maskedArray : `numpy.ma.masked_array`
561 Masked array of input overscan data.
565 collapsed : `numpy.ma.masked_array`
566 Single dimensional overscan data, combined with the mean.
568 collapsed = np.mean(maskedArray, axis=1)
569 if collapsed.mask.sum() > 0:
570 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
574 """Collapse overscan array (and mask) to a 1-D vector of using the
575 correct integer median of row-values.
579 maskedArray : `numpy.ma.masked_array`
580 Masked array of input overscan data.
584 collapsed : `numpy.ma.masked_array`
585 Single dimensional overscan data, combined with the afwMath median.
590 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
591 for row
in integerMI:
592 newRow = row.compressed()
594 rowMedian = afwMath.makeStatistics(newRow, fitType, self.
statControl).getValue()
597 collapsed.append(rowMedian)
599 return np.array(collapsed)
602 """Wrapper function to match spline fit API to polynomial fit API.
606 indices : `numpy.ndarray`
607 Locations to evaluate the spline.
608 collapsed : `numpy.ndarray`
609 Collapsed overscan values corresponding to the spline
612 Number of bins to use in constructing the spline.
617 Interpolation object
for later evaluation.
619 if not np.ma.is_masked(collapsed):
620 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
622 numPerBin, binEdges = np.histogram(indices, bins=numBins,
623 weights=1 - collapsed.mask.astype(int))
624 with np.errstate(invalid=
"ignore"):
625 values = np.histogram(indices, bins=numBins,
626 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
627 binCenters = np.histogram(indices, bins=numBins,
628 weights=indices*~collapsed.mask)[0]/numPerBin
630 if len(binCenters[numPerBin > 0]) < 5:
631 self.log.warn(
"Cannot do spline fitting for overscan: %s valid points.",
632 len(binCenters[numPerBin > 0]))
636 if len(values[numPerBin > 0]) != 0:
637 return float(values[numPerBin > 0][0])
641 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
642 values.astype(float)[numPerBin > 0],
643 afwMath.stringToInterpStyle(self.config.fitType))
648 """Wrapper function to match spline evaluation API to polynomial fit
653 indices : `numpy.ndarray`
654 Locations to evaluate the spline.
655 interp : `lsst.afw.math.interpolate`
656 Interpolation object to use.
660 values : `numpy.ndarray`
661 Evaluated spline values at each index.
664 return interp.interpolate(indices.astype(float))
668 """Create mask if edges are extrapolated.
672 collapsed : `numpy.ma.masked_array`
673 Masked array to check the edges of.
677 maskArray : `numpy.ndarray`
678 Boolean numpy array of pixels to mask.
680 maskArray = np.full_like(collapsed, False, dtype=bool)
681 if np.ma.is_masked(collapsed):
683 for low
in range(num):
684 if not collapsed.mask[low]:
687 maskArray[:low] =
True
688 for high
in range(1, num):
689 if not collapsed.mask[-high]:
692 maskArray[-high:] =
True
696 """Calculate the 1-d vector overscan from the input overscan image.
701 Image containing the overscan data.
702 isTransposed : `bool`
703 If true, the image has been transposed.
707 results : `lsst.pipe.base.Struct`
708 Overscan result with entries:
711 Overscan value to subtract (`float`)
713 List of rows that should be masked
as ``SUSPECT`` when the
714 overscan solution
is applied. (`list` [ `bool` ])
716 Indicates
if the overscan data was transposed during
717 calcuation, noting along which axis the overscan should be
724 calcImage = np.transpose(calcImage)
727 if self.config.fitType ==
'MEDIAN_PER_ROW':
728 mi = afwImage.MaskedImageI(image.getBBox())
729 masked = masked.astype(int)
731 masked = masked.transpose()
733 mi.image.array[:, :] = masked.data[:, :]
734 if bool(masked.mask.shape):
735 mi.mask.array[:, :] = masked.mask[:, :]
737 overscanVector = fitOverscanImage(mi, self.config.maskPlanes, isTransposed)
743 indices = 2.0*np.arange(num)/float(num) - 1.0
747 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
748 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
749 'LEG': (poly.legendre.legfit, poly.legendre.legval),
753 }[self.config.fitType]
757 coeffs = fitter(indices, collapsed, self.config.order)
759 if isinstance(coeffs, float):
760 self.log.warn(
"Using fallback value %f due to fitter failure. Amplifier will be masked.",
762 overscanVector = np.full_like(indices, coeffs)
763 maskArray = np.full_like(collapsed,
True, dtype=bool)
766 overscanVector = evaler(indices, coeffs)
769 return pipeBase.Struct(overscanValue=np.array(overscanVector),
771 isTransposed=isTransposed)
774 """Debug display for the final overscan solution.
779 Input image the overscan solution was determined from.
780 model : `numpy.ndarray`
or `float`
781 Overscan model determined
for the image.
783 Amplifier to extract diagnostic information.
793 calcImage = np.transpose(calcImage)
798 indices = 2.0 * np.arange(num)/float(num) - 1.0
800 if np.ma.is_masked(collapsed):
801 collapsedMask = collapsed.mask
803 collapsedMask = np.array(num*[np.ma.nomask])
805 import matplotlib.pyplot
as plot
806 figure = plot.figure(1)
808 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
809 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask],
'k+')
810 if collapsedMask.sum() > 0:
811 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask],
'b+')
812 if isinstance(model, np.ndarray):
815 plotModel = np.zeros_like(indices)
817 axes.plot(indices, plotModel,
'r-')
818 plot.xlabel(
"centered/scaled position along overscan region")
819 plot.ylabel(
"pixel value/fit value")
821 plot.title(f
"{amp.getName()} DataX: "
822 f
"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]"
823 f
"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:"
824 f
"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}")
826 plot.title(
"No amp supplied.")
828 prompt =
"Press Enter or c to continue [chp]..."
830 ans = input(prompt).lower()
831 if ans
in (
"",
" ",
"c",):
840 print(
"[h]elp [c]ontinue [p]db e[x]itDebug")
def collapseArrayMedian(self, maskedArray)
def maskExtrapolated(collapsed)
def fitOverscan(self, overscanImage, isTransposed=False)
def measureVectorOverscan(self, image, isTransposed=False)
def __init__(self, statControl=None, **kwargs)
def collapseArray(maskedArray)
def maskOutliers(self, imageArray)
def debugView(self, image, model, amp=None)
def getImageArray(self, image)
def integerConvert(image)
def splineFit(self, indices, collapsed, numBins)
def broadcastFitToImage(self, overscanValue, imageArray, transpose=False)
def correctOverscan(self, exposure, amp, imageBBox, overscanBBox, isTransposed=True)
def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False)
def run(self, exposure, amp, isTransposed=False)
def splineEval(indices, interp)
def measureConstantOverscan(self, image)