22__all__ = [
"OverscanCorrectionTaskConfig",
"OverscanCorrectionTask"]
29import lsst.pipe.base
as pipeBase
32from .isr
import fitOverscanImage
36 """Overscan correction options.
38 fitType = pexConfig.ChoiceField(
40 doc="The method for fitting the overscan bias level.",
43 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
44 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
45 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
46 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
47 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
48 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
49 "MEAN":
"Correct using the mean of the overscan region",
50 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
51 "MEDIAN":
"Correct using the median of the overscan region",
52 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
55 order = pexConfig.Field(
57 doc=(
"Order of polynomial to fit if overscan fit type is a polynomial, "
58 "or number of spline knots if overscan fit type is a spline."),
61 numSigmaClip = pexConfig.Field(
63 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
66 maskPlanes = pexConfig.ListField(
68 doc=
"Mask planes to reject when measuring overscan",
69 default=[
'BAD',
'SAT'],
71 overscanIsInt = pexConfig.Field(
73 doc=
"Treat overscan as an integer image for purposes of fitType=MEDIAN"
74 " and fitType=MEDIAN_PER_ROW.",
78 doParallelOverscan = pexConfig.Field(
80 doc=
"Correct using parallel overscan after serial overscan correction?",
84 leadingColumnsToSkip = pexConfig.Field(
86 doc=
"Number of leading columns to skip in serial overscan correction.",
89 trailingColumnsToSkip = pexConfig.Field(
91 doc=
"Number of trailing columns to skip in serial overscan correction.",
94 leadingRowsToSkip = pexConfig.Field(
96 doc=
"Number of leading rows to skip in parallel overscan correction.",
99 trailingRowsToSkip = pexConfig.Field(
101 doc=
"Number of trailing rows to skip in parallel overscan correction.",
105 maxDeviation = pexConfig.Field(
107 doc=
"Maximum deviation from median (in ADU) to mask in overscan correction.",
108 default=1000.0, check=
lambda x: x > 0,
113 """Correction task for overscan.
115 This class contains a number of utilities that are easier to
116 understand
and use when they are
not embedded
in nested
if/
else
122 Statistics control object.
124 ConfigClass = OverscanCorrectionTaskConfig
125 _DefaultName = "overscan"
135 self.
statControl.setNumSigmaClip(self.config.numSigmaClip)
136 self.
statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes))
138 def run(self, exposure, amp, isTransposed=False):
139 """Measure and remove an overscan from an amplifier image.
144 Image data that will have the overscan corrections applied.
146 Amplifier to use for debugging purposes.
147 isTransposed : `bool`, optional
148 Is the image transposed, such that serial
and parallel
149 overscan regions are reversed? Default
is False.
153 overscanResults : `lsst.pipe.base.Struct`
154 Result struct
with components:
157 Value
or fit subtracted
from the amplifier image data
160 Value
or fit subtracted
from the overscan image data
163 Image of the overscan region
with the overscan
165 quantity
is used to estimate the amplifier read noise
171 Raised
if an invalid overscan type
is set.
174 serialOverscanBBox = amp.getRawSerialOverscanBBox()
175 imageBBox = amp.getRawDataBBox()
177 if self.config.doParallelOverscan:
180 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
181 imageBBox = imageBBox.expandedTo(parallelOverscanBBox)
184 imageBBox.getMinY()),
186 imageBBox.getHeight()))
188 imageBBox, serialOverscanBBox, isTransposed=isTransposed)
189 overscanMean = serialResults.overscanMean
190 overscanSigma = serialResults.overscanSigma
191 residualMean = serialResults.overscanMeanResidual
192 residualSigma = serialResults.overscanSigmaResidual
195 if self.config.doParallelOverscan:
198 parallelOverscanBBox = amp.getRawParallelOverscanBBox()
199 imageBBox = amp.getRawDataBBox()
202 imageBBox, parallelOverscanBBox,
203 isTransposed=
not isTransposed)
205 overscanMean = (overscanMean, parallelResults.overscanMean)
206 overscanSigma = (overscanSigma, parallelResults.overscanSigma)
207 residualMean = (residualMean, parallelResults.overscanMeanResidual)
208 residualSigma = (residualSigma, parallelResults.overscanSigmaResidual)
210 return pipeBase.Struct(imageFit=serialResults.ampOverscanModel,
211 overscanFit=serialResults.overscanOverscanModel,
212 overscanImage=serialResults.overscanImage,
214 overscanMean=overscanMean,
215 overscanSigma=overscanSigma,
216 residualMean=residualMean,
217 residualSigma=residualSigma)
222 overscanBox = self.trimOverscan(exposure, amp, overscanBBox,
223 self.config.leadingColumnsToSkip,
224 self.config.trailingColumnsToSkip,
225 transpose=isTransposed)
226 overscanImage = exposure[overscanBox].getMaskedImage()
227 overscanArray = overscanImage.image.array
230 maskVal = overscanImage.mask.getPlaneBitMask(self.config.maskPlanes)
231 overscanMask = ~((overscanImage.mask.array & maskVal) == 0)
233 median = np.ma.median(np.ma.masked_where(overscanMask, overscanArray))
234 bad = np.where(np.abs(overscanArray - median) > self.config.maxDeviation)
235 overscanMask[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
239 overscanResults = self.
fitOverscan(overscanImage, isTransposed=isTransposed)
242 ampImage = exposure[imageBBox]
244 ampImage.image.array,
245 transpose=isTransposed)
246 ampImage.image.array -= ampOverscanModel
250 overscanImage = exposure[overscanBBox]
253 overscanImage.image.array)
254 overscanImage.image.array -= overscanOverscanModel
256 self.
debugView(overscanImage, overscanResults.overscanValue, amp)
259 stats = afwMath.makeStatistics(overscanImage.getMaskedImage(),
260 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
261 residualMean = stats.getValue(afwMath.MEDIAN)
262 residualSigma = stats.getValue(afwMath.STDEVCLIP)
264 return pipeBase.Struct(ampOverscanModel=ampOverscanModel,
265 overscanOverscanModel=overscanOverscanModel,
266 overscanImage=overscanImage,
267 overscanValue=overscanResults.overscanValue,
269 overscanMean=overscanResults.overscanMean,
270 overscanSigma=overscanResults.overscanSigma,
271 overscanMeanResidual=residualMean,
272 overscanSigmaResidual=residualSigma
276 """Broadcast 0 or 1 dimension fit to appropriate shape.
280 overscanValue : `numpy.ndarray`, (Nrows, ) or scalar
281 Overscan fit to broadcast.
282 imageArray : `numpy.ndarray`, (Nrows, Ncols)
283 Image array that we want to match.
284 transpose : `bool`, optional
285 Switch order to broadcast along the other axis.
289 overscanModel : `numpy.ndarray`, (Nrows, Ncols)
or scalar
290 Expanded overscan fit.
295 Raised
if no axis has the appropriate dimension.
297 if isinstance(overscanValue, np.ndarray):
298 overscanModel = np.zeros_like(imageArray)
300 if transpose
is False:
301 if imageArray.shape[0] == overscanValue.shape[0]:
302 overscanModel[:, :] = overscanValue[:, np.newaxis]
303 elif imageArray.shape[1] == overscanValue.shape[0]:
304 overscanModel[:, :] = overscanValue[np.newaxis, :]
305 elif imageArray.shape[0] == overscanValue.shape[1]:
306 overscanModel[:, :] = overscanValue[np.newaxis, :]
308 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
309 f
"match {imageArray.shape}")
311 if imageArray.shape[1] == overscanValue.shape[0]:
312 overscanModel[:, :] = overscanValue[np.newaxis, :]
313 elif imageArray.shape[0] == overscanValue.shape[0]:
314 overscanModel[:, :] = overscanValue[:, np.newaxis]
315 elif imageArray.shape[1] == overscanValue.shape[1]:
316 overscanModel[:, :] = overscanValue[:, np.newaxis]
318 raise RuntimeError(f
"Could not broadcast {overscanValue.shape} to "
319 f
"match {imageArray.shape}")
321 overscanModel = overscanValue
325 def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False):
326 """Trim overscan region to remove edges.
331 Exposure containing data.
333 Amplifier containing geometry information.
335 Bounding box of the overscan region.
337 Number of leading (towards data region) rows/columns to skip.
339 Number of trailing (away from data region) rows/columns to skip.
340 transpose : `bool`, optional
341 Operate on the transposed array.
345 overscanArray : `numpy.array`, (N, M)
347 overscanMask : `numpy.array`, (N, M)
350 dx0, dy0, dx1, dy1 = (0, 0, 0, 0)
351 dataBBox = amp.getRawDataBBox()
353 if dataBBox.getBeginY() < bbox.getBeginY():
360 if dataBBox.getBeginX() < bbox.getBeginX():
369 bbox.getHeight() - dy0 + dy1))
373 if self.config.fitType
in (
'MEAN',
'MEANCLIP',
'MEDIAN'):
376 overscanValue = overscanResult.overscanValue
377 overscanMean = overscanValue
379 elif self.config.fitType
in (
'MEDIAN_PER_ROW',
'POLY',
'CHEB',
'LEG',
380 'NATURAL_SPLINE',
'CUBIC_SPLINE',
'AKIMA_SPLINE'):
383 overscanValue = overscanResult.overscanValue
385 stats = afwMath.makeStatistics(overscanResult.overscanValue,
386 afwMath.MEDIAN | afwMath.STDEVCLIP, self.
statControl)
387 overscanMean = stats.getValue(afwMath.MEDIAN)
388 overscanSigma = stats.getValue(afwMath.STDEVCLIP)
390 raise ValueError(
'%s : %s an invalid overscan type' %
391 (
"overscanCorrection", self.config.fitType))
393 return pipeBase.Struct(overscanValue=overscanValue,
394 overscanMean=overscanMean,
395 overscanSigma=overscanSigma,
400 """Return an integer version of the input image.
405 Image to convert to integers.
410 The integer converted image.
415 Raised
if the input image could
not be converted.
417 if hasattr(image,
"image"):
419 imageI = image.image.convertI()
420 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
421 elif hasattr(image,
"convertI"):
423 outI = image.convertI()
424 elif hasattr(image,
"astype"):
426 outI = image.astype(int)
428 raise RuntimeError(
"Could not convert this to integers: %s %s %s",
429 image, type(image), dir(image))
434 """Measure a constant overscan value.
439 Image data to measure the overscan
from.
443 results : `lsst.pipe.base.Struct`
444 Overscan result
with entries:
445 - ``overscanValue``: Overscan value to subtract (`float`)
446 - ``isTransposed``: Orientation of the overscan (`bool`)
448 if self.config.fitType ==
'MEDIAN':
452 fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
453 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.
statControl).getValue()
455 return pipeBase.Struct(overscanValue=overscanValue,
460 """Extract the numpy array from the input image.
465 Image data to pull array
from.
467 calcImage : `numpy.ndarray`
468 Image data array
for numpy operating.
470 if hasattr(image,
"getImage"):
471 calcImage = image.getImage().getArray()
472 calcImage = np.ma.masked_where(image.getMask().getArray() & self.
statControl.getAndMask(),
475 calcImage = image.getArray()
479 """Mask outliers in a row of overscan data
from a robust sigma
484 imageArray : `numpy.ndarray`
485 Image to filter along numpy axis=1.
489 maskedArray : `numpy.ma.masked_array`
490 Masked image marking outliers.
492 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1)
494 axisStdev = 0.74*(uq - lq)
496 diff = np.abs(imageArray - axisMedians[:, np.newaxis])
497 return np.ma.masked_where(diff > self.
statControl.getNumSigmaClip()
498 * axisStdev[:, np.newaxis], imageArray)
502 """Collapse overscan array (and mask) to a 1-D vector of values.
506 maskedArray : `numpy.ma.masked_array`
507 Masked array of input overscan data.
511 collapsed : `numpy.ma.masked_array`
512 Single dimensional overscan data, combined with the mean.
514 collapsed = np.mean(maskedArray, axis=1)
515 if collapsed.mask.sum() > 0:
516 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1)
520 """Collapse overscan array (and mask) to a 1-D vector of using the
521 correct integer median of row-values.
525 maskedArray : `numpy.ma.masked_array`
526 Masked array of input overscan data.
530 collapsed : `numpy.ma.masked_array`
531 Single dimensional overscan data, combined with the afwMath median.
536 fitType = afwMath.stringToStatisticsProperty('MEDIAN')
537 for row
in integerMI:
538 newRow = row.compressed()
540 rowMedian = afwMath.makeStatistics(newRow, fitType, self.
statControl).getValue()
543 collapsed.append(rowMedian)
545 return np.array(collapsed)
548 """Wrapper function to match spline fit API to polynomial fit API.
552 indices : `numpy.ndarray`
553 Locations to evaluate the spline.
554 collapsed : `numpy.ndarray`
555 Collapsed overscan values corresponding to the spline
558 Number of bins to use in constructing the spline.
563 Interpolation object
for later evaluation.
565 if not np.ma.is_masked(collapsed):
566 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask])
568 numPerBin, binEdges = np.histogram(indices, bins=numBins,
569 weights=1 - collapsed.mask.astype(int))
570 with np.errstate(invalid=
"ignore"):
571 values = np.histogram(indices, bins=numBins,
572 weights=collapsed.data*~collapsed.mask)[0]/numPerBin
573 binCenters = np.histogram(indices, bins=numBins,
574 weights=indices*~collapsed.mask)[0]/numPerBin
576 if len(binCenters[numPerBin > 0]) < 5:
577 self.log.warn(
"Cannot do spline fitting for overscan: %s valid points.",
578 len(binCenters[numPerBin > 0]))
582 if len(values[numPerBin > 0]) != 0:
583 return float(values[numPerBin > 0][0])
587 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
588 values.astype(float)[numPerBin > 0],
589 afwMath.stringToInterpStyle(self.config.fitType))
594 """Wrapper function to match spline evaluation API to polynomial fit
599 indices : `numpy.ndarray`
600 Locations to evaluate the spline.
601 interp : `lsst.afw.math.interpolate`
602 Interpolation object to use.
606 values : `numpy.ndarray`
607 Evaluated spline values at each index.
610 return interp.interpolate(indices.astype(float))
614 """Create mask if edges are extrapolated.
618 collapsed : `numpy.ma.masked_array`
619 Masked array to check the edges of.
623 maskArray : `numpy.ndarray`
624 Boolean numpy array of pixels to mask.
626 maskArray = np.full_like(collapsed, False, dtype=bool)
627 if np.ma.is_masked(collapsed):
629 for low
in range(num):
630 if not collapsed.mask[low]:
633 maskArray[:low] =
True
634 for high
in range(1, num):
635 if not collapsed.mask[-high]:
638 maskArray[-high:] =
True
642 """Calculate the 1-d vector overscan from the input overscan image.
647 Image containing the overscan data.
648 isTransposed : `bool`
649 If true, the image has been transposed.
653 results : `lsst.pipe.base.Struct`
654 Overscan result with entries:
657 Overscan value to subtract (`float`)
659 List of rows that should be masked
as ``SUSPECT`` when the
660 overscan solution
is applied. (`list` [ `bool` ])
662 Indicates
if the overscan data was transposed during
663 calcuation, noting along which axis the overscan should be
670 calcImage = np.transpose(calcImage)
673 startTime = time.perf_counter()
675 if self.config.fitType ==
'MEDIAN_PER_ROW':
676 mi = afwImage.MaskedImageI(image.getBBox())
677 masked = masked.astype(int)
679 masked = masked.transpose()
681 mi.image.array[:, :] = masked.data[:, :]
682 if bool(masked.mask.shape):
683 mi.mask.array[:, :] = masked.mask[:, :]
685 overscanVector = fitOverscanImage(mi, self.config.maskPlanes, isTransposed)
691 indices = 2.0*np.arange(num)/float(num) - 1.0
695 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
696 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
697 'LEG': (poly.legendre.legfit, poly.legendre.legval),
701 }[self.config.fitType]
705 coeffs = fitter(indices, collapsed, self.config.order)
707 if isinstance(coeffs, float):
708 self.log.warn(
"Using fallback value %f due to fitter failure. Amplifier will be masked.",
710 overscanVector = np.full_like(indices, coeffs)
711 maskArray = np.full_like(collapsed,
True, dtype=bool)
714 overscanVector = evaler(indices, coeffs)
717 endTime = time.perf_counter()
718 self.log.info(f
"Overscan measurement took {endTime - startTime}s for {self.config.fitType}")
719 return pipeBase.Struct(overscanValue=np.array(overscanVector),
721 isTransposed=isTransposed)
724 """Debug display for the final overscan solution.
729 Input image the overscan solution was determined from.
730 model : `numpy.ndarray`
or `float`
731 Overscan model determined
for the image.
733 Amplifier to extract diagnostic information.
743 calcImage = np.transpose(calcImage)
748 indices = 2.0 * np.arange(num)/float(num) - 1.0
750 if np.ma.is_masked(collapsed):
751 collapsedMask = collapsed.mask
753 collapsedMask = np.array(num*[np.ma.nomask])
755 import matplotlib.pyplot
as plot
756 figure = plot.figure(1)
758 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
759 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask],
'k+')
760 if collapsedMask.sum() > 0:
761 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask],
'b+')
762 if isinstance(model, np.ndarray):
765 plotModel = np.zeros_like(indices)
767 axes.plot(indices, plotModel,
'r-')
768 plot.xlabel(
"centered/scaled position along overscan region")
769 plot.ylabel(
"pixel value/fit value")
771 plot.title(f
"{amp.getName()} DataX: "
772 f
"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]"
773 f
"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:"
774 f
"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}")
776 plot.title(
"No amp supplied.")
778 prompt =
"Press Enter or c to continue [chp]..."
780 ans = input(prompt).lower()
781 if ans
in (
"",
" ",
"c",):
790 print(
"[h]elp [c]ontinue [p]db e[x]itDebug")
def collapseArrayMedian(self, maskedArray)
def maskExtrapolated(collapsed)
def fitOverscan(self, overscanImage, isTransposed=False)
def measureVectorOverscan(self, image, isTransposed=False)
def __init__(self, statControl=None, **kwargs)
def collapseArray(maskedArray)
def maskOutliers(self, imageArray)
def debugView(self, image, model, amp=None)
def getImageArray(self, image)
def integerConvert(image)
def splineFit(self, indices, collapsed, numBins)
def broadcastFitToImage(self, overscanValue, imageArray, transpose=False)
def correctOverscan(self, exposure, amp, imageBBox, overscanBBox, isTransposed=True)
def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False)
def run(self, exposure, amp, isTransposed=False)
def splineEval(indices, interp)
def measureConstantOverscan(self, image)