Coverage for python/lsst/pipe/tasks/matchBackgrounds.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# # LSST Data Management System # Copyright 2008, 2009, 2010 LSST Corporation. # # This product includes software developed by the # LSST Project (http://www.lsst.org/). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILIY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>.
dtype=bool, doc="Fit background difference with Chebychev polynomial interpolation " "(using afw.math.Approximate)? If False, fit with spline interpolation using afw.math.Background", default=False ) dtype=int, doc="Order of Chebyshev polynomial background model. Ignored if usePolynomial False", default=8 ) doc="Names of mask planes to ignore while estimating the background", dtype=str, default=["NO_DATA", "DETECTED", "DETECTED_NEGATIVE", "SAT", "BAD", "INTRP", "CR"], itemCheck=lambda x: x in afwImage.Mask().getMaskPlaneDict(), ) dtype=str, doc="Type of statistic to estimate pixel value for the grid points", default="MEAN", allowed={ "MEAN": "mean", "MEDIAN": "median", "MEANCLIP": "clipped mean" } ) doc="Behaviour if there are too few points in grid for requested interpolation style. " "Note: INCREASE_NXNYSAMPLE only allowed for usePolynomial=True.", dtype=str, default="REDUCE_INTERP_ORDER", allowed={ "THROW_EXCEPTION": "throw an exception if there are too few points", "REDUCE_INTERP_ORDER": "use an interpolation style with a lower order.", "INCREASE_NXNYSAMPLE": "Increase the number of samples used to make the interpolation grid.", } ) doc="Bin size for gridding the difference image and fitting a spatial model", dtype=int, default=256 ) dtype=str, doc="Algorithm to interpolate the background values; ignored if usePolynomial is True" "Maps to an enum; see afw.math.Background", default="AKIMA_SPLINE", allowed={ "CONSTANT": "Use a single constant value", "LINEAR": "Use linear interpolation", "NATURAL_SPLINE": "cubic spline with zero second derivative at endpoints", "AKIMA_SPLINE": "higher-level nonlinear spline that is more robust to outliers", "NONE": "No background estimation is to be attempted", } ) dtype=int, doc="Sigma for outlier rejection; ignored if gridStatistic != 'MEANCLIP'.", default=3 ) dtype=int, doc="Number of iterations of outlier rejection; ignored if gridStatistic != 'MEANCLIP'.", default=2 ) dtype=float, doc="Weight given to coverage (number of pixels that overlap with patch), " "when calculating best reference exposure. Higher weight prefers exposures with high coverage." "Ignored when reference visit is supplied", default=0.4, min=0., max=1. ) dtype=float, doc="Weight given to image variance when calculating best reference exposure. " "Higher weight prefers exposures with low image variance. Ignored when reference visit is supplied", default=0.4, min=0., max=1. ) dtype=float, doc="Weight given to mean background level when calculating best reference exposure. " "Higher weight prefers exposures with low mean background level. " "Ignored when reference visit is supplied.", default=0.2, min=0., max=1. ) dtype=bool, doc=("Use inverse-variance weighting when approximating background offset model? " + "This will fail when the background offset is constant " + "(this is usually only the case in testing with artificial images)." + "(usePolynomial=True)"), default=True, ) dtype=float, doc="Tolerance on almost zero standard deviation in a background-offset grid bin. " "If all bins have a standard deviation below this value, the background offset model " "is approximated without inverse-variance weighting. (usePolynomial=True)", default=1e-8, min=0. )
pipeBase.Task.__init__(self, *args, **kwargs)
self.sctrl = afwMath.StatisticsControl() self.sctrl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes)) self.sctrl.setNanSafe(True)
"""Match the backgrounds of a list of coadd temp exposures to a reference coadd temp exposure.
Choose a refExpDataRef automatically if none supplied.
@param[in] expRefList: list of data references to science exposures to be background-matched; all exposures must exist. @param[in] expDatasetType: dataset type of exposures, e.g. 'goodSeeingCoadd_tempExp' @param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler); if None then the images are not scaled @param[in] refExpDataRef: data reference for the reference exposure. If None, then this task selects the best exposures from expRefList. if not None then must be one of the exposures in expRefList. @param[in] refImageScaler: image scaler for reference image; ignored if refExpDataRef is None, else scaling is not performed if None
@return: a pipBase.Struct containing these fields: - backgroundInfoList: a list of pipeBase.Struct, one per exposure in expRefList, each of which contains these fields: - isReference: this is the reference exposure (only one returned Struct will contain True for this value, unless the ref exposure is listed multiple times) - backgroundModel: differential background model (afw.Math.Background or afw.Math.Approximate). Add this to the science exposure to match the reference exposure. - fitRMS: rms of the fit. This is the sqrt(mean(residuals**2)). - matchedMSE: the MSE of the reference and matched images: mean((refImage - matchedSciImage)**2); should be comparable to difference image's mean variance. - diffImVar: the mean variance of the difference image. All fields except isReference will be None if isReference True or the fit failed.
@warning: all exposures must exist on disk """
numExp = len(expRefList) if numExp < 1: raise pipeBase.TaskError("No exposures to match")
if expDatasetType is None: raise pipeBase.TaskError("Must specify expDatasetType")
if imageScalerList is None: self.log.info("imageScalerList is None; no scaling will be performed") imageScalerList = [None] * numExp
if len(expRefList) != len(imageScalerList): raise RuntimeError("len(expRefList) = %s != %s = len(imageScalerList)" % (len(expRefList), len(imageScalerList)))
refInd = None if refExpDataRef is None: # select the best reference exposure from expRefList refInd = self.selectRefExposure( expRefList=expRefList, imageScalerList=imageScalerList, expDatasetType=expDatasetType, ) refExpDataRef = expRefList[refInd] refImageScaler = imageScalerList[refInd]
# refIndSet is the index of all exposures in expDataList that match the reference. # It is used to avoid background-matching an exposure to itself. It is a list # because it is possible (though unlikely) that expDataList will contain duplicates. expKeyList = refExpDataRef.butlerSubset.butler.getKeys(expDatasetType) refMatcher = DataRefMatcher(refExpDataRef.butlerSubset.butler, expDatasetType) refIndSet = set(refMatcher.matchList(ref0=refExpDataRef, refList=expRefList))
if refInd is not None and refInd not in refIndSet: raise RuntimeError("Internal error: selected reference %s not found in expRefList")
refExposure = refExpDataRef.get(expDatasetType, immediate=True) if refImageScaler is not None: refMI = refExposure.getMaskedImage() refImageScaler.scaleMaskedImage(refMI)
debugIdKeyList = tuple(set(expKeyList) - set(['tract', 'patch']))
self.log.info("Matching %d Exposures" % (numExp))
backgroundInfoList = [] for ind, (toMatchRef, imageScaler) in enumerate(zip(expRefList, imageScalerList)): if ind in refIndSet: backgroundInfoStruct = pipeBase.Struct( isReference=True, backgroundModel=None, fitRMS=0.0, matchedMSE=None, diffImVar=None, ) else: self.log.info("Matching background of %s to %s" % (toMatchRef.dataId, refExpDataRef.dataId)) try: toMatchExposure = toMatchRef.get(expDatasetType, immediate=True) if imageScaler is not None: toMatchMI = toMatchExposure.getMaskedImage() imageScaler.scaleMaskedImage(toMatchMI) # store a string specifying the visit to label debug plot self.debugDataIdString = ''.join([str(toMatchRef.dataId[vk]) for vk in debugIdKeyList]) backgroundInfoStruct = self.matchBackgrounds( refExposure=refExposure, sciExposure=toMatchExposure, ) backgroundInfoStruct.isReference = False except Exception as e: self.log.warn("Failed to fit background %s: %s" % (toMatchRef.dataId, e)) backgroundInfoStruct = pipeBase.Struct( isReference=False, backgroundModel=None, fitRMS=None, matchedMSE=None, diffImVar=None, )
backgroundInfoList.append(backgroundInfoStruct)
return pipeBase.Struct( backgroundInfoList=backgroundInfoList)
def selectRefExposure(self, expRefList, imageScalerList, expDatasetType): """Find best exposure to use as the reference exposure
Calculate an appropriate reference exposure by minimizing a cost function that penalizes high variance, high background level, and low coverage. Use the following config parameters: - bestRefWeightCoverage - bestRefWeightVariance - bestRefWeightLevel
@param[in] expRefList: list of data references to exposures. Retrieves dataset type specified by expDatasetType. If an exposure is not found, it is skipped with a warning. @param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler); must be the same length as expRefList @param[in] expDatasetType: dataset type of exposure: e.g. 'goodSeeingCoadd_tempExp'
@return: index of best exposure
@raise pipeBase.TaskError if none of the exposures in expRefList are found. """ self.log.info("Calculating best reference visit") varList = [] meanBkgdLevelList = [] coverageList = []
if len(expRefList) != len(imageScalerList): raise RuntimeError("len(expRefList) = %s != %s = len(imageScalerList)" % (len(expRefList), len(imageScalerList)))
for expRef, imageScaler in zip(expRefList, imageScalerList): exposure = expRef.get(expDatasetType, immediate=True) maskedImage = exposure.getMaskedImage() if imageScaler is not None: try: imageScaler.scaleMaskedImage(maskedImage) except Exception: # need to put a place holder in Arr varList.append(numpy.nan) meanBkgdLevelList.append(numpy.nan) coverageList.append(numpy.nan) continue statObjIm = afwMath.makeStatistics(maskedImage.getImage(), maskedImage.getMask(), afwMath.MEAN | afwMath.NPOINT | afwMath.VARIANCE, self.sctrl) meanVar, meanVarErr = statObjIm.getResult(afwMath.VARIANCE) meanBkgdLevel, meanBkgdLevelErr = statObjIm.getResult(afwMath.MEAN) npoints, npointsErr = statObjIm.getResult(afwMath.NPOINT) varList.append(meanVar) meanBkgdLevelList.append(meanBkgdLevel) coverageList.append(npoints) if not coverageList: raise pipeBase.TaskError( "None of the candidate %s exist; cannot select best reference exposure" % (expDatasetType,))
# Normalize metrics to range from 0 to 1 varArr = numpy.array(varList)/numpy.nanmax(varList) meanBkgdLevelArr = numpy.array(meanBkgdLevelList)/numpy.nanmax(meanBkgdLevelList) coverageArr = numpy.nanmin(coverageList)/numpy.array(coverageList)
costFunctionArr = self.config.bestRefWeightVariance * varArr costFunctionArr += self.config.bestRefWeightLevel * meanBkgdLevelArr costFunctionArr += self.config.bestRefWeightCoverage * coverageArr return numpy.nanargmin(costFunctionArr)
def matchBackgrounds(self, refExposure, sciExposure): """ Match science exposure's background level to that of reference exposure.
Process creates a difference image of the reference exposure minus the science exposure, and then generates an afw.math.Background object. It assumes (but does not require/check) that the mask plane already has detections set. If detections have not been set/masked, sources will bias the background estimation. The 'background' of the difference image is smoothed by spline interpolation (by the Background class) or by polynomial interpolation by the Approximate class. This model of difference image is added to the science exposure in memory. Fit diagnostics are also calculated and returned.
@param[in] refExposure: reference exposure @param[in,out] sciExposure: science exposure; modified by changing the background level to match that of the reference exposure @returns a pipBase.Struct with fields: - backgroundModel: an afw.math.Approximate or an afw.math.Background. - fitRMS: rms of the fit. This is the sqrt(mean(residuals**2)). - matchedMSE: the MSE of the reference and matched images: mean((refImage - matchedSciImage)**2); should be comparable to difference image's mean variance. - diffImVar: the mean variance of the difference image. """
if lsstDebug.Info(__name__).savefits: refExposure.writeFits(lsstDebug.Info(__name__).figpath + 'refExposure.fits') sciExposure.writeFits(lsstDebug.Info(__name__).figpath + 'sciExposure.fits')
# Check Configs for polynomials: if self.config.usePolynomial: x, y = sciExposure.getDimensions() shortSideLength = min(x, y) if shortSideLength < self.config.binSize: raise ValueError("%d = config.binSize > shorter dimension = %d" % (self.config.binSize, shortSideLength)) npoints = shortSideLength // self.config.binSize if shortSideLength % self.config.binSize != 0: npoints += 1
if self.config.order > npoints - 1: raise ValueError("%d = config.order > npoints - 1 = %d" % (self.config.order, npoints - 1))
# Check that exposures are same shape if (sciExposure.getDimensions() != refExposure.getDimensions()): wSci, hSci = sciExposure.getDimensions() wRef, hRef = refExposure.getDimensions() raise RuntimeError( "Exposures are different dimensions. sci:(%i, %i) vs. ref:(%i, %i)" % (wSci, hSci, wRef, hRef))
statsFlag = getattr(afwMath, self.config.gridStatistic) self.sctrl.setNumSigmaClip(self.config.numSigmaClip) self.sctrl.setNumIter(self.config.numIter)
im = refExposure.getMaskedImage() diffMI = im.Factory(im, True) diffMI -= sciExposure.getMaskedImage()
width = diffMI.getWidth() height = diffMI.getHeight() nx = width // self.config.binSize if width % self.config.binSize != 0: nx += 1 ny = height // self.config.binSize if height % self.config.binSize != 0: ny += 1
bctrl = afwMath.BackgroundControl(nx, ny, self.sctrl, statsFlag) bctrl.setUndersampleStyle(self.config.undersampleStyle) bctrl.setInterpStyle(self.config.interpStyle)
bkgd = afwMath.makeBackground(diffMI, bctrl)
# Some config and input checks if config.usePolynomial: # 1) Check that order/bin size make sense: # 2) Change binsize or order if underconstrained. if self.config.usePolynomial: order = self.config.order bgX, bgY, bgZ, bgdZ = self._gridImage(diffMI, self.config.binSize, statsFlag) minNumberGridPoints = min(len(set(bgX)), len(set(bgY))) if len(bgZ) == 0: raise ValueError("No overlap with reference. Nothing to match") elif minNumberGridPoints <= self.config.order: # must either lower order or raise number of bins or throw exception if self.config.undersampleStyle == "THROW_EXCEPTION": raise ValueError("Image does not cover enough of ref image for order and binsize") elif self.config.undersampleStyle == "REDUCE_INTERP_ORDER": self.log.warn("Reducing order to %d"%(minNumberGridPoints - 1)) order = minNumberGridPoints - 1 elif self.config.undersampleStyle == "INCREASE_NXNYSAMPLE": newBinSize = (minNumberGridPoints*self.config.binSize) // (self.config.order + 1) bctrl.setNxSample(newBinSize) bctrl.setNySample(newBinSize) bkgd = afwMath.makeBackground(diffMI, bctrl) # do over self.log.warn("Decreasing binsize to %d"%(newBinSize))
# If there is no variance in any image pixels, do not weight bins by inverse variance isUniformImageDiff = not numpy.any(bgdZ > self.config.gridStdevEpsilon) weightByInverseVariance = False if isUniformImageDiff else self.config.approxWeighting
# Add offset to sciExposure try: if self.config.usePolynomial: actrl = afwMath.ApproximateControl(afwMath.ApproximateControl.CHEBYSHEV, order, order, weightByInverseVariance) undersampleStyle = getattr(afwMath, self.config.undersampleStyle) approx = bkgd.getApproximate(actrl, undersampleStyle) bkgdImage = approx.getImage() else: bkgdImage = bkgd.getImageF() except Exception as e: raise RuntimeError("Background/Approximation failed to interp image %s: %s" % ( self.debugDataIdString, e))
sciMI = sciExposure.getMaskedImage() sciMI += bkgdImage del sciMI
# Need RMS from fit: 2895 will replace this: rms = 0.0 X, Y, Z, dZ = self._gridImage(diffMI, self.config.binSize, statsFlag) x0, y0 = diffMI.getXY0() modelValueArr = numpy.empty(len(Z)) for i in range(len(X)): modelValueArr[i] = bkgdImage[int(X[i]-x0), int(Y[i]-y0), afwImage.LOCAL] resids = Z - modelValueArr rms = numpy.sqrt(numpy.mean(resids[~numpy.isnan(resids)]**2))
if lsstDebug.Info(__name__).savefits: sciExposure.writeFits(lsstDebug.Info(__name__).figpath + 'sciMatchedExposure.fits')
if lsstDebug.Info(__name__).savefig: bbox = afwGeom.Box2D(refExposure.getMaskedImage().getBBox()) try: self._debugPlot(X, Y, Z, dZ, bkgdImage, bbox, modelValueArr, resids) except Exception as e: self.log.warn('Debug plot not generated: %s'%(e))
meanVar = afwMath.makeStatistics(diffMI.getVariance(), diffMI.getMask(), afwMath.MEANCLIP, self.sctrl).getValue()
diffIm = diffMI.getImage() diffIm -= bkgdImage # diffMI should now have a mean ~ 0 del diffIm mse = afwMath.makeStatistics(diffMI, afwMath.MEANSQUARE, self.sctrl).getValue()
outBkgd = approx if self.config.usePolynomial else bkgd
return pipeBase.Struct( backgroundModel=outBkgd, fitRMS=rms, matchedMSE=mse, diffImVar=meanVar)
"""Generate a plot showing the background fit and residuals.
It is called when lsstDebug.Info(__name__).savefig = True Saves the fig to lsstDebug.Info(__name__).figpath Displays on screen if lsstDebug.Info(__name__).display = True
@param X: array of x positions @param Y: array of y positions @param Z: array of the grid values that were interpolated @param dZ: array of the error on the grid values @param modelImage: image ofthe model of the fit @param model: array of len(Z) containing the grid values predicted by the model @param resids: Z - model """ import matplotlib.pyplot as plt import matplotlib.colors from mpl_toolkits.axes_grid1 import ImageGrid zeroIm = afwImage.MaskedImageF(afwGeom.Box2I(bbox)) zeroIm += modelImage x0, y0 = zeroIm.getXY0() dx, dy = zeroIm.getDimensions() if len(Z) == 0: self.log.warn("No grid. Skipping plot generation.") else: max, min = numpy.max(Z), numpy.min(Z) norm = matplotlib.colors.normalize(vmax=max, vmin=min) maxdiff = numpy.max(numpy.abs(resids)) diffnorm = matplotlib.colors.normalize(vmax=maxdiff, vmin=-maxdiff) rms = numpy.sqrt(numpy.mean(resids**2)) fig = plt.figure(1, (8, 6)) meanDz = numpy.mean(dZ) grid = ImageGrid(fig, 111, nrows_ncols=(1, 2), axes_pad=0.1, share_all=True, label_mode="L", cbar_mode="each", cbar_size="7%", cbar_pad="2%", cbar_location="top") im = grid[0].imshow(zeroIm.getImage().getArray(), extent=(x0, x0+dx, y0+dy, y0), norm=norm, cmap='Spectral') im = grid[0].scatter(X, Y, c=Z, s=15.*meanDz/dZ, edgecolor='none', norm=norm, marker='o', cmap='Spectral') im2 = grid[1].scatter(X, Y, c=resids, edgecolor='none', norm=diffnorm, marker='s', cmap='seismic') grid.cbar_axes[0].colorbar(im) grid.cbar_axes[1].colorbar(im2) grid[0].axis([x0, x0+dx, y0+dy, y0]) grid[1].axis([x0, x0+dx, y0+dy, y0]) grid[0].set_xlabel("model and grid") grid[1].set_xlabel("residuals. rms = %0.3f"%(rms)) if lsstDebug.Info(__name__).savefig: fig.savefig(lsstDebug.Info(__name__).figpath + self.debugDataIdString + '.png') if lsstDebug.Info(__name__).display: plt.show() plt.clf()
"""Private method to grid an image for debugging""" width, height = maskedImage.getDimensions() x0, y0 = maskedImage.getXY0() xedges = numpy.arange(0, width, binsize) yedges = numpy.arange(0, height, binsize) xedges = numpy.hstack((xedges, width)) # add final edge yedges = numpy.hstack((yedges, height)) # add final edge
# Use lists/append to protect against the case where # a bin has no valid pixels and should not be included in the fit bgX = [] bgY = [] bgZ = [] bgdZ = []
for ymin, ymax in zip(yedges[0:-1], yedges[1:]): for xmin, xmax in zip(xedges[0:-1], xedges[1:]): subBBox = afwGeom.Box2I(afwGeom.PointI(int(x0 + xmin), int(y0 + ymin)), afwGeom.PointI(int(x0 + xmax-1), int(y0 + ymax-1))) subIm = afwImage.MaskedImageF(maskedImage, subBBox, afwImage.PARENT, False) stats = afwMath.makeStatistics(subIm, afwMath.MEAN | afwMath.MEANCLIP | afwMath.MEDIAN | afwMath.NPOINT | afwMath.STDEV, self.sctrl) npoints, _ = stats.getResult(afwMath.NPOINT) if npoints >= 2: stdev, _ = stats.getResult(afwMath.STDEV) if stdev < self.config.gridStdevEpsilon: stdev = self.config.gridStdevEpsilon bgX.append(0.5 * (x0 + xmin + x0 + xmax)) bgY.append(0.5 * (y0 + ymin + y0 + ymax)) bgdZ.append(stdev/numpy.sqrt(npoints)) est, _ = stats.getResult(statsFlag) bgZ.append(est)
return numpy.array(bgX), numpy.array(bgY), numpy.array(bgZ), numpy.array(bgdZ)
"""Match data references for a specified dataset type
Note that this is not exact, but should suffice for this task until there is better support for this kind of thing in the butler. """
"""Construct a DataRefMatcher
@param[in] butler @param[in] datasetType: dataset type to match """ self._datasetType = datasetType # for diagnostics self._keyNames = butler.getKeys(datasetType)
"""Return a tuple of values for the specified keyNames
@param[in] ref: data reference
@raise KeyError if ref.dataId is missing a key in keyNames """ return tuple(ref.dataId[key] for key in self._keyNames)
"""Return True if ref0 == ref1
@param[in] ref0: data ref 0 @param[in] ref1: data ref 1
@raise KeyError if either ID is missing a key in keyNames """ return self._makeKey(ref0) == self._makeKey(ref1)
"""Return a list of indices of matches
@return tuple of indices of matches
@raise KeyError if any ID is missing a key in keyNames """ key0 = self._makeKey(ref0) return tuple(ind for ind, ref in enumerate(refList) if self._makeKey(ref) == key0) |