warpType = pexConfig.Field(
doc="Warp name: one of 'direct' or 'psfMatched'",
dtype=str,
default="direct",
)
subregionSize = pexConfig.ListField(
dtype=int,
doc="Width, height of stack subregion size; "
"make small enough that a full stack of images will fit into memory at once.",
length=2,
default=(2000, 2000),
)
statistic = pexConfig.Field(
dtype=str,
doc="Main stacking statistic for aggregating over the epochs.",
default="MEANCLIP",
)
doOnlineForMean = pexConfig.Field(
dtype=bool,
doc="Perform online coaddition when statistic=\"MEAN\" to save memory?",
default=False,
)
doSigmaClip = pexConfig.Field(
dtype=bool,
doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
default=False,
)
sigmaClip = pexConfig.Field(
dtype=float,
doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
default=3.0,
)
clipIter = pexConfig.Field(
dtype=int,
doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
default=2,
)
calcErrorFromInputVariance = pexConfig.Field(
dtype=bool,
doc="Calculate coadd variance from input variance by stacking statistic."
"Passed to StatisticsControl.setCalcErrorFromInputVariance()",
default=True,
)
scaleZeroPoint = pexConfig.ConfigurableField(
target=ScaleZeroPointTask,
doc="Task to adjust the photometric zero point of the coadd temp exposures",
)
doInterp = pexConfig.Field(
doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
dtype=bool,
default=True,
)
interpImage = pexConfig.ConfigurableField(
target=InterpImageTask,
doc="Task to interpolate (and extrapolate) over NaN pixels",
)
doWrite = pexConfig.Field(
doc="Persist coadd?",
dtype=bool,
default=True,
)
doNImage = pexConfig.Field(
doc="Create image of number of contributing exposures for each pixel",
dtype=bool,
default=False,
)
doUsePsfMatchedPolygons = pexConfig.Field(
doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
dtype=bool,
default=False,
)
maskPropagationThresholds = pexConfig.DictField(
keytype=str,
itemtype=float,
doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
"the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
"would have contributed exceeds this value."),
default={"SAT": 0.1},
)
removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
doc="Mask planes to remove before coadding")
doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
doc="Set mask and flag bits for bright objects?")
brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
doc="Name of mask bit used for bright objects")
coaddPsf = pexConfig.ConfigField(
doc="Configuration for CoaddPsf",
dtype=measAlg.CoaddPsfConfig,
)
doAttachTransmissionCurve = pexConfig.Field(
dtype=bool, default=False, optional=False,
doc=("Attach a piecewise TransmissionCurve for the coadd? "
"(requires all input Exposures to have TransmissionCurves).")
)
hasFakes = pexConfig.Field(
dtype=bool,
default=False,
doc="Should be set to True if fake sources have been inserted into the input data."
)
doSelectVisits = pexConfig.Field(
doc="Coadd only visits selected by a SelectVisitsTask",
dtype=bool,
default=False,
)
doInputMap = pexConfig.Field(
doc="Create a bitwise map of coadd inputs",
dtype=bool,
default=False,
)
inputMapper = pexConfig.ConfigurableField(
doc="Input map creation subtask.",
target=HealSparseInputMapTask,
)
def setDefaults(self):
super().setDefaults()
self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
def validate(self):
super().validate()
if self.doPsfMatch:
# Backwards compatibility.
# Configs do not have loggers
log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
self.warpType = 'psfMatched'
if self.doSigmaClip and self.statistic != "MEANCLIP":
log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
self.statistic = "MEANCLIP"
if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
raise ValueError("Must set doInterp=False for statistic=%s, which does not "
"compute and set a non-zero coadd variance estimate." % (self.statistic))
unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
if str(k) not in unstackableStats]
raise ValueError("statistic %s is not allowed. Please choose one of %s."
% (self.statistic, stackableStats))
class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
ConfigClass = AssembleCoaddConfig
_DefaultName = "assembleCoadd"
def __init__(self, *args, **kwargs):
# TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
if args:
argNames = ["config", "name", "parentTask", "log"]
kwargs.update({k: v for k, v in zip(argNames, args)})
warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
"PipelineTask will not take positional args" % argNames, FutureWarning)
super().__init__(**kwargs)
self.makeSubtask("interpImage")
self.makeSubtask("scaleZeroPoint")
if self.config.doMaskBrightObjects:
mask = afwImage.Mask()
try:
self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
except pexExceptions.LsstCppException:
raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
mask.getMaskPlaneDict().keys())
del mask
if self.config.doInputMap:
self.makeSubtask("inputMapper")
self.warpType = self.config.warpType
@utils.inheritDoc(pipeBase.PipelineTask)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputData = butlerQC.get(inputRefs)
# Construct skyInfo expected by run
# Do not remove skyMap from inputData in case _makeSupplementaryData needs it
skyMap = inputData["skyMap"]
outputDataId = butlerQC.quantum.dataId
inputData['skyInfo'] = makeSkyInfo(skyMap,
tractId=outputDataId['tract'],
patchId=outputDataId['patch'])
if self.config.doSelectVisits:
warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
else:
warpRefList = inputData['inputWarps']
inputs = self.prepareInputs(warpRefList)
self.log.info("Found %d %s", len(inputs.tempExpRefList),
self.getTempExpDatasetName(self.warpType))
if len(inputs.tempExpRefList) == 0:
raise pipeBase.NoWorkFound("No coadd temporary exposures found")
supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
inputs.weightList, supplementaryData=supplementaryData)
inputData.setdefault('brightObjectMask', None)
self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
if self.config.doWrite:
butlerQC.put(retStruct, outputRefs)
return retStruct
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
if self.config.doInterp:
self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
# The variance must be positive; work around for DM-3201.
varArray = coaddExposure.variance.array
with numpy.errstate(invalid="ignore"):
varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
if self.config.doMaskBrightObjects:
self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
return pipeBase.Struct()
@deprecated(
reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData",
version="v25.0",
category=FutureWarning
)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
def prepareInputs(self, refList):
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(self.config.sigmaClip)
statsCtrl.setNumIter(self.config.clipIter)
statsCtrl.setAndMask(self.getBadPixelMask())
statsCtrl.setNanSafe(True)
# compute tempExpRefList: a list of tempExpRef that actually exist
# and weightList: a list of the weight of the associated coadd tempExp
# and imageScalerList: a list of scale factors for the associated coadd tempExp
tempExpRefList = []
weightList = []
imageScalerList = []
tempExpName = self.getTempExpDatasetName(self.warpType)
for tempExpRef in refList:
tempExp = tempExpRef.get()
# Ignore any input warp that is empty of data
if numpy.isnan(tempExp.image.array).all():
continue
maskedImage = tempExp.getMaskedImage()
imageScaler = self.scaleZeroPoint.computeImageScaler(
exposure=tempExp,
dataRef=tempExpRef, # FIXME
)
try:
imageScaler.scaleMaskedImage(maskedImage)
except Exception as e:
self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
continue
statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
afwMath.MEANCLIP, statsCtrl)
meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
weight = 1.0 / float(meanVar)
if not numpy.isfinite(weight):
self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId)
continue
self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
del maskedImage
del tempExp
tempExpRefList.append(tempExpRef)
weightList.append(weight)
imageScalerList.append(imageScaler)
return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
imageScalerList=imageScalerList)
def prepareStats(self, mask=None):
if mask is None:
mask = self.getBadPixelMask()
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(self.config.sigmaClip)
statsCtrl.setNumIter(self.config.clipIter)
statsCtrl.setAndMask(mask)
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
for plane, threshold in self.config.maskPropagationThresholds.items():
bit = afwImage.Mask.getMaskPlane(plane)
statsCtrl.setMaskPropagationThreshold(bit, threshold)
statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
@timeMethod
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
altMaskList=None, mask=None, supplementaryData=None):
tempExpName = self.getTempExpDatasetName(self.warpType)
self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
stats = self.prepareStats(mask=mask)
if altMaskList is None:
altMaskList = [None]*len(tempExpRefList)
coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
coaddMaskedImage = coaddExposure.getMaskedImage()
subregionSizeArr = self.config.subregionSize
subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
# if nImage is requested, create a zero one which can be passed to assembleSubregion
if self.config.doNImage:
nImage = afwImage.ImageU(skyInfo.bbox)
else:
nImage = None
# If inputMap is requested, create the initial version that can be masked in
# assembleSubregion.
if self.config.doInputMap:
self.inputMapper.build_ccd_input_map(skyInfo.bbox,
skyInfo.wcs,
coaddExposure.getInfo().getCoaddInputs().ccds)
if self.config.doOnlineForMean and self.config.statistic == "MEAN":
try:
self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
weightList, altMaskList, stats.ctrl,
nImage=nImage)
except Exception as e:
self.log.exception("Cannot compute online coadd %s", e)
raise
else:
for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
try:
self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
weightList, altMaskList, stats.flags, stats.ctrl,
nImage=nImage)
except Exception as e:
self.log.exception("Cannot compute coadd %s: %s", subBBox, e)
raise
# If inputMap is requested, we must finalize the map after the accumulation.
if self.config.doInputMap:
self.inputMapper.finalize_ccd_input_map_mask()
inputMap = self.inputMapper.ccd_input_map
else:
inputMap = None
self.setInexactPsf(coaddMaskedImage.getMask())
# Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
# pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
warpRefList=tempExpRefList, imageScalerList=imageScalerList,
weightList=weightList, inputMap=inputMap)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
assert len(tempExpRefList) == len(weightList), "Length mismatch"
# We load a single pixel of each coaddTempExp, because we just want to get at the metadata
# (and we need more than just the PropertySet that contains the header), which is not possible
# with the current butler (see #2777).
bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
# Set the coadd FilterLabel to the band of the first input exposure:
# Coadds are calibrated, so the physical label is now meaningless.
coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
coaddInputs = coaddExposure.getInfo().getCoaddInputs()
coaddInputs.ccds.reserve(numCcds)
coaddInputs.visits.reserve(len(tempExpList))
for tempExp, weight in zip(tempExpList, weightList):
self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
if self.config.doUsePsfMatchedPolygons:
self.shrinkValidPolygons(coaddInputs)
coaddInputs.visits.sort()
coaddInputs.ccds.sort()
if self.warpType == "psfMatched":
# The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
# ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
# Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
# having the maximum width (sufficient because square)
modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
for modelPsf in modelPsfList]
psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
else:
psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
self.config.coaddPsf.makeControl())
coaddExposure.setPsf(psf)
apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
coaddExposure.getWcs())
coaddExposure.getInfo().setApCorrMap(apCorrMap)
if self.config.doAttachTransmissionCurve:
transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
altMaskList, statsFlags, statsCtrl, nImage=None):
self.log.debug("Computing coadd over %s", bbox)
coaddExposure.mask.addMaskPlane("REJECTED")
coaddExposure.mask.addMaskPlane("CLIPPED")
coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
maskMap = self.setRejectedMaskMapping(statsCtrl)
clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
maskedImageList = []
if nImage is not None:
subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
exposure = tempExpRef.get(parameters={'bbox': bbox})
maskedImage = exposure.getMaskedImage()
mask = maskedImage.getMask()
if altMask is not None:
self.applyAltMaskPlanes(mask, altMask)
imageScaler.scaleMaskedImage(maskedImage)
# Add 1 for each pixel which is not excluded by the exclude mask.
# In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
if nImage is not None:
subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
if self.config.removeMaskPlanes:
self.removeMaskPlanes(maskedImage)
maskedImageList.append(maskedImage)
if self.config.doInputMap:
visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
with self.timer("stack"):
coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
clipped, # also set output to CLIPPED if sigma-clipped
maskMap)
coaddExposure.maskedImage.assign(coaddSubregion, bbox)
if nImage is not None:
nImage.assign(subNImage, bbox)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList,
altMaskList, statsCtrl, nImage=None):
Definition at line 705 of file assembleCoadd.py.