37import lsst.utils
as utils
39from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
40from .interpImage
import InterpImageTask
41from .scaleZeroPoint
import ScaleZeroPointTask
42from .maskStreaks
import MaskStreaksTask
43from .healSparseMapping
import HealSparseInputMapTask
45from lsst.utils.timer
import timeMethod
46from deprecated.sphinx
import deprecated
48__all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
49 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
51log = logging.getLogger(__name__)
55 dimensions=(
"tract",
"patch",
"band",
"skymap"),
56 defaultTemplates={
"inputCoaddName":
"deep",
57 "outputCoaddName":
"deep",
59 "warpTypeSuffix":
""}):
61 inputWarps = pipeBase.connectionTypes.Input(
62 doc=(
"Input list of warps to be assemebled i.e. stacked."
63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
64 name=
"{inputCoaddName}Coadd_{warpType}Warp",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
70 skyMap = pipeBase.connectionTypes.Input(
71 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
72 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
73 storageClass=
"SkyMap",
74 dimensions=(
"skymap", ),
76 selectedVisits = pipeBase.connectionTypes.Input(
77 doc=
"Selected visits to be coadded.",
78 name=
"{outputCoaddName}Visits",
79 storageClass=
"StructuredDataDict",
80 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band")
82 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
83 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
85 name=
"brightObjectMask",
86 storageClass=
"ObjectMaskCatalog",
87 dimensions=(
"tract",
"patch",
"skymap",
"band"),
89 coaddExposure = pipeBase.connectionTypes.Output(
90 doc=
"Output coadded exposure, produced by stacking input warps",
91 name=
"{outputCoaddName}Coadd{warpTypeSuffix}",
92 storageClass=
"ExposureF",
93 dimensions=(
"tract",
"patch",
"skymap",
"band"),
95 nImage = pipeBase.connectionTypes.Output(
96 doc=
"Output image of number of input images per pixel",
97 name=
"{outputCoaddName}Coadd_nImage",
98 storageClass=
"ImageU",
99 dimensions=(
"tract",
"patch",
"skymap",
"band"),
101 inputMap = pipeBase.connectionTypes.Output(
102 doc=
"Output healsparse map of input images",
103 name=
"{outputCoaddName}Coadd_inputMap",
104 storageClass=
"HealSparseMap",
105 dimensions=(
"tract",
"patch",
"skymap",
"band"),
108 def __init__(self, *, config=None):
109 super().__init__(config=config)
111 if not config.doMaskBrightObjects:
112 self.prerequisiteInputs.remove(
"brightObjectMask")
114 if not config.doSelectVisits:
115 self.inputs.remove(
"selectedVisits")
117 if not config.doNImage:
118 self.outputs.remove(
"nImage")
120 if not self.config.doInputMap:
121 self.outputs.remove(
"inputMap")
124class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
125 pipelineConnections=AssembleCoaddConnections):
126 """Configuration parameters for the `AssembleCoaddTask`.
130 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
131 only set the bitplane config.brightObjectMaskName. To make this useful you
132 *must* also configure the flags.pixel algorithm,
for example by adding
136 config.measurement.plugins[
"base_PixelFlags"].masksFpCenter.append(
"BRIGHT_OBJECT")
137 config.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere.append(
"BRIGHT_OBJECT")
139 to your measureCoaddSources.py
and forcedPhotCoadd.py config overrides.
141 warpType = pexConfig.Field(
142 doc="Warp name: one of 'direct' or 'psfMatched'",
146 subregionSize = pexConfig.ListField(
148 doc=
"Width, height of stack subregion size; "
149 "make small enough that a full stack of images will fit into memory at once.",
151 default=(2000, 2000),
153 statistic = pexConfig.Field(
155 doc=
"Main stacking statistic for aggregating over the epochs.",
158 doOnlineForMean = pexConfig.Field(
160 doc=
"Perform online coaddition when statistic=\"MEAN\" to save memory?",
163 doSigmaClip = pexConfig.Field(
165 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
168 sigmaClip = pexConfig.Field(
170 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
173 clipIter = pexConfig.Field(
175 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
178 calcErrorFromInputVariance = pexConfig.Field(
180 doc=
"Calculate coadd variance from input variance by stacking statistic."
181 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
184 scaleZeroPoint = pexConfig.ConfigurableField(
185 target=ScaleZeroPointTask,
186 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
188 doInterp = pexConfig.Field(
189 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
193 interpImage = pexConfig.ConfigurableField(
194 target=InterpImageTask,
195 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
197 doWrite = pexConfig.Field(
198 doc=
"Persist coadd?",
202 doNImage = pexConfig.Field(
203 doc=
"Create image of number of contributing exposures for each pixel",
207 doUsePsfMatchedPolygons = pexConfig.Field(
208 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
212 maskPropagationThresholds = pexConfig.DictField(
215 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
216 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
217 "would have contributed exceeds this value."),
218 default={
"SAT": 0.1},
220 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
221 doc=
"Mask planes to remove before coadding")
222 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
223 doc=
"Set mask and flag bits for bright objects?")
224 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
225 doc=
"Name of mask bit used for bright objects")
226 coaddPsf = pexConfig.ConfigField(
227 doc=
"Configuration for CoaddPsf",
228 dtype=measAlg.CoaddPsfConfig,
230 doAttachTransmissionCurve = pexConfig.Field(
231 dtype=bool, default=
False, optional=
False,
232 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
233 "(requires all input Exposures to have TransmissionCurves).")
235 hasFakes = pexConfig.Field(
238 doc=
"Should be set to True if fake sources have been inserted into the input data."
240 doSelectVisits = pexConfig.Field(
241 doc=
"Coadd only visits selected by a SelectVisitsTask",
245 doInputMap = pexConfig.Field(
246 doc=
"Create a bitwise map of coadd inputs",
250 inputMapper = pexConfig.ConfigurableField(
251 doc=
"Input map creation subtask.",
252 target=HealSparseInputMapTask,
255 def setDefaults(self):
256 super().setDefaults()
257 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
264 log.warning(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
265 self.warpType =
'psfMatched'
266 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
267 log.warning(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
268 self.statistic =
"MEANCLIP"
269 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
270 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
271 "compute and set a non-zero coadd variance estimate." % (self.statistic))
273 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
274 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
275 stackableStats = [
str(k)
for k
in afwMath.Property.__members__.keys()
276 if str(k)
not in unstackableStats]
277 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
278 % (self.statistic, stackableStats))
281class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
282 """Assemble a coadded image from a set of warps.
284 Each Warp that goes into a coadd will typically have an independent
285 photometric zero-point. Therefore, we must scale each Warp to set it to
286 a common photometric zeropoint. WarpType may be one of 'direct' or
287 'psfMatched',
and the boolean configs `config.makeDirect`
and
288 `config.makePsfMatched` set which of the warp types will be coadded.
289 The coadd
is computed
as a mean
with optional outlier rejection.
290 Criteria
for outlier rejection are set
in `AssembleCoaddConfig`.
291 Finally, Warps can have bad
'NaN' pixels which received no input
from the
292 source calExps. We interpolate over these bad (NaN) pixels.
294 ConfigClass = AssembleCoaddConfig
295 _DefaultName = "assembleCoadd"
297 def __init__(self, *args, **kwargs):
300 argNames = [
"config",
"name",
"parentTask",
"log"]
301 kwargs.update({k: v
for k, v
in zip(argNames, args)})
302 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
303 "PipelineTask will not take positional args" % argNames, FutureWarning)
305 super().__init__(**kwargs)
306 self.makeSubtask(
"interpImage")
307 self.makeSubtask(
"scaleZeroPoint")
309 if self.config.doMaskBrightObjects:
310 mask = afwImage.Mask()
312 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
313 except pexExceptions.LsstCppException:
314 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
315 mask.getMaskPlaneDict().
keys())
318 if self.config.doInputMap:
319 self.makeSubtask(
"inputMapper")
321 self.warpType = self.config.warpType
323 @utils.inheritDoc(pipeBase.PipelineTask)
324 def runQuantum(self, butlerQC, inputRefs, outputRefs):
325 inputData = butlerQC.get(inputRefs)
329 skyMap = inputData[
"skyMap"]
330 outputDataId = butlerQC.quantum.dataId
333 tractId=outputDataId[
'tract'],
334 patchId=outputDataId[
'patch'])
336 if self.config.doSelectVisits:
337 warpRefList = self.filterWarps(inputData[
'inputWarps'], inputData[
'selectedVisits'])
339 warpRefList = inputData[
'inputWarps']
341 inputs = self.prepareInputs(warpRefList)
342 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
343 self.getTempExpDatasetName(self.warpType))
344 if len(inputs.tempExpRefList) == 0:
345 raise pipeBase.NoWorkFound(
"No coadd temporary exposures found")
347 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
348 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
349 inputs.weightList, supplementaryData=supplementaryData)
351 inputData.setdefault(
'brightObjectMask',
None)
352 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
354 if self.config.doWrite:
355 butlerQC.put(retStruct, outputRefs)
358 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
359 """Interpolate over missing data and mask bright stars.
364 The coadded exposure to process.
366 Table of bright objects to mask.
367 dataId : `lsst.daf.butler.DataId`, optional
370 if self.config.doInterp:
371 self.interpImage.run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
373 varArray = coaddExposure.variance.array
374 with numpy.errstate(invalid=
"ignore"):
375 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
377 if self.config.doMaskBrightObjects:
378 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
380 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
381 """Make additional inputs to run() specific to subclasses (Gen3)
383 Duplicates interface of `runQuantum` method.
384 Available to be implemented by subclasses only if they need the
385 coadd dataRef
for performing preliminary processing before
386 assembling the coadd.
390 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
391 Gen3 Butler object
for fetching additional data products before
392 running the Task specialized
for quantum being processed
393 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
394 Attributes are the names of the connections describing input dataset types.
395 Values are DatasetRefs that task consumes
for corresponding dataset type.
396 DataIds are guaranteed to match data objects
in ``inputData``.
397 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
398 Attributes are the names of the connections describing output dataset types.
399 Values are DatasetRefs that task
is to produce
400 for corresponding dataset type.
402 return pipeBase.Struct()
405 reason=
"makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData",
407 category=FutureWarning
409 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
410 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
412 def prepareInputs(self, refList):
413 """Prepare the input warps for coaddition by measuring the weight for
414 each warp and the scaling
for the photometric zero point.
416 Each Warp has its own photometric zeropoint
and background variance.
417 Before coadding these Warps together, compute a scale factor to
418 normalize the photometric zeropoint
and compute the weight
for each Warp.
423 List of data references to tempExp
427 result : `lsst.pipe.base.Struct`
428 Result struct
with components:
430 - ``tempExprefList``: `list` of data references to tempExp.
431 - ``weightList``: `list` of weightings.
432 - ``imageScalerList``: `list` of image scalers.
434 statsCtrl = afwMath.StatisticsControl()
435 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
436 statsCtrl.setNumIter(self.config.clipIter)
437 statsCtrl.setAndMask(self.getBadPixelMask())
438 statsCtrl.setNanSafe(True)
445 tempExpName = self.getTempExpDatasetName(self.warpType)
446 for tempExpRef
in refList:
447 tempExp = tempExpRef.get()
449 if numpy.isnan(tempExp.image.array).all():
451 maskedImage = tempExp.getMaskedImage()
452 imageScaler = self.scaleZeroPoint.computeImageScaler(
457 imageScaler.scaleMaskedImage(maskedImage)
458 except Exception
as e:
459 self.log.warning(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
461 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
462 afwMath.MEANCLIP, statsCtrl)
463 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
464 weight = 1.0 / float(meanVar)
465 if not numpy.isfinite(weight):
466 self.log.warning(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
468 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
473 tempExpRefList.append(tempExpRef)
474 weightList.append(weight)
475 imageScalerList.append(imageScaler)
477 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
478 imageScalerList=imageScalerList)
480 def prepareStats(self, mask=None):
481 """Prepare the statistics for coadding images.
485 mask : `int`, optional
486 Bit mask value to exclude from coaddition.
490 stats : `lsst.pipe.base.Struct`
491 Statistics structure
with the following fields:
493 - ``statsCtrl``: Statistics control object
for coadd
495 - ``statsFlags``: Statistic
for coadd (`lsst.afw.math.Property`)
498 mask = self.getBadPixelMask()
499 statsCtrl = afwMath.StatisticsControl()
500 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
501 statsCtrl.setNumIter(self.config.clipIter)
502 statsCtrl.setAndMask(mask)
503 statsCtrl.setNanSafe(
True)
504 statsCtrl.setWeighted(
True)
505 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
506 for plane, threshold
in self.config.maskPropagationThresholds.items():
507 bit = afwImage.Mask.getMaskPlane(plane)
508 statsCtrl.setMaskPropagationThreshold(bit, threshold)
509 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
510 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
513 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
514 altMaskList=None, mask=None, supplementaryData=None):
515 """Assemble a coadd from input warps
517 Assemble the coadd using the provided list of coaddTempExps. Since
518 the full coadd covers a patch (a large area), the assembly is
519 performed over small areas on the image at a time
in order to
520 conserve memory usage. Iterate over subregions within the outer
521 bbox of the patch using `assembleSubregion` to stack the corresponding
522 subregions
from the coaddTempExps
with the statistic specified.
523 Set the edge bits the coadd mask based on the weight map.
527 skyInfo : `lsst.pipe.base.Struct`
528 Struct
with geometric information about the patch.
529 tempExpRefList : `list`
530 List of data references to Warps (previously called CoaddTempExps).
531 imageScalerList : `list`
532 List of image scalers.
535 altMaskList : `list`, optional
536 List of alternate masks to use rather than those stored
with
538 mask : `int`, optional
539 Bit mask value to exclude
from coaddition.
540 supplementaryData : lsst.pipe.base.Struct, optional
541 Struct
with additional data products needed to assemble coadd.
542 Only used by subclasses that implement `_makeSupplementaryData`
547 result : `lsst.pipe.base.Struct`
548 Result struct
with components:
552 - ``inputMap``: bit-wise map of inputs,
if requested.
553 - ``warpRefList``: input list of refs to the warps (
554 ``lsst.daf.butler.DeferredDatasetHandle``)
556 - ``imageScalerList``: input list of image scalers (unmodified)
557 - ``weightList``: input list of weights (unmodified)
559 tempExpName = self.getTempExpDatasetName(self.warpType)
560 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
561 stats = self.prepareStats(mask=mask)
563 if altMaskList
is None:
564 altMaskList = [
None]*len(tempExpRefList)
566 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
567 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
568 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
569 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
570 coaddMaskedImage = coaddExposure.getMaskedImage()
571 subregionSizeArr = self.config.subregionSize
572 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
574 if self.config.doNImage:
575 nImage = afwImage.ImageU(skyInfo.bbox)
580 if self.config.doInputMap:
581 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
583 coaddExposure.getInfo().getCoaddInputs().ccds)
585 if self.config.doOnlineForMean
and self.config.statistic ==
"MEAN":
587 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
588 weightList, altMaskList, stats.ctrl,
590 except Exception
as e:
591 self.log.exception(
"Cannot compute online coadd %s", e)
594 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
596 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
597 weightList, altMaskList, stats.flags, stats.ctrl,
599 except Exception
as e:
600 self.log.exception(
"Cannot compute coadd %s: %s", subBBox, e)
604 if self.config.doInputMap:
605 self.inputMapper.finalize_ccd_input_map_mask()
606 inputMap = self.inputMapper.ccd_input_map
610 self.setInexactPsf(coaddMaskedImage.getMask())
613 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
614 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
615 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
616 weightList=weightList, inputMap=inputMap)
618 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
619 """Set the metadata for the coadd.
621 This basic implementation sets the filter from the first input.
626 The target exposure
for the coadd.
627 tempExpRefList : `list`
628 List of data references to tempExp.
632 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
639 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
641 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
645 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
646 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
647 coaddInputs.ccds.reserve(numCcds)
648 coaddInputs.visits.reserve(len(tempExpList))
650 for tempExp, weight
in zip(tempExpList, weightList):
651 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
653 if self.config.doUsePsfMatchedPolygons:
654 self.shrinkValidPolygons(coaddInputs)
656 coaddInputs.visits.sort()
657 coaddInputs.ccds.sort()
658 if self.warpType ==
"psfMatched":
663 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
664 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
665 for modelPsf
in modelPsfList]
666 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
668 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
669 self.config.coaddPsf.makeControl())
670 coaddExposure.setPsf(psf)
671 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
672 coaddExposure.getWcs())
673 coaddExposure.getInfo().setApCorrMap(apCorrMap)
674 if self.config.doAttachTransmissionCurve:
675 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
676 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
678 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
679 altMaskList, statsFlags, statsCtrl, nImage=None):
680 """Assemble the coadd for a sub-region.
682 For each coaddTempExp, check for (
and swap
in) an alternative mask
683 if one
is passed. Remove mask planes listed
in
684 `config.removeMaskPlanes`. Finally, stack the actual exposures using
685 `lsst.afw.math.statisticsStack`
with the statistic specified by
686 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
for
687 a mean-stack
or `lsst.afw.math.MEANCLIP`
for outlier rejection using
688 an N-sigma clipped mean where N
and iterations are specified by
689 statsCtrl. Assign the stacked subregion back to the coadd.
694 The target exposure
for the coadd.
695 bbox : `lsst.geom.Box`
697 tempExpRefList : `list`
698 List of data reference to tempExp.
699 imageScalerList : `list`
700 List of image scalers.
704 List of alternate masks to use rather than those stored
with
705 tempExp,
or None. Each element
is dict
with keys = mask plane
706 name to which to add the spans.
707 statsFlags : `lsst.afw.math.Property`
708 Property object
for statistic
for coadd.
710 Statistics control object
for coadd.
711 nImage : `lsst.afw.image.ImageU`, optional
712 Keeps track of exposure count
for each pixel.
714 self.log.debug("Computing coadd over %s", bbox)
716 coaddExposure.mask.addMaskPlane(
"REJECTED")
717 coaddExposure.mask.addMaskPlane(
"CLIPPED")
718 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
719 maskMap = self.setRejectedMaskMapping(statsCtrl)
720 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
722 if nImage
is not None:
723 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
724 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
726 exposure = tempExpRef.get(parameters={
'bbox': bbox})
728 maskedImage = exposure.getMaskedImage()
729 mask = maskedImage.getMask()
730 if altMask
is not None:
731 self.applyAltMaskPlanes(mask, altMask)
732 imageScaler.scaleMaskedImage(maskedImage)
736 if nImage
is not None:
737 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
738 if self.config.removeMaskPlanes:
739 self.removeMaskPlanes(maskedImage)
740 maskedImageList.append(maskedImage)
742 if self.config.doInputMap:
743 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
744 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
746 with self.timer(
"stack"):
747 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
750 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
751 if nImage
is not None:
752 nImage.assign(subNImage, bbox)
754 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList,
755 altMaskList, statsCtrl, nImage=None):
756 """Assemble the coadd using the "online" method.
758 This method takes a running sum of images and weights to save memory.
759 It only works
for MEAN statistics.
764 The target exposure
for the coadd.
765 tempExpRefList : `list`
766 List of data reference to tempExp.
767 imageScalerList : `list`
768 List of image scalers.
772 List of alternate masks to use rather than those stored
with
773 tempExp,
or None. Each element
is dict
with keys = mask plane
774 name to which to add the spans.
776 Statistics control object
for coadd
777 nImage : `lsst.afw.image.ImageU`, optional
778 Keeps track of exposure count
for each pixel.
780 self.log.debug("Computing online coadd.")
782 coaddExposure.mask.addMaskPlane(
"REJECTED")
783 coaddExposure.mask.addMaskPlane(
"CLIPPED")
784 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
785 maskMap = self.setRejectedMaskMapping(statsCtrl)
786 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
788 bbox = coaddExposure.maskedImage.getBBox()
790 stacker = AccumulatorMeanStack(
791 coaddExposure.image.array.shape,
792 statsCtrl.getAndMask(),
793 mask_threshold_dict=thresholdDict,
795 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
796 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
797 compute_n_image=(nImage
is not None)
800 for tempExpRef, imageScaler, altMask, weight
in zip(tempExpRefList,
804 exposure = tempExpRef.get()
805 maskedImage = exposure.getMaskedImage()
806 mask = maskedImage.getMask()
807 if altMask
is not None:
808 self.applyAltMaskPlanes(mask, altMask)
809 imageScaler.scaleMaskedImage(maskedImage)
810 if self.config.removeMaskPlanes:
811 self.removeMaskPlanes(maskedImage)
813 stacker.add_masked_image(maskedImage, weight=weight)
815 if self.config.doInputMap:
816 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
817 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
819 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
821 if nImage
is not None:
822 nImage.array[:, :] = stacker.n_image
824 def removeMaskPlanes(self, maskedImage):
825 """Unset the mask of an image for mask planes specified in the config.
830 The masked image to be modified.
832 mask = maskedImage.getMask()
833 for maskPlane
in self.config.removeMaskPlanes:
835 mask &= ~mask.getPlaneBitMask(maskPlane)
836 except pexExceptions.InvalidParameterError:
837 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
841 def setRejectedMaskMapping(statsCtrl):
842 """Map certain mask planes of the warps to new planes for the coadd.
844 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
845 or CLIPPED, set it to REJECTED on the coadd.
846 If a pixel
is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
847 If a pixel
is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
852 Statistics control object
for coadd
856 maskMap : `list` of `tuple` of `int`
857 A list of mappings of mask planes of the warped exposures to
858 mask planes of the coadd.
860 edge = afwImage.Mask.getPlaneBitMask("EDGE")
861 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
862 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
863 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
864 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
865 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
869 def applyAltMaskPlanes(self, mask, altMaskSpans):
870 """Apply in place alt mask formatted as SpanSets to a mask.
876 altMaskSpans : `dict`
877 SpanSet lists to apply. Each element contains the new mask
878 plane name (e.g. "CLIPPED and/or "NO_DATA
") as the key,
879 and list of SpanSets to apply to the mask.
886 if self.config.doUsePsfMatchedPolygons:
887 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
892 for spanSet
in altMaskSpans[
'NO_DATA']:
893 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
895 for plane, spanSetList
in altMaskSpans.items():
896 maskClipValue = mask.addMaskPlane(plane)
897 for spanSet
in spanSetList:
898 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
902 """Shrink coaddInputs' ccds' ValidPolygons in place.
904 Either modify each ccd's validPolygon in place, or if CoaddInputs
905 does not have a validPolygon, create one
from its bbox.
909 coaddInputs : `lsst.afw.image.coaddInputs`
913 for ccd
in coaddInputs.ccds:
914 polyOrig = ccd.getValidPolygon()
915 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
916 validPolyBBox.grow(-self.config.matchingKernelSize//2)
918 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
920 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
921 ccd.setValidPolygon(validPolygon)
924 """Set the bright object masks.
929 Exposure under consideration.
931 Table of bright objects to mask.
932 dataId : `lsst.daf.butler.DataId`, optional
933 Data identifier dict for patch.
936 if brightObjectMasks
is None:
937 self.log.warning(
"Unable to apply bright object mask: none supplied")
939 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
940 mask = exposure.getMaskedImage().getMask()
941 wcs = exposure.getWcs()
942 plateScale = wcs.getPixelScale().asArcseconds()
944 for rec
in brightObjectMasks:
945 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
946 if rec[
"type"] ==
"box":
947 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
948 width = rec[
"width"].asArcseconds()/plateScale
949 height = rec[
"height"].asArcseconds()/plateScale
952 bbox =
geom.Box2I(center - halfSize, center + halfSize)
955 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
956 spans = afwGeom.SpanSet(bbox)
957 elif rec[
"type"] ==
"circle":
958 radius = int(rec[
"radius"].asArcseconds()/plateScale)
959 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
961 self.log.warning(
"Unexpected region type %s at %s", rec[
"type"], center)
963 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
966 """Set INEXACT_PSF mask plane.
968 If any of the input images isn't represented in the coadd (due to
969 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
975 Coadded exposure
's mask, modified in-place.
977 mask.addMaskPlane("INEXACT_PSF")
978 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
979 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
980 clipped = mask.getPlaneBitMask(
"CLIPPED")
981 rejected = mask.getPlaneBitMask(
"REJECTED")
982 array = mask.getArray()
983 selected = array & (sensorEdge | clipped | rejected) > 0
984 array[selected] |= inexactPsf
987 def _subBBoxIter(bbox, subregionSize):
988 """Iterate over subregions of a bbox.
993 Bounding box over which to iterate.
1000 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1001 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1002 the edges of ``bbox``, but it will never be empty.
1005 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1006 if subregionSize[0] < 1
or subregionSize[1] < 1:
1007 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1009 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1010 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1013 if subBBox.isEmpty():
1014 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1015 "colShift=%s, rowShift=%s" %
1016 (bbox, subregionSize, colShift, rowShift))
1020 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1025 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1027 Dictionary
with good visitIds
as the keys. Value ignored.
1031 filteredInputs : `list`
1032 Filtered
and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1034 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef
for inputRef
in inputs}
1036 for visit
in goodVisits.keys():
1037 if visit
in inputWarpDict:
1038 filteredInputs.append(inputWarpDict[visit])
1039 return filteredInputs
1043 """Function to count the number of pixels with a specific mask in a
1046 Find the intersection of mask & footprint. Count all pixels in the mask
1047 that are
in the intersection that have bitmask set but do
not have
1048 ignoreMask set. Return the count.
1053 Mask to define intersection region by.
1055 Footprint to define the intersection region by.
1057 Specific mask that we wish to count the number of occurances of.
1059 Pixels to
not consider.
1064 Count of number of pixels
in footprint
with specified mask.
1066 bbox = footprint.getBBox()
1067 bbox.clip(mask.getBBox(afwImage.PARENT))
1068 fp = afwImage.Mask(bbox)
1069 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1070 footprint.spans.setMask(fp, bitmask)
1071 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1072 (subMask.getArray() & ignoreMask) == 0).sum()
1076 psfMatchedWarps = pipeBase.connectionTypes.Input(
1077 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1078 "Only PSF-Matched Warps make sense for image subtraction. "
1079 "Therefore, they must be an additional declared input."),
1080 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1081 storageClass=
"ExposureF",
1082 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1086 templateCoadd = pipeBase.connectionTypes.Output(
1087 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1088 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1089 name=
"{outputCoaddName}CoaddPsfMatched",
1090 storageClass=
"ExposureF",
1091 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1096 if not config.assembleStaticSkyModel.doWrite:
1097 self.outputs.remove(
"templateCoadd")
1102 pipelineConnections=CompareWarpAssembleCoaddConnections):
1103 assembleStaticSkyModel = pexConfig.ConfigurableField(
1104 target=AssembleCoaddTask,
1105 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1106 " naive/first-iteration model of the static sky.",
1108 detect = pexConfig.ConfigurableField(
1109 target=SourceDetectionTask,
1110 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1112 detectTemplate = pexConfig.ConfigurableField(
1113 target=SourceDetectionTask,
1114 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1116 maskStreaks = pexConfig.ConfigurableField(
1117 target=MaskStreaksTask,
1118 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1119 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1122 streakMaskName = pexConfig.Field(
1125 doc=
"Name of mask bit used for streaks"
1127 maxNumEpochs = pexConfig.Field(
1128 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1129 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1130 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1131 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1132 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1133 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1134 "than transient and not masked.",
1138 maxFractionEpochsLow = pexConfig.RangeField(
1139 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1140 "Effective maxNumEpochs = "
1141 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1146 maxFractionEpochsHigh = pexConfig.RangeField(
1147 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1148 "Effective maxNumEpochs = "
1149 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1154 spatialThreshold = pexConfig.RangeField(
1155 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1156 "temporal criteria. If 0, clip all. If 1, clip none.",
1160 inclusiveMin=
True, inclusiveMax=
True
1162 doScaleWarpVariance = pexConfig.Field(
1163 doc=
"Rescale Warp variance plane using empirical noise?",
1167 scaleWarpVariance = pexConfig.ConfigurableField(
1168 target=ScaleVarianceTask,
1169 doc=
"Rescale variance on warps",
1171 doPreserveContainedBySource = pexConfig.Field(
1172 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1173 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1177 doPrefilterArtifacts = pexConfig.Field(
1178 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1179 "because they will be excluded anyway. This prevents them from contributing "
1180 "to the outlier epoch count image and potentially being labeled as persistant."
1181 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1185 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1186 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1188 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1190 prefilterArtifactsRatio = pexConfig.Field(
1191 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1195 doFilterMorphological = pexConfig.Field(
1196 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1201 growStreakFp = pexConfig.Field(
1202 doc=
"Grow streak footprints by this number multiplied by the PSF width",
1208 AssembleCoaddConfig.setDefaults(self)
1214 if "EDGE" in self.badMaskPlanes:
1215 self.badMaskPlanes.remove(
'EDGE')
1216 self.removeMaskPlanes.append(
'EDGE')
1225 self.
detect.doTempLocalBackground =
False
1226 self.
detect.reEstimateBackground =
False
1227 self.
detect.returnOriginalFootprints =
False
1228 self.
detect.thresholdPolarity =
"both"
1229 self.
detect.thresholdValue = 5
1230 self.
detect.minPixels = 4
1231 self.
detect.isotropicGrow =
True
1232 self.
detect.thresholdType =
"pixel_stdev"
1233 self.
detect.nSigmaToGrow = 0.4
1244 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
1245 "Please set assembleStaticSkyModel.doNImage=False")
1248 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1249 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1250 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1255 """Assemble a compareWarp coadded image from a set of warps
1256 by masking artifacts detected by comparing PSF-matched warps.
1258 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1259 we clip outliers). The problem
with doing this
is that when computing the
1260 coadd PSF at a given location, individual visit PSFs
from visits
with
1261 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
1262 In this task, we correct
for this behavior by creating a new badMaskPlane
1263 'CLIPPED' which marks pixels
in the individual warps suspected to contain
1264 an artifact. We populate this plane on the input warps by comparing
1265 PSF-matched warps
with a PSF-matched median coadd which serves
as a
1266 model of the static sky. Any group of pixels that deviates
from the
1267 PSF-matched template coadd by more than config.detect.threshold sigma,
1268 is an artifact candidate. The candidates are then filtered to remove
1269 variable sources
and sources that are difficult to subtract such
as
1270 bright stars. This filter
is configured using the config parameters
1271 ``temporalThreshold``
and ``spatialThreshold``. The temporalThreshold
is
1272 the maximum fraction of epochs that the deviation can appear
in and still
1273 be considered an artifact. The spatialThreshold
is the maximum fraction of
1274 pixels
in the footprint of the deviation that appear
in other epochs
1275 (where other epochs
is defined by the temporalThreshold). If the deviant
1276 region meets this criteria of having a significant percentage of pixels
1277 that deviate
in only a few epochs, these pixels have the
'CLIPPED' bit
1278 set
in the mask. These regions will
not contribute to the final coadd.
1279 Furthermore, any routine to determine the coadd PSF can now be cognizant
1280 of clipped regions. Note that the algorithm implemented by this task
is
1281 preliminary
and works correctly
for HSC data. Parameter modifications
and
1282 or considerable redesigning of the algorithm
is likley required
for other
1285 ``CompareWarpAssembleCoaddTask`` sub-classes
1286 ``AssembleCoaddTask``
and instantiates ``AssembleCoaddTask``
1287 as a subtask to generate the TemplateCoadd (the model of the static sky).
1289 ConfigClass = CompareWarpAssembleCoaddConfig
1290 _DefaultName = "compareWarpAssembleCoadd"
1293 AssembleCoaddTask.__init__(self, *args, **kwargs)
1294 self.makeSubtask(
"assembleStaticSkyModel")
1295 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1296 self.makeSubtask(
"detect", schema=detectionSchema)
1297 if self.config.doPreserveContainedBySource:
1298 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1299 if self.config.doScaleWarpVariance:
1300 self.makeSubtask(
"scaleWarpVariance")
1301 if self.config.doFilterMorphological:
1302 self.makeSubtask(
"maskStreaks")
1304 @utils.inheritDoc(AssembleCoaddTask)
1305 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
1307 Generate a templateCoadd to use as a naive model of static sky to
1308 subtract
from PSF-Matched warps.
1312 result : `lsst.pipe.base.Struct`
1313 Result struct
with components:
1319 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
1320 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
1324 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
1325 if self.config.assembleStaticSkyModel.doWrite:
1326 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
1329 del outputRefs.templateCoadd
1330 del staticSkyModelOutputRefs.templateCoadd
1333 if 'nImage' in staticSkyModelOutputRefs.keys():
1334 del staticSkyModelOutputRefs.nImage
1336 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
1337 staticSkyModelOutputRefs)
1338 if templateCoadd
is None:
1341 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
1342 nImage=templateCoadd.nImage,
1343 warpRefList=templateCoadd.warpRefList,
1344 imageScalerList=templateCoadd.imageScalerList,
1345 weightList=templateCoadd.weightList)
1347 def _noTemplateMessage(self, warpType):
1348 warpName = (warpType[0].upper() + warpType[1:])
1349 message =
"""No %(warpName)s warps were found to build the template coadd which is
1350 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1351 first either rerun makeCoaddTempExp
with config.make%(warpName)s=
True or
1352 coaddDriver
with config.makeCoadTempExp.make%(warpName)s=
True, before assembleCoadd.
1354 Alternatively, to use another algorithm
with existing warps, retarget the CoaddDriverConfig to
1355 another algorithm like:
1358 config.assemble.retarget(SafeClipAssembleCoaddTask)
1359 """ % {"warpName": warpName}
1362 @utils.inheritDoc(AssembleCoaddTask)
1364 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1365 supplementaryData, *args, **kwargs):
1366 """Assemble the coadd.
1368 Find artifacts and apply them to the warps
' masks creating a list of
1369 alternative masks with a new
"CLIPPED" plane
and updated
"NO_DATA"
1370 plane. Then
pass these alternative masks to the base
class's `run`
1373 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
1374 that must contain a ``templateCoadd`` that serves
as the
1375 model of the static sky.
1381 dataIds = [ref.dataId
for ref
in tempExpRefList]
1382 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
1384 if dataIds != psfMatchedDataIds:
1385 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
1386 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
1387 psfMatchedDataIds, dataIds)
1388 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
1389 psfMatchedDataIds, dataIds)
1392 spanSetMaskList = self.
findArtifacts(supplementaryData.templateCoadd,
1393 supplementaryData.warpRefList,
1394 supplementaryData.imageScalerList)
1396 badMaskPlanes = self.config.badMaskPlanes[:]
1397 badMaskPlanes.append(
"CLIPPED")
1398 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1400 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1401 spanSetMaskList, mask=badPixelMask)
1405 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1409 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1415 altMaskList : `list`
1416 List of Dicts containing ``spanSet`` lists.
1417 Each element contains the new mask plane name (e.g. "CLIPPED
1418 and/
or "NO_DATA")
as the key,
and list of ``SpanSets`` to apply to
1421 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE",
"INEXACT_PSF"])
1422 for visitMask
in altMaskList:
1423 if "EDGE" in visitMask:
1424 for spanSet
in visitMask[
'EDGE']:
1425 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1430 Loop through warps twice. The first loop builds a map with the count
1431 of how many epochs each pixel deviates
from the templateCoadd by more
1432 than ``config.chiThreshold`` sigma. The second loop takes each
1433 difference image
and filters the artifacts detected
in each using
1434 count map to filter out variable sources
and sources that are
1435 difficult to subtract cleanly.
1440 Exposure to serve
as model of static sky.
1441 tempExpRefList : `list`
1442 List of data references to warps.
1443 imageScalerList : `list`
1444 List of image scalers.
1449 List of dicts containing information about CLIPPED
1450 (i.e., artifacts), NO_DATA,
and EDGE pixels.
1453 self.log.debug("Generating Count Image, and mask lists.")
1454 coaddBBox = templateCoadd.getBBox()
1455 slateIm = afwImage.ImageU(coaddBBox)
1456 epochCountImage = afwImage.ImageU(coaddBBox)
1457 nImage = afwImage.ImageU(coaddBBox)
1458 spanSetArtifactList = []
1459 spanSetNoDataMaskList = []
1460 spanSetEdgeList = []
1461 spanSetBadMorphoList = []
1462 badPixelMask = self.getBadPixelMask()
1465 templateCoadd.mask.clearAllMaskPlanes()
1467 if self.config.doPreserveContainedBySource:
1468 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1470 templateFootprints =
None
1472 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1474 if warpDiffExp
is not None:
1476 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
1477 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1478 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1479 fpSet.positive.merge(fpSet.negative)
1480 footprints = fpSet.positive
1482 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1485 if self.config.doPrefilterArtifacts:
1489 self.detect.clearMask(warpDiffExp.mask)
1490 for spans
in spanSetList:
1491 spans.setImage(slateIm, 1, doClip=
True)
1492 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
1493 epochCountImage += slateIm
1495 if self.config.doFilterMorphological:
1496 maskName = self.config.streakMaskName
1497 _ = self.maskStreaks.
run(warpDiffExp)
1498 streakMask = warpDiffExp.mask
1499 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
1500 streakMask.getPlaneBitMask(maskName)).split()
1502 psf = warpDiffExp.getPsf()
1503 for s, sset
in enumerate(spanSetStreak):
1504 psfShape = psf.computeShape(sset.computeCentroid())
1505 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius()
1506 sset_dilated = sset.dilated(int(dilation))
1507 spanSetStreak[s] = sset_dilated
1513 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1514 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1515 nansMask.setXY0(warpDiffExp.getXY0())
1516 edgeMask = warpDiffExp.mask
1517 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1518 edgeMask.getPlaneBitMask(
"EDGE")).split()
1522 nansMask = afwImage.MaskX(coaddBBox, 1)
1524 spanSetEdgeMask = []
1527 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1529 spanSetNoDataMaskList.append(spanSetNoDataMask)
1530 spanSetArtifactList.append(spanSetList)
1531 spanSetEdgeList.append(spanSetEdgeMask)
1532 if self.config.doFilterMorphological:
1533 spanSetBadMorphoList.append(spanSetStreak)
1536 path = self._dataRef2DebugPath(
"epochCountIm", tempExpRefList[0], coaddLevel=
True)
1537 epochCountImage.writeFits(path)
1539 for i, spanSetList
in enumerate(spanSetArtifactList):
1541 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
1543 spanSetArtifactList[i] = filteredSpanSetList
1544 if self.config.doFilterMorphological:
1545 spanSetArtifactList[i] += spanSetBadMorphoList[i]
1548 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1549 altMasks.append({
'CLIPPED': artifacts,
1555 """Remove artifact candidates covered by bad mask plane.
1557 Any future editing of the candidate list that does not depend on
1558 temporal information should go
in this method.
1562 spanSetList : `list`
1563 List of SpanSets representing artifact candidates.
1565 Exposure containing mask planes used to prefilter.
1569 returnSpanSetList : `list`
1570 List of SpanSets
with artifacts.
1572 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1573 goodArr = (exp.mask.array & badPixelMask) == 0
1574 returnSpanSetList = []
1575 bbox = exp.getBBox()
1576 x0, y0 = exp.getXY0()
1577 for i, span
in enumerate(spanSetList):
1578 y, x = span.clippedTo(bbox).indices()
1579 yIndexLocal = numpy.array(y) - y0
1580 xIndexLocal = numpy.array(x) - x0
1581 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1582 if goodRatio > self.config.prefilterArtifactsRatio:
1583 returnSpanSetList.append(span)
1584 return returnSpanSetList
1586 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1587 """Filter artifact candidates.
1591 spanSetList : `list`
1592 List of SpanSets representing artifact candidates.
1594 Image of accumulated number of warpDiff detections.
1596 Image of the accumulated number of total epochs contributing.
1600 maskSpanSetList : `list`
1601 List of SpanSets with artifacts.
1604 maskSpanSetList = []
1605 x0, y0 = epochCountImage.getXY0()
1606 for i, span
in enumerate(spanSetList):
1607 y, x = span.indices()
1608 yIdxLocal = [y1 - y0
for y1
in y]
1609 xIdxLocal = [x1 - x0
for x1
in x]
1610 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1611 totalN = nImage.array[yIdxLocal, xIdxLocal]
1614 effMaxNumEpochsHighN = (self.config.maxNumEpochs
1615 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1616 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1617 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1618 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
1619 & (outlierN <= effectiveMaxNumEpochs))
1620 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1621 if percentBelowThreshold > self.config.spatialThreshold:
1622 maskSpanSetList.append(span)
1624 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
1626 filteredMaskSpanSetList = []
1627 for span
in maskSpanSetList:
1629 for footprint
in footprintsToExclude.positive.getFootprints():
1630 if footprint.spans.contains(span):
1634 filteredMaskSpanSetList.append(span)
1635 maskSpanSetList = filteredMaskSpanSetList
1637 return maskSpanSetList
1639 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1640 """Fetch a warp from the butler and return a warpDiff.
1643 warpRef : `lsst.daf.butler.DeferredDatasetHandle`
1644 Handle for the warp.
1646 An image scaler object.
1648 Exposure to be substracted
from the scaled warp.
1652 Exposure of the image difference between the warp
and template.
1660 warp = warpRef.get()
1662 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1663 mi = warp.getMaskedImage()
1664 if self.config.doScaleWarpVariance:
1666 self.scaleWarpVariance.
run(mi)
1667 except Exception
as exc:
1668 self.log.warning(
"Unable to rescale variance of warp (%s); leaving it as-is", exc)
1669 mi -= templateCoadd.getMaskedImage()
def __init__(self, *config=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltEdgeMask(self, mask, altMaskList)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
Base class for coaddition.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def filterWarps(self, inputs, goodVisits)
def setInexactPsf(self, mask)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)