41 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
42 from .interpImage
import InterpImageTask
43 from .scaleZeroPoint
import ScaleZeroPointTask
44 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
45 from .scaleVariance
import ScaleVarianceTask
46 from .maskStreaks
import MaskStreaksTask
48 from lsst.daf.butler
import DeferredDatasetHandle
50 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
51 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
52 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
56 dimensions=(
"tract",
"patch",
"band",
"skymap"),
57 defaultTemplates={
"inputCoaddName":
"deep",
58 "outputCoaddName":
"deep",
62 inputWarps = pipeBase.connectionTypes.Input(
63 doc=(
"Input list of warps to be assemebled i.e. stacked."
64 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
65 name=
"{inputCoaddName}Coadd_{warpType}Warp",
66 storageClass=
"ExposureF",
67 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
71 skyMap = pipeBase.connectionTypes.Input(
72 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
73 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
74 storageClass=
"SkyMap",
75 dimensions=(
"skymap", ),
77 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
78 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
80 name=
"brightObjectMask",
81 storageClass=
"ObjectMaskCatalog",
82 dimensions=(
"tract",
"patch",
"skymap",
"band"),
84 coaddExposure = pipeBase.connectionTypes.Output(
85 doc=
"Output coadded exposure, produced by stacking input warps",
86 name=
"{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
87 storageClass=
"ExposureF",
88 dimensions=(
"tract",
"patch",
"skymap",
"band"),
90 nImage = pipeBase.connectionTypes.Output(
91 doc=
"Output image of number of input images per pixel",
92 name=
"{outputCoaddName}Coadd_nImage",
93 storageClass=
"ImageU",
94 dimensions=(
"tract",
"patch",
"skymap",
"band"),
97 def __init__(self, *, config=None):
98 super().__init__(config=config)
103 templateValues = {name: getattr(config.connections, name)
for name
in self.defaultTemplates}
104 templateValues[
'warpType'] = config.warpType
107 templateValues[
'fakesType'] =
"_fakes"
108 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
109 for name
in self.allConnections}
110 self._typeNameToVarName = {v: k
for k, v
in self._nameOverrides.items()}
113 if not config.doMaskBrightObjects:
114 self.prerequisiteInputs.remove(
"brightObjectMask")
116 if not config.doNImage:
117 self.outputs.remove(
"nImage")
120 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
121 pipelineConnections=AssembleCoaddConnections):
122 """Configuration parameters for the `AssembleCoaddTask`.
126 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
127 only set the bitplane config.brightObjectMaskName. To make this useful you
128 *must* also configure the flags.pixel algorithm, for example by adding
132 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
133 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
135 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
137 warpType = pexConfig.Field(
138 doc=
"Warp name: one of 'direct' or 'psfMatched'",
142 subregionSize = pexConfig.ListField(
144 doc=
"Width, height of stack subregion size; "
145 "make small enough that a full stack of images will fit into memory at once.",
147 default=(2000, 2000),
149 statistic = pexConfig.Field(
151 doc=
"Main stacking statistic for aggregating over the epochs.",
154 doSigmaClip = pexConfig.Field(
156 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
159 sigmaClip = pexConfig.Field(
161 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
164 clipIter = pexConfig.Field(
166 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
169 calcErrorFromInputVariance = pexConfig.Field(
171 doc=
"Calculate coadd variance from input variance by stacking statistic."
172 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
175 scaleZeroPoint = pexConfig.ConfigurableField(
176 target=ScaleZeroPointTask,
177 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
179 doInterp = pexConfig.Field(
180 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
184 interpImage = pexConfig.ConfigurableField(
185 target=InterpImageTask,
186 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
188 doWrite = pexConfig.Field(
189 doc=
"Persist coadd?",
193 doNImage = pexConfig.Field(
194 doc=
"Create image of number of contributing exposures for each pixel",
198 doUsePsfMatchedPolygons = pexConfig.Field(
199 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
203 maskPropagationThresholds = pexConfig.DictField(
206 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
207 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
208 "would have contributed exceeds this value."),
209 default={
"SAT": 0.1},
211 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
212 doc=
"Mask planes to remove before coadding")
213 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
214 doc=
"Set mask and flag bits for bright objects?")
215 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
216 doc=
"Name of mask bit used for bright objects")
217 coaddPsf = pexConfig.ConfigField(
218 doc=
"Configuration for CoaddPsf",
219 dtype=measAlg.CoaddPsfConfig,
221 doAttachTransmissionCurve = pexConfig.Field(
222 dtype=bool, default=
False, optional=
False,
223 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
224 "(requires all input Exposures to have TransmissionCurves).")
226 hasFakes = pexConfig.Field(
229 doc=
"Should be set to True if fake sources have been inserted into the input data."
232 def setDefaults(self):
233 super().setDefaults()
234 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
241 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
242 self.warpType =
'psfMatched'
243 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
244 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
245 self.statistic =
"MEANCLIP"
246 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
247 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
248 "compute and set a non-zero coadd variance estimate." % (self.statistic))
250 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
251 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
252 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
253 if str(k)
not in unstackableStats]
254 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
255 % (self.statistic, stackableStats))
258 class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
259 """Assemble a coadded image from a set of warps (coadded temporary exposures).
261 We want to assemble a coadded image from a set of Warps (also called
262 coadded temporary exposures or ``coaddTempExps``).
263 Each input Warp covers a patch on the sky and corresponds to a single
264 run/visit/exposure of the covered patch. We provide the task with a list
265 of Warps (``selectDataList``) from which it selects Warps that cover the
266 specified patch (pointed at by ``dataRef``).
267 Each Warp that goes into a coadd will typically have an independent
268 photometric zero-point. Therefore, we must scale each Warp to set it to
269 a common photometric zeropoint. WarpType may be one of 'direct' or
270 'psfMatched', and the boolean configs `config.makeDirect` and
271 `config.makePsfMatched` set which of the warp types will be coadded.
272 The coadd is computed as a mean with optional outlier rejection.
273 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
274 Finally, Warps can have bad 'NaN' pixels which received no input from the
275 source calExps. We interpolate over these bad (NaN) pixels.
277 `AssembleCoaddTask` uses several sub-tasks. These are
279 - `ScaleZeroPointTask`
280 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
282 - interpolate across bad pixels (NaN) in the final coadd
284 You can retarget these subtasks if you wish.
288 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
289 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
290 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
291 no debug variables of its own. Some of the subtasks may support debug
292 variables. See the documentation for the subtasks for further information.
296 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
297 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
298 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
299 inputs: a data reference to the tract patch and filter to be coadded, and
300 a list of Warps to attempt to coadd. These are specified using ``--id`` and
301 ``--selectId``, respectively:
305 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
306 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
308 Only the Warps that cover the specified tract and patch will be coadded.
309 A list of the available optional arguments can be obtained by calling
310 ``assembleCoadd.py`` with the ``--help`` command line argument:
314 assembleCoadd.py --help
316 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
317 multi-band processing, we will generate the HSC-I & -R band coadds from
318 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
319 assuming that the lsst stack has been already set up, we must set up the
320 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
321 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
322 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
323 coadds, we must first
326 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
328 - create a skymap that covers the area of the sky present in the raw exposures
330 - warp the individual calibrated exposures to the tangent plane of the coadd
332 We can perform all of these steps by running
336 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
338 This will produce warped exposures for each visit. To coadd the warped
339 data, we call assembleCoadd.py as follows:
343 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
344 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
345 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
346 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
347 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
348 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
349 --selectId visit=903988 ccd=24
351 that will process the HSC-I band data. The results are written in
352 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
354 You may also choose to run:
358 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
359 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
360 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
361 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
362 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
363 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
364 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
365 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
367 to generate the coadd for the HSC-R band if you are interested in
368 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
369 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
370 rather than `AssembleCoaddTask` to make the coadd.
372 ConfigClass = AssembleCoaddConfig
373 _DefaultName =
"assembleCoadd"
375 def __init__(self, *args, **kwargs):
378 argNames = [
"config",
"name",
"parentTask",
"log"]
379 kwargs.update({k: v
for k, v
in zip(argNames, args)})
380 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
381 "PipelineTask will not take positional args" % argNames, FutureWarning)
383 super().__init__(**kwargs)
384 self.makeSubtask(
"interpImage")
385 self.makeSubtask(
"scaleZeroPoint")
387 if self.config.doMaskBrightObjects:
388 mask = afwImage.Mask()
390 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
391 except pexExceptions.LsstCppException:
392 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
393 mask.getMaskPlaneDict().keys())
396 self.warpType = self.config.warpType
398 @utils.inheritDoc(pipeBase.PipelineTask)
399 def runQuantum(self, butlerQC, inputRefs, outputRefs):
404 Assemble a coadd from a set of Warps.
406 PipelineTask (Gen3) entry point to Coadd a set of Warps.
407 Analogous to `runDataRef`, it prepares all the data products to be
408 passed to `run`, and processes the results before returning a struct
409 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
410 Therefore, its inputs are accessed subregion by subregion
411 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
412 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
413 correspond to an update in `runDataRef` while both entry points
416 inputData = butlerQC.get(inputRefs)
420 skyMap = inputData[
"skyMap"]
421 outputDataId = butlerQC.quantum.dataId
424 tractId=outputDataId[
'tract'],
425 patchId=outputDataId[
'patch'])
429 warpRefList = inputData[
'inputWarps']
431 inputs = self.prepareInputs(warpRefList)
432 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
433 self.getTempExpDatasetName(self.warpType))
434 if len(inputs.tempExpRefList) == 0:
435 self.log.warn(
"No coadd temporary exposures found")
438 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
439 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
440 inputs.weightList, supplementaryData=supplementaryData)
442 inputData.setdefault(
'brightObjectMask',
None)
443 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
445 if self.config.doWrite:
446 butlerQC.put(retStruct, outputRefs)
450 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
451 """Assemble a coadd from a set of Warps.
453 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
454 Compute weights to be applied to each Warp and
455 find scalings to match the photometric zeropoint to a reference Warp.
456 Assemble the Warps using `run`. Interpolate over NaNs and
457 optionally write the coadd to disk. Return the coadded exposure.
461 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
462 Data reference defining the patch for coaddition and the
463 reference Warp (if ``config.autoReference=False``).
464 Used to access the following data products:
465 - ``self.config.coaddName + "Coadd_skyMap"``
466 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
467 - ``self.config.coaddName + "Coadd"``
468 selectDataList : `list`
469 List of data references to Calexps. Data to be coadded will be
470 selected from this list based on overlap with the patch defined
471 by dataRef, grouped by visit, and converted to a list of data
474 List of data references to Warps to be coadded.
475 Note: `warpRefList` is just the new name for `tempExpRefList`.
479 retStruct : `lsst.pipe.base.Struct`
480 Result struct with components:
482 - ``coaddExposure``: coadded exposure (``Exposure``).
483 - ``nImage``: exposure count image (``Image``).
485 if selectDataList
and warpRefList:
486 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, "
487 "and which to use is ambiguous. Please pass only one.")
489 skyInfo = self.getSkyInfo(dataRef)
490 if warpRefList
is None:
491 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
492 if len(calExpRefList) == 0:
493 self.log.warn(
"No exposures to coadd")
495 self.log.info(
"Coadding %d exposures", len(calExpRefList))
497 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
499 inputData = self.prepareInputs(warpRefList)
500 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
501 self.getTempExpDatasetName(self.warpType))
502 if len(inputData.tempExpRefList) == 0:
503 self.log.warn(
"No coadd temporary exposures found")
506 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
508 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
509 inputData.weightList, supplementaryData=supplementaryData)
511 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None
512 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
514 if self.config.doWrite:
515 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
516 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
518 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
519 self.log.info(
"Persisting %s" % coaddDatasetName)
520 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
521 if self.config.doNImage
and retStruct.nImage
is not None:
522 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
527 """Interpolate over missing data and mask bright stars.
531 coaddExposure : `lsst.afw.image.Exposure`
532 The coadded exposure to process.
533 dataRef : `lsst.daf.persistence.ButlerDataRef`
534 Butler data reference for supplementary data.
536 if self.config.doInterp:
537 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
539 varArray = coaddExposure.variance.array
540 with numpy.errstate(invalid=
"ignore"):
541 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
543 if self.config.doMaskBrightObjects:
544 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
547 """Make additional inputs to run() specific to subclasses (Gen2)
549 Duplicates interface of `runDataRef` method
550 Available to be implemented by subclasses only if they need the
551 coadd dataRef for performing preliminary processing before
552 assembling the coadd.
556 dataRef : `lsst.daf.persistence.ButlerDataRef`
557 Butler data reference for supplementary data.
558 selectDataList : `list` (optional)
559 Optional List of data references to Calexps.
560 warpRefList : `list` (optional)
561 Optional List of data references to Warps.
563 return pipeBase.Struct()
566 """Make additional inputs to run() specific to subclasses (Gen3)
568 Duplicates interface of `runQuantum` method.
569 Available to be implemented by subclasses only if they need the
570 coadd dataRef for performing preliminary processing before
571 assembling the coadd.
575 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
576 Gen3 Butler object for fetching additional data products before
577 running the Task specialized for quantum being processed
578 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
579 Attributes are the names of the connections describing input dataset types.
580 Values are DatasetRefs that task consumes for corresponding dataset type.
581 DataIds are guaranteed to match data objects in ``inputData``.
582 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
583 Attributes are the names of the connections describing output dataset types.
584 Values are DatasetRefs that task is to produce
585 for corresponding dataset type.
587 return pipeBase.Struct()
590 """Generate list data references corresponding to warped exposures
591 that lie within the patch to be coadded.
596 Data reference for patch.
597 calExpRefList : `list`
598 List of data references for input calexps.
602 tempExpRefList : `list`
603 List of Warp/CoaddTempExp data references.
605 butler = patchRef.getButler()
606 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
607 self.getTempExpDatasetName(self.warpType))
608 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
609 g, groupData.keys)
for
610 g
in groupData.groups.keys()]
611 return tempExpRefList
614 """Prepare the input warps for coaddition by measuring the weight for
615 each warp and the scaling for the photometric zero point.
617 Each Warp has its own photometric zeropoint and background variance.
618 Before coadding these Warps together, compute a scale factor to
619 normalize the photometric zeropoint and compute the weight for each Warp.
624 List of data references to tempExp
628 result : `lsst.pipe.base.Struct`
629 Result struct with components:
631 - ``tempExprefList``: `list` of data references to tempExp.
632 - ``weightList``: `list` of weightings.
633 - ``imageScalerList``: `list` of image scalers.
635 statsCtrl = afwMath.StatisticsControl()
636 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
637 statsCtrl.setNumIter(self.config.clipIter)
638 statsCtrl.setAndMask(self.getBadPixelMask())
639 statsCtrl.setNanSafe(
True)
646 tempExpName = self.getTempExpDatasetName(self.warpType)
647 for tempExpRef
in refList:
650 if not isinstance(tempExpRef, DeferredDatasetHandle):
651 if not tempExpRef.datasetExists(tempExpName):
652 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
655 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
657 if numpy.isnan(tempExp.image.array).all():
659 maskedImage = tempExp.getMaskedImage()
660 imageScaler = self.scaleZeroPoint.computeImageScaler(
665 imageScaler.scaleMaskedImage(maskedImage)
666 except Exception
as e:
667 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
669 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
670 afwMath.MEANCLIP, statsCtrl)
671 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
672 weight = 1.0 / float(meanVar)
673 if not numpy.isfinite(weight):
674 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
676 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
681 tempExpRefList.append(tempExpRef)
682 weightList.append(weight)
683 imageScalerList.append(imageScaler)
685 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
686 imageScalerList=imageScalerList)
689 """Prepare the statistics for coadding images.
693 mask : `int`, optional
694 Bit mask value to exclude from coaddition.
698 stats : `lsst.pipe.base.Struct`
699 Statistics structure with the following fields:
701 - ``statsCtrl``: Statistics control object for coadd
702 (`lsst.afw.math.StatisticsControl`)
703 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
706 mask = self.getBadPixelMask()
707 statsCtrl = afwMath.StatisticsControl()
708 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
709 statsCtrl.setNumIter(self.config.clipIter)
710 statsCtrl.setAndMask(mask)
711 statsCtrl.setNanSafe(
True)
712 statsCtrl.setWeighted(
True)
713 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
714 for plane, threshold
in self.config.maskPropagationThresholds.items():
715 bit = afwImage.Mask.getMaskPlane(plane)
716 statsCtrl.setMaskPropagationThreshold(bit, threshold)
717 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
718 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
721 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
722 altMaskList=None, mask=None, supplementaryData=None):
723 """Assemble a coadd from input warps
725 Assemble the coadd using the provided list of coaddTempExps. Since
726 the full coadd covers a patch (a large area), the assembly is
727 performed over small areas on the image at a time in order to
728 conserve memory usage. Iterate over subregions within the outer
729 bbox of the patch using `assembleSubregion` to stack the corresponding
730 subregions from the coaddTempExps with the statistic specified.
731 Set the edge bits the coadd mask based on the weight map.
735 skyInfo : `lsst.pipe.base.Struct`
736 Struct with geometric information about the patch.
737 tempExpRefList : `list`
738 List of data references to Warps (previously called CoaddTempExps).
739 imageScalerList : `list`
740 List of image scalers.
743 altMaskList : `list`, optional
744 List of alternate masks to use rather than those stored with
746 mask : `int`, optional
747 Bit mask value to exclude from coaddition.
748 supplementaryData : lsst.pipe.base.Struct, optional
749 Struct with additional data products needed to assemble coadd.
750 Only used by subclasses that implement `makeSupplementaryData`
755 result : `lsst.pipe.base.Struct`
756 Result struct with components:
758 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
759 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
760 - ``warpRefList``: input list of refs to the warps (
761 ``lsst.daf.butler.DeferredDatasetHandle`` or
762 ``lsst.daf.persistence.ButlerDataRef``)
764 - ``imageScalerList``: input list of image scalers (unmodified)
765 - ``weightList``: input list of weights (unmodified)
767 tempExpName = self.getTempExpDatasetName(self.warpType)
768 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
769 stats = self.prepareStats(mask=mask)
771 if altMaskList
is None:
772 altMaskList = [
None]*len(tempExpRefList)
774 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
775 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
776 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
777 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
778 coaddMaskedImage = coaddExposure.getMaskedImage()
779 subregionSizeArr = self.config.subregionSize
780 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
782 if self.config.doNImage:
783 nImage = afwImage.ImageU(skyInfo.bbox)
786 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
788 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
789 weightList, altMaskList, stats.flags, stats.ctrl,
791 except Exception
as e:
792 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
794 self.setInexactPsf(coaddMaskedImage.getMask())
797 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
798 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
799 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
800 weightList=weightList)
803 """Set the metadata for the coadd.
805 This basic implementation sets the filter from the first input.
809 coaddExposure : `lsst.afw.image.Exposure`
810 The target exposure for the coadd.
811 tempExpRefList : `list`
812 List of data references to tempExp.
816 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
817 tempExpName = self.getTempExpDatasetName(self.warpType)
823 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
825 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
828 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
829 for tempExpRef
in tempExpRefList]
830 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
834 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
835 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
836 coaddInputs.ccds.reserve(numCcds)
837 coaddInputs.visits.reserve(len(tempExpList))
839 for tempExp, weight
in zip(tempExpList, weightList):
840 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
842 if self.config.doUsePsfMatchedPolygons:
843 self.shrinkValidPolygons(coaddInputs)
845 coaddInputs.visits.sort()
846 if self.warpType ==
"psfMatched":
851 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
852 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
853 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
855 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
856 self.config.coaddPsf.makeControl())
857 coaddExposure.setPsf(psf)
858 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
859 coaddExposure.getWcs())
860 coaddExposure.getInfo().setApCorrMap(apCorrMap)
861 if self.config.doAttachTransmissionCurve:
862 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
863 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
866 altMaskList, statsFlags, statsCtrl, nImage=None):
867 """Assemble the coadd for a sub-region.
869 For each coaddTempExp, check for (and swap in) an alternative mask
870 if one is passed. Remove mask planes listed in
871 `config.removeMaskPlanes`. Finally, stack the actual exposures using
872 `lsst.afw.math.statisticsStack` with the statistic specified by
873 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
874 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
875 an N-sigma clipped mean where N and iterations are specified by
876 statsCtrl. Assign the stacked subregion back to the coadd.
880 coaddExposure : `lsst.afw.image.Exposure`
881 The target exposure for the coadd.
882 bbox : `lsst.geom.Box`
884 tempExpRefList : `list`
885 List of data reference to tempExp.
886 imageScalerList : `list`
887 List of image scalers.
891 List of alternate masks to use rather than those stored with
892 tempExp, or None. Each element is dict with keys = mask plane
893 name to which to add the spans.
894 statsFlags : `lsst.afw.math.Property`
895 Property object for statistic for coadd.
896 statsCtrl : `lsst.afw.math.StatisticsControl`
897 Statistics control object for coadd.
898 nImage : `lsst.afw.image.ImageU`, optional
899 Keeps track of exposure count for each pixel.
901 self.log.debug(
"Computing coadd over %s", bbox)
902 tempExpName = self.getTempExpDatasetName(self.warpType)
903 coaddExposure.mask.addMaskPlane(
"REJECTED")
904 coaddExposure.mask.addMaskPlane(
"CLIPPED")
905 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
906 maskMap = self.setRejectedMaskMapping(statsCtrl)
907 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
909 if nImage
is not None:
910 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
911 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
913 if isinstance(tempExpRef, DeferredDatasetHandle):
915 exposure = tempExpRef.get(parameters={
'bbox': bbox})
918 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
920 maskedImage = exposure.getMaskedImage()
921 mask = maskedImage.getMask()
922 if altMask
is not None:
923 self.applyAltMaskPlanes(mask, altMask)
924 imageScaler.scaleMaskedImage(maskedImage)
928 if nImage
is not None:
929 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
930 if self.config.removeMaskPlanes:
931 self.removeMaskPlanes(maskedImage)
932 maskedImageList.append(maskedImage)
934 with self.timer(
"stack"):
935 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
938 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
939 if nImage
is not None:
940 nImage.assign(subNImage, bbox)
943 """Unset the mask of an image for mask planes specified in the config.
947 maskedImage : `lsst.afw.image.MaskedImage`
948 The masked image to be modified.
950 mask = maskedImage.getMask()
951 for maskPlane
in self.config.removeMaskPlanes:
953 mask &= ~mask.getPlaneBitMask(maskPlane)
954 except pexExceptions.InvalidParameterError:
955 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
959 def setRejectedMaskMapping(statsCtrl):
960 """Map certain mask planes of the warps to new planes for the coadd.
962 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
963 or CLIPPED, set it to REJECTED on the coadd.
964 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
965 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
969 statsCtrl : `lsst.afw.math.StatisticsControl`
970 Statistics control object for coadd
974 maskMap : `list` of `tuple` of `int`
975 A list of mappings of mask planes of the warped exposures to
976 mask planes of the coadd.
978 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
979 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
980 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
981 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
982 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
983 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
988 """Apply in place alt mask formatted as SpanSets to a mask.
992 mask : `lsst.afw.image.Mask`
994 altMaskSpans : `dict`
995 SpanSet lists to apply. Each element contains the new mask
996 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
997 and list of SpanSets to apply to the mask.
1001 mask : `lsst.afw.image.Mask`
1004 if self.config.doUsePsfMatchedPolygons:
1005 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1010 for spanSet
in altMaskSpans[
'NO_DATA']:
1011 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1013 for plane, spanSetList
in altMaskSpans.items():
1014 maskClipValue = mask.addMaskPlane(plane)
1015 for spanSet
in spanSetList:
1016 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1020 """Shrink coaddInputs' ccds' ValidPolygons in place.
1022 Either modify each ccd's validPolygon in place, or if CoaddInputs
1023 does not have a validPolygon, create one from its bbox.
1027 coaddInputs : `lsst.afw.image.coaddInputs`
1031 for ccd
in coaddInputs.ccds:
1032 polyOrig = ccd.getValidPolygon()
1033 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1034 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1036 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1038 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1039 ccd.setValidPolygon(validPolygon)
1042 """Retrieve the bright object masks.
1044 Returns None on failure.
1048 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1053 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1054 Bright object mask from the Butler object, or None if it cannot
1058 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1059 except Exception
as e:
1060 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1064 """Set the bright object masks.
1068 exposure : `lsst.afw.image.Exposure`
1069 Exposure under consideration.
1070 dataId : `lsst.daf.persistence.dataId`
1071 Data identifier dict for patch.
1072 brightObjectMasks : `lsst.afw.table`
1073 Table of bright objects to mask.
1076 if brightObjectMasks
is None:
1077 self.log.warn(
"Unable to apply bright object mask: none supplied")
1079 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1080 mask = exposure.getMaskedImage().getMask()
1081 wcs = exposure.getWcs()
1082 plateScale = wcs.getPixelScale().asArcseconds()
1084 for rec
in brightObjectMasks:
1085 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1086 if rec[
"type"] ==
"box":
1087 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1088 width = rec[
"width"].asArcseconds()/plateScale
1089 height = rec[
"height"].asArcseconds()/plateScale
1092 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1095 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1096 spans = afwGeom.SpanSet(bbox)
1097 elif rec[
"type"] ==
"circle":
1098 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1099 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1101 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
1103 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1106 """Set INEXACT_PSF mask plane.
1108 If any of the input images isn't represented in the coadd (due to
1109 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1114 mask : `lsst.afw.image.Mask`
1115 Coadded exposure's mask, modified in-place.
1117 mask.addMaskPlane(
"INEXACT_PSF")
1118 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1119 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1120 clipped = mask.getPlaneBitMask(
"CLIPPED")
1121 rejected = mask.getPlaneBitMask(
"REJECTED")
1122 array = mask.getArray()
1123 selected = array & (sensorEdge | clipped | rejected) > 0
1124 array[selected] |= inexactPsf
1127 def _makeArgumentParser(cls):
1128 """Create an argument parser.
1130 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1131 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_"
1132 + cls.ConfigClass().warpType +
"Warp",
1133 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1134 ContainerClass=AssembleCoaddDataIdContainer)
1135 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1136 ContainerClass=SelectDataIdContainer)
1140 def _subBBoxIter(bbox, subregionSize):
1141 """Iterate over subregions of a bbox.
1145 bbox : `lsst.geom.Box2I`
1146 Bounding box over which to iterate.
1147 subregionSize: `lsst.geom.Extent2I`
1152 subBBox : `lsst.geom.Box2I`
1153 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1154 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1155 the edges of ``bbox``, but it will never be empty.
1158 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1159 if subregionSize[0] < 1
or subregionSize[1] < 1:
1160 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1162 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1163 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1166 if subBBox.isEmpty():
1167 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1168 "colShift=%s, rowShift=%s" %
1169 (bbox, subregionSize, colShift, rowShift))
1174 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1178 """Make self.refList from self.idList.
1183 Results of parsing command-line (with ``butler`` and ``log`` elements).
1185 datasetType = namespace.config.coaddName +
"Coadd"
1186 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1188 for dataId
in self.idList:
1190 for key
in keysCoadd:
1191 if key
not in dataId:
1192 raise RuntimeError(
"--id must include " + key)
1194 dataRef = namespace.butler.dataRef(
1195 datasetType=datasetType,
1198 self.refList.append(dataRef)
1202 """Function to count the number of pixels with a specific mask in a
1205 Find the intersection of mask & footprint. Count all pixels in the mask
1206 that are in the intersection that have bitmask set but do not have
1207 ignoreMask set. Return the count.
1211 mask : `lsst.afw.image.Mask`
1212 Mask to define intersection region by.
1213 footprint : `lsst.afw.detection.Footprint`
1214 Footprint to define the intersection region by.
1216 Specific mask that we wish to count the number of occurances of.
1218 Pixels to not consider.
1223 Count of number of pixels in footprint with specified mask.
1225 bbox = footprint.getBBox()
1226 bbox.clip(mask.getBBox(afwImage.PARENT))
1227 fp = afwImage.Mask(bbox)
1228 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1229 footprint.spans.setMask(fp, bitmask)
1230 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1231 (subMask.getArray() & ignoreMask) == 0).sum()
1235 """Configuration parameters for the SafeClipAssembleCoaddTask.
1237 assembleMeanCoadd = pexConfig.ConfigurableField(
1238 target=AssembleCoaddTask,
1239 doc=
"Task to assemble an initial Coadd using the MEAN statistic.",
1241 assembleMeanClipCoadd = pexConfig.ConfigurableField(
1242 target=AssembleCoaddTask,
1243 doc=
"Task to assemble an initial Coadd using the MEANCLIP statistic.",
1245 clipDetection = pexConfig.ConfigurableField(
1246 target=SourceDetectionTask,
1247 doc=
"Detect sources on difference between unclipped and clipped coadd")
1248 minClipFootOverlap = pexConfig.Field(
1249 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1253 minClipFootOverlapSingle = pexConfig.Field(
1254 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1255 "clipped when only one visit overlaps",
1259 minClipFootOverlapDouble = pexConfig.Field(
1260 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1261 "clipped when two visits overlap",
1265 maxClipFootOverlapDouble = pexConfig.Field(
1266 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
1267 "considering two visits",
1271 minBigOverlap = pexConfig.Field(
1272 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1273 "when labeling clipped footprints",
1279 """Set default values for clipDetection.
1283 The numeric values for these configuration parameters were
1284 empirically determined, future work may further refine them.
1286 AssembleCoaddConfig.setDefaults(self)
1287 self.
clipDetectionclipDetection.doTempLocalBackground =
False
1288 self.
clipDetectionclipDetection.reEstimateBackground =
False
1289 self.
clipDetectionclipDetection.returnOriginalFootprints =
False
1295 self.
clipDetectionclipDetection.thresholdType =
"pixel_stdev"
1306 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1307 "Ignoring doSigmaClip.")
1310 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1311 "(%s chosen). Please set statistic to MEAN."
1313 AssembleCoaddTask.ConfigClass.validate(self)
1317 """Assemble a coadded image from a set of coadded temporary exposures,
1318 being careful to clip & flag areas with potential artifacts.
1320 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1321 we clip outliers). The problem with doing this is that when computing the
1322 coadd PSF at a given location, individual visit PSFs from visits with
1323 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1324 In this task, we correct for this behavior by creating a new
1325 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1326 coaddTempExps and the final coadd where
1328 i. difference imaging suggests that there is an outlier and
1329 ii. this outlier appears on only one or two images.
1331 Such regions will not contribute to the final coadd. Furthermore, any
1332 routine to determine the coadd PSF can now be cognizant of clipped regions.
1333 Note that the algorithm implemented by this task is preliminary and works
1334 correctly for HSC data. Parameter modifications and or considerable
1335 redesigning of the algorithm is likley required for other surveys.
1337 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1338 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1339 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1344 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1345 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1346 see `baseDebug` for more about ``debug.py`` files.
1347 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1348 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1349 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1350 for further information.
1354 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1355 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1356 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1358 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1359 and filter to be coadded (specified using
1360 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1361 along with a list of coaddTempExps to attempt to coadd (specified using
1362 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1363 Only the coaddTempExps that cover the specified tract and patch will be
1364 coadded. A list of the available optional arguments can be obtained by
1365 calling assembleCoadd.py with the --help command line argument:
1367 .. code-block:: none
1369 assembleCoadd.py --help
1371 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1372 context of multi-band processing, we will generate the HSC-I & -R band
1373 coadds from HSC engineering test data provided in the ci_hsc package.
1374 To begin, assuming that the lsst stack has been already set up, we must
1375 set up the obs_subaru and ci_hsc packages. This defines the environment
1376 variable $CI_HSC_DIR and points at the location of the package. The raw
1377 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1378 the coadds, we must first
1381 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1383 create a skymap that covers the area of the sky present in the raw exposures
1384 - ``makeCoaddTempExp``
1385 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1387 We can perform all of these steps by running
1389 .. code-block:: none
1391 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1393 This will produce warped coaddTempExps for each visit. To coadd the
1394 warped data, we call ``assembleCoadd.py`` as follows:
1396 .. code-block:: none
1398 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1399 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1400 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1401 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1402 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1403 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1404 --selectId visit=903988 ccd=24
1406 This will process the HSC-I band data. The results are written in
1407 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1409 You may also choose to run:
1411 .. code-block:: none
1413 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1414 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1415 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1416 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1417 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1418 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1419 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1420 --selectId visit=903346 ccd=12
1422 to generate the coadd for the HSC-R band if you are interested in following
1423 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1425 ConfigClass = SafeClipAssembleCoaddConfig
1426 _DefaultName =
"safeClipAssembleCoadd"
1429 AssembleCoaddTask.__init__(self, *args, **kwargs)
1430 schema = afwTable.SourceTable.makeMinimalSchema()
1431 self.makeSubtask(
"clipDetection", schema=schema)
1432 self.makeSubtask(
"assembleMeanClipCoadd")
1433 self.makeSubtask(
"assembleMeanCoadd")
1435 @utils.inheritDoc(AssembleCoaddTask)
1436 @pipeBase.timeMethod
1437 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1438 """Assemble the coadd for a region.
1440 Compute the difference of coadds created with and without outlier
1441 rejection to identify coadd pixels that have outlier values in some
1443 Detect clipped regions on the difference image and mark these regions
1444 on the one or two individual coaddTempExps where they occur if there
1445 is significant overlap between the clipped region and a source. This
1446 leaves us with a set of footprints from the difference image that have
1447 been identified as having occured on just one or two individual visits.
1448 However, these footprints were generated from a difference image. It
1449 is conceivable for a large diffuse source to have become broken up
1450 into multiple footprints acrosss the coadd difference in this process.
1451 Determine the clipped region from all overlapping footprints from the
1452 detected sources in each visit - these are big footprints.
1453 Combine the small and big clipped footprints and mark them on a new
1455 Generate the coadd using `AssembleCoaddTask.run` without outlier
1456 removal. Clipped footprints will no longer make it into the coadd
1457 because they are marked in the new bad mask plane.
1461 args and kwargs are passed but ignored in order to match the call
1462 signature expected by the parent task.
1464 exp = self.
buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1465 mask = exp.getMaskedImage().getMask()
1466 mask.addMaskPlane(
"CLIPPED")
1468 result = self.
detectClipdetectClip(exp, tempExpRefList)
1470 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1472 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1473 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1475 bigFootprints = self.
detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1476 result.detectionFootprints, maskClipValue, maskDetValue,
1479 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1480 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1482 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1483 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1484 maskClip |= maskClipBig
1487 badMaskPlanes = self.config.badMaskPlanes[:]
1488 badMaskPlanes.append(
"CLIPPED")
1489 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1490 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1491 result.clipSpans, mask=badPixelMask)
1494 """Return an exposure that contains the difference between unclipped
1497 Generate a difference image between clipped and unclipped coadds.
1498 Compute the difference image by subtracting an outlier-clipped coadd
1499 from an outlier-unclipped coadd. Return the difference image.
1503 skyInfo : `lsst.pipe.base.Struct`
1504 Patch geometry information, from getSkyInfo
1505 tempExpRefList : `list`
1506 List of data reference to tempExp
1507 imageScalerList : `list`
1508 List of image scalers
1514 exp : `lsst.afw.image.Exposure`
1515 Difference image of unclipped and clipped coadd wrapped in an Exposure
1517 coaddMean = self.assembleMeanCoadd.
run(skyInfo, tempExpRefList,
1518 imageScalerList, weightList).coaddExposure
1520 coaddClip = self.assembleMeanClipCoadd.
run(skyInfo, tempExpRefList,
1521 imageScalerList, weightList).coaddExposure
1523 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1524 coaddDiff -= coaddClip.getMaskedImage()
1525 exp = afwImage.ExposureF(coaddDiff)
1526 exp.setPsf(coaddMean.getPsf())
1530 """Detect clipped regions on an exposure and set the mask on the
1531 individual tempExp masks.
1533 Detect footprints in the difference image after smoothing the
1534 difference image with a Gaussian kernal. Identify footprints that
1535 overlap with one or two input ``coaddTempExps`` by comparing the
1536 computed overlap fraction to thresholds set in the config. A different
1537 threshold is applied depending on the number of overlapping visits
1538 (restricted to one or two). If the overlap exceeds the thresholds,
1539 the footprint is considered "CLIPPED" and is marked as such on the
1540 coaddTempExp. Return a struct with the clipped footprints, the indices
1541 of the ``coaddTempExps`` that end up overlapping with the clipped
1542 footprints, and a list of new masks for the ``coaddTempExps``.
1546 exp : `lsst.afw.image.Exposure`
1547 Exposure to run detection on.
1548 tempExpRefList : `list`
1549 List of data reference to tempExp.
1553 result : `lsst.pipe.base.Struct`
1554 Result struct with components:
1556 - ``clipFootprints``: list of clipped footprints.
1557 - ``clipIndices``: indices for each ``clippedFootprint`` in
1559 - ``clipSpans``: List of dictionaries containing spanSet lists
1560 to clip. Each element contains the new maskplane name
1561 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1562 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1563 compressed into footprints.
1565 mask = exp.getMaskedImage().getMask()
1566 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1567 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1569 fpSet.positive.merge(fpSet.negative)
1570 footprints = fpSet.positive
1571 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1572 ignoreMask = self.getBadPixelMask()
1576 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1579 visitDetectionFootprints = []
1581 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1582 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1583 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1586 for i, warpRef
in enumerate(tempExpRefList):
1587 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1588 immediate=
True).getMaskedImage().getMask()
1589 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1590 afwImage.PARENT,
True)
1591 maskVisitDet &= maskDetValue
1592 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1593 visitDetectionFootprints.append(visitFootprints)
1595 for j, footprint
in enumerate(footprints.getFootprints()):
1600 for j, footprint
in enumerate(footprints.getFootprints()):
1601 nPixel = footprint.getArea()
1604 for i
in range(len(tempExpRefList)):
1605 ignore = ignoreArr[i, j]
1606 overlapDet = overlapDetArr[i, j]
1607 totPixel = nPixel - ignore
1610 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1612 overlap.append(overlapDet/float(totPixel))
1615 overlap = numpy.array(overlap)
1616 if not len(overlap):
1623 if len(overlap) == 1:
1624 if overlap[0] > self.config.minClipFootOverlapSingle:
1629 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1630 if len(clipIndex) == 1:
1632 keepIndex = [clipIndex[0]]
1635 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1636 if len(clipIndex) == 2
and len(overlap) > 3:
1637 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1638 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1640 keepIndex = clipIndex
1645 for index
in keepIndex:
1646 globalIndex = indexList[index]
1647 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1649 clipIndices.append(numpy.array(indexList)[keepIndex])
1650 clipFootprints.append(footprint)
1652 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1653 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1655 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1656 maskClipValue, maskDetValue, coaddBBox):
1657 """Return individual warp footprints for large artifacts and append
1658 them to ``clipList`` in place.
1660 Identify big footprints composed of many sources in the coadd
1661 difference that may have originated in a large diffuse source in the
1662 coadd. We do this by indentifying all clipped footprints that overlap
1663 significantly with each source in all the coaddTempExps.
1668 List of alt mask SpanSets with clipping information. Modified.
1669 clipFootprints : `list`
1670 List of clipped footprints.
1671 clipIndices : `list`
1672 List of which entries in tempExpClipList each footprint belongs to.
1674 Mask value of clipped pixels.
1676 Mask value of detected pixels.
1677 coaddBBox : `lsst.geom.Box`
1678 BBox of the coadd and warps.
1682 bigFootprintsCoadd : `list`
1683 List of big footprints
1685 bigFootprintsCoadd = []
1686 ignoreMask = self.getBadPixelMask()
1687 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1688 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1689 for footprint
in visitFootprints.getFootprints():
1690 footprint.spans.setMask(maskVisitDet, maskDetValue)
1693 clippedFootprintsVisit = []
1694 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1695 if index
not in clipIndex:
1697 clippedFootprintsVisit.append(foot)
1698 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1699 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1701 bigFootprintsVisit = []
1702 for foot
in visitFootprints.getFootprints():
1703 if foot.getArea() < self.config.minBigOverlap:
1706 if nCount > self.config.minBigOverlap:
1707 bigFootprintsVisit.append(foot)
1708 bigFootprintsCoadd.append(foot)
1710 for footprint
in bigFootprintsVisit:
1711 clippedSpans[
"CLIPPED"].append(footprint.spans)
1713 return bigFootprintsCoadd
1717 psfMatchedWarps = pipeBase.connectionTypes.Input(
1718 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1719 "Only PSF-Matched Warps make sense for image subtraction. "
1720 "Therefore, they must be an additional declared input."),
1721 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1722 storageClass=
"ExposureF",
1723 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1727 templateCoadd = pipeBase.connectionTypes.Output(
1728 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1729 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1730 name=
"{fakesType}{outputCoaddName}CoaddPsfMatched",
1731 storageClass=
"ExposureF",
1732 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1737 if not config.assembleStaticSkyModel.doWrite:
1738 self.outputs.remove(
"templateCoadd")
1743 pipelineConnections=CompareWarpAssembleCoaddConnections):
1744 assembleStaticSkyModel = pexConfig.ConfigurableField(
1745 target=AssembleCoaddTask,
1746 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1747 " naive/first-iteration model of the static sky.",
1749 detect = pexConfig.ConfigurableField(
1750 target=SourceDetectionTask,
1751 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1753 detectTemplate = pexConfig.ConfigurableField(
1754 target=SourceDetectionTask,
1755 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1757 maskStreaks = pexConfig.ConfigurableField(
1758 target=MaskStreaksTask,
1759 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1760 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1763 streakMaskName = pexConfig.Field(
1766 doc=
"Name of mask bit used for streaks"
1768 maxNumEpochs = pexConfig.Field(
1769 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1770 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1771 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1772 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1773 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1774 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1775 "than transient and not masked.",
1779 maxFractionEpochsLow = pexConfig.RangeField(
1780 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1781 "Effective maxNumEpochs = "
1782 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1787 maxFractionEpochsHigh = pexConfig.RangeField(
1788 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1789 "Effective maxNumEpochs = "
1790 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1795 spatialThreshold = pexConfig.RangeField(
1796 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1797 "temporal criteria. If 0, clip all. If 1, clip none.",
1801 inclusiveMin=
True, inclusiveMax=
True
1803 doScaleWarpVariance = pexConfig.Field(
1804 doc=
"Rescale Warp variance plane using empirical noise?",
1808 scaleWarpVariance = pexConfig.ConfigurableField(
1809 target=ScaleVarianceTask,
1810 doc=
"Rescale variance on warps",
1812 doPreserveContainedBySource = pexConfig.Field(
1813 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1814 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1818 doPrefilterArtifacts = pexConfig.Field(
1819 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1820 "because they will be excluded anyway. This prevents them from contributing "
1821 "to the outlier epoch count image and potentially being labeled as persistant."
1822 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1826 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1827 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1829 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1831 prefilterArtifactsRatio = pexConfig.Field(
1832 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1836 doFilterMorphological = pexConfig.Field(
1837 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1844 AssembleCoaddConfig.setDefaults(self)
1850 if "EDGE" in self.badMaskPlanes:
1851 self.badMaskPlanes.remove(
'EDGE')
1852 self.removeMaskPlanes.append(
'EDGE')
1861 self.
detectdetect.doTempLocalBackground =
False
1862 self.
detectdetect.reEstimateBackground =
False
1863 self.
detectdetect.returnOriginalFootprints =
False
1864 self.
detectdetect.thresholdPolarity =
"both"
1865 self.
detectdetect.thresholdValue = 5
1866 self.
detectdetect.minPixels = 4
1867 self.
detectdetect.isotropicGrow =
True
1868 self.
detectdetect.thresholdType =
"pixel_stdev"
1869 self.
detectdetect.nSigmaToGrow = 0.4
1875 self.
detectTemplatedetectTemplate.returnOriginalFootprints =
False
1880 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
1881 "Please set assembleStaticSkyModel.doNImage=False")
1884 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1885 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1886 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1891 """Assemble a compareWarp coadded image from a set of warps
1892 by masking artifacts detected by comparing PSF-matched warps.
1894 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1895 we clip outliers). The problem with doing this is that when computing the
1896 coadd PSF at a given location, individual visit PSFs from visits with
1897 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1898 In this task, we correct for this behavior by creating a new badMaskPlane
1899 'CLIPPED' which marks pixels in the individual warps suspected to contain
1900 an artifact. We populate this plane on the input warps by comparing
1901 PSF-matched warps with a PSF-matched median coadd which serves as a
1902 model of the static sky. Any group of pixels that deviates from the
1903 PSF-matched template coadd by more than config.detect.threshold sigma,
1904 is an artifact candidate. The candidates are then filtered to remove
1905 variable sources and sources that are difficult to subtract such as
1906 bright stars. This filter is configured using the config parameters
1907 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1908 the maximum fraction of epochs that the deviation can appear in and still
1909 be considered an artifact. The spatialThreshold is the maximum fraction of
1910 pixels in the footprint of the deviation that appear in other epochs
1911 (where other epochs is defined by the temporalThreshold). If the deviant
1912 region meets this criteria of having a significant percentage of pixels
1913 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1914 set in the mask. These regions will not contribute to the final coadd.
1915 Furthermore, any routine to determine the coadd PSF can now be cognizant
1916 of clipped regions. Note that the algorithm implemented by this task is
1917 preliminary and works correctly for HSC data. Parameter modifications and
1918 or considerable redesigning of the algorithm is likley required for other
1921 ``CompareWarpAssembleCoaddTask`` sub-classes
1922 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1923 as a subtask to generate the TemplateCoadd (the model of the static sky).
1927 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1928 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1929 ``baseDebug`` for more about ``debug.py`` files.
1931 This task supports the following debug variables:
1934 If True then save the Epoch Count Image as a fits file in the `figPath`
1936 Path to save the debug fits images and figures
1938 For example, put something like:
1940 .. code-block:: python
1943 def DebugInfo(name):
1944 di = lsstDebug.getInfo(name)
1945 if name == "lsst.pipe.tasks.assembleCoadd":
1946 di.saveCountIm = True
1947 di.figPath = "/desired/path/to/debugging/output/images"
1949 lsstDebug.Info = DebugInfo
1951 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1952 ``--debug`` flag. Some subtasks may have their own debug variables;
1953 see individual Task documentation.
1957 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1958 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1959 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1960 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1961 and filter to be coadded (specified using
1962 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1963 along with a list of coaddTempExps to attempt to coadd (specified using
1964 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1965 Only the warps that cover the specified tract and patch will be coadded.
1966 A list of the available optional arguments can be obtained by calling
1967 ``assembleCoadd.py`` with the ``--help`` command line argument:
1969 .. code-block:: none
1971 assembleCoadd.py --help
1973 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1974 context of multi-band processing, we will generate the HSC-I & -R band
1975 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1976 To begin, assuming that the lsst stack has been already set up, we must
1977 set up the ``obs_subaru`` and ``ci_hsc`` packages.
1978 This defines the environment variable ``$CI_HSC_DIR`` and points at the
1979 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1980 directory. To begin assembling the coadds, we must first
1983 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1985 create a skymap that covers the area of the sky present in the raw exposures
1987 warp the individual calibrated exposures to the tangent plane of the coadd
1989 We can perform all of these steps by running
1991 .. code-block:: none
1993 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1995 This will produce warped ``coaddTempExps`` for each visit. To coadd the
1996 warped data, we call ``assembleCoadd.py`` as follows:
1998 .. code-block:: none
2000 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2001 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2002 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2003 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2004 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2005 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2006 --selectId visit=903988 ccd=24
2008 This will process the HSC-I band data. The results are written in
2009 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2011 ConfigClass = CompareWarpAssembleCoaddConfig
2012 _DefaultName =
"compareWarpAssembleCoadd"
2015 AssembleCoaddTask.__init__(self, *args, **kwargs)
2016 self.makeSubtask(
"assembleStaticSkyModel")
2017 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2018 self.makeSubtask(
"detect", schema=detectionSchema)
2019 if self.config.doPreserveContainedBySource:
2020 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2021 if self.config.doScaleWarpVariance:
2022 self.makeSubtask(
"scaleWarpVariance")
2023 if self.config.doFilterMorphological:
2024 self.makeSubtask(
"maskStreaks")
2026 @utils.inheritDoc(AssembleCoaddTask)
2029 Generate a templateCoadd to use as a naive model of static sky to
2030 subtract from PSF-Matched warps.
2034 result : `lsst.pipe.base.Struct`
2035 Result struct with components:
2037 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2038 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2041 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2042 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2046 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2047 if self.config.assembleStaticSkyModel.doWrite:
2048 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2051 del outputRefs.templateCoadd
2052 del staticSkyModelOutputRefs.templateCoadd
2055 if 'nImage' in staticSkyModelOutputRefs.keys():
2056 del staticSkyModelOutputRefs.nImage
2058 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2059 staticSkyModelOutputRefs)
2060 if templateCoadd
is None:
2061 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2063 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2064 nImage=templateCoadd.nImage,
2065 warpRefList=templateCoadd.warpRefList,
2066 imageScalerList=templateCoadd.imageScalerList,
2067 weightList=templateCoadd.weightList)
2069 @utils.inheritDoc(AssembleCoaddTask)
2072 Generate a templateCoadd to use as a naive model of static sky to
2073 subtract from PSF-Matched warps.
2077 result : `lsst.pipe.base.Struct`
2078 Result struct with components:
2080 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2081 - ``nImage``: N Image (``lsst.afw.image.Image``)
2083 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2084 if templateCoadd
is None:
2085 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2087 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2088 nImage=templateCoadd.nImage,
2089 warpRefList=templateCoadd.warpRefList,
2090 imageScalerList=templateCoadd.imageScalerList,
2091 weightList=templateCoadd.weightList)
2093 def _noTemplateMessage(self, warpType):
2094 warpName = (warpType[0].upper() + warpType[1:])
2095 message =
"""No %(warpName)s warps were found to build the template coadd which is
2096 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2097 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2098 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2100 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2101 another algorithm like:
2103 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2104 config.assemble.retarget(SafeClipAssembleCoaddTask)
2105 """ % {
"warpName": warpName}
2108 @utils.inheritDoc(AssembleCoaddTask)
2109 @pipeBase.timeMethod
2110 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2111 supplementaryData, *args, **kwargs):
2112 """Assemble the coadd.
2114 Find artifacts and apply them to the warps' masks creating a list of
2115 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2116 plane. Then pass these alternative masks to the base class's `run`
2119 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2120 that must contain a ``templateCoadd`` that serves as the
2121 model of the static sky.
2127 dataIds = [ref.dataId
for ref
in tempExpRefList]
2128 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
2130 if dataIds != psfMatchedDataIds:
2131 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2132 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2133 psfMatchedDataIds, dataIds)
2134 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2135 psfMatchedDataIds, dataIds)
2138 spanSetMaskList = self.
findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2139 supplementaryData.warpRefList,
2140 supplementaryData.imageScalerList)
2142 badMaskPlanes = self.config.badMaskPlanes[:]
2143 badMaskPlanes.append(
"CLIPPED")
2144 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2146 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2147 spanSetMaskList, mask=badPixelMask)
2151 self.
applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2155 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2159 mask : `lsst.afw.image.Mask`
2161 altMaskList : `list`
2162 List of Dicts containing ``spanSet`` lists.
2163 Each element contains the new mask plane name (e.g. "CLIPPED
2164 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2167 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2168 for visitMask
in altMaskList:
2169 if "EDGE" in visitMask:
2170 for spanSet
in visitMask[
'EDGE']:
2171 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2176 Loop through warps twice. The first loop builds a map with the count
2177 of how many epochs each pixel deviates from the templateCoadd by more
2178 than ``config.chiThreshold`` sigma. The second loop takes each
2179 difference image and filters the artifacts detected in each using
2180 count map to filter out variable sources and sources that are
2181 difficult to subtract cleanly.
2185 templateCoadd : `lsst.afw.image.Exposure`
2186 Exposure to serve as model of static sky.
2187 tempExpRefList : `list`
2188 List of data references to warps.
2189 imageScalerList : `list`
2190 List of image scalers.
2195 List of dicts containing information about CLIPPED
2196 (i.e., artifacts), NO_DATA, and EDGE pixels.
2199 self.log.debug(
"Generating Count Image, and mask lists.")
2200 coaddBBox = templateCoadd.getBBox()
2201 slateIm = afwImage.ImageU(coaddBBox)
2202 epochCountImage = afwImage.ImageU(coaddBBox)
2203 nImage = afwImage.ImageU(coaddBBox)
2204 spanSetArtifactList = []
2205 spanSetNoDataMaskList = []
2206 spanSetEdgeList = []
2207 spanSetBadMorphoList = []
2208 badPixelMask = self.getBadPixelMask()
2211 templateCoadd.mask.clearAllMaskPlanes()
2213 if self.config.doPreserveContainedBySource:
2214 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2216 templateFootprints =
None
2218 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2220 if warpDiffExp
is not None:
2222 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2223 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2224 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2225 fpSet.positive.merge(fpSet.negative)
2226 footprints = fpSet.positive
2228 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2231 if self.config.doPrefilterArtifacts:
2235 self.detect.clearMask(warpDiffExp.mask)
2236 for spans
in spanSetList:
2237 spans.setImage(slateIm, 1, doClip=
True)
2238 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
2239 epochCountImage += slateIm
2241 if self.config.doFilterMorphological:
2242 maskName = self.config.streakMaskName
2243 _ = self.maskStreaks.
run(warpDiffExp)
2244 streakMask = warpDiffExp.mask
2245 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2246 streakMask.getPlaneBitMask(maskName)).split()
2252 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2253 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2254 nansMask.setXY0(warpDiffExp.getXY0())
2255 edgeMask = warpDiffExp.mask
2256 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2257 edgeMask.getPlaneBitMask(
"EDGE")).split()
2261 nansMask = afwImage.MaskX(coaddBBox, 1)
2263 spanSetEdgeMask = []
2266 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2268 spanSetNoDataMaskList.append(spanSetNoDataMask)
2269 spanSetArtifactList.append(spanSetList)
2270 spanSetEdgeList.append(spanSetEdgeMask)
2271 if self.config.doFilterMorphological:
2272 spanSetBadMorphoList.append(spanSetStreak)
2275 path = self.
_dataRef2DebugPath_dataRef2DebugPath(
"epochCountIm", tempExpRefList[0], coaddLevel=
True)
2276 epochCountImage.writeFits(path)
2278 for i, spanSetList
in enumerate(spanSetArtifactList):
2280 filteredSpanSetList = self.
filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2282 spanSetArtifactList[i] = filteredSpanSetList
2283 if self.config.doFilterMorphological:
2284 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2287 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2288 altMasks.append({
'CLIPPED': artifacts,
2294 """Remove artifact candidates covered by bad mask plane.
2296 Any future editing of the candidate list that does not depend on
2297 temporal information should go in this method.
2301 spanSetList : `list`
2302 List of SpanSets representing artifact candidates.
2303 exp : `lsst.afw.image.Exposure`
2304 Exposure containing mask planes used to prefilter.
2308 returnSpanSetList : `list`
2309 List of SpanSets with artifacts.
2311 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2312 goodArr = (exp.mask.array & badPixelMask) == 0
2313 returnSpanSetList = []
2314 bbox = exp.getBBox()
2315 x0, y0 = exp.getXY0()
2316 for i, span
in enumerate(spanSetList):
2317 y, x = span.clippedTo(bbox).indices()
2318 yIndexLocal = numpy.array(y) - y0
2319 xIndexLocal = numpy.array(x) - x0
2320 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2321 if goodRatio > self.config.prefilterArtifactsRatio:
2322 returnSpanSetList.append(span)
2323 return returnSpanSetList
2325 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2326 """Filter artifact candidates.
2330 spanSetList : `list`
2331 List of SpanSets representing artifact candidates.
2332 epochCountImage : `lsst.afw.image.Image`
2333 Image of accumulated number of warpDiff detections.
2334 nImage : `lsst.afw.image.Image`
2335 Image of the accumulated number of total epochs contributing.
2339 maskSpanSetList : `list`
2340 List of SpanSets with artifacts.
2343 maskSpanSetList = []
2344 x0, y0 = epochCountImage.getXY0()
2345 for i, span
in enumerate(spanSetList):
2346 y, x = span.indices()
2347 yIdxLocal = [y1 - y0
for y1
in y]
2348 xIdxLocal = [x1 - x0
for x1
in x]
2349 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2350 totalN = nImage.array[yIdxLocal, xIdxLocal]
2353 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2354 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2355 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2356 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2357 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2358 & (outlierN <= effectiveMaxNumEpochs))
2359 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2360 if percentBelowThreshold > self.config.spatialThreshold:
2361 maskSpanSetList.append(span)
2363 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2365 filteredMaskSpanSetList = []
2366 for span
in maskSpanSetList:
2368 for footprint
in footprintsToExclude.positive.getFootprints():
2369 if footprint.spans.contains(span):
2373 filteredMaskSpanSetList.append(span)
2374 maskSpanSetList = filteredMaskSpanSetList
2376 return maskSpanSetList
2378 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2379 """Fetch a warp from the butler and return a warpDiff.
2383 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2384 Butler dataRef for the warp.
2385 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2386 An image scaler object.
2387 templateCoadd : `lsst.afw.image.Exposure`
2388 Exposure to be substracted from the scaled warp.
2392 warp : `lsst.afw.image.Exposure`
2393 Exposure of the image difference between the warp and template.
2401 warpName = self.getTempExpDatasetName(
'psfMatched')
2402 if not isinstance(warpRef, DeferredDatasetHandle):
2403 if not warpRef.datasetExists(warpName):
2404 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2406 warp = warpRef.get(datasetType=warpName, immediate=
True)
2408 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2409 mi = warp.getMaskedImage()
2410 if self.config.doScaleWarpVariance:
2412 self.scaleWarpVariance.
run(mi)
2413 except Exception
as exc:
2414 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2415 mi -= templateCoadd.getMaskedImage()
2418 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2419 """Return a path to which to write debugging output.
2421 Creates a hyphen-delimited string of dataId values for simple filenames.
2426 Prefix for filename.
2427 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2428 Butler dataRef to make the path from.
2429 coaddLevel : `bool`, optional.
2430 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2431 'filter', but no 'visit').
2436 Path for debugging output.
2439 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2441 keys = warpRef.dataId.keys()
2442 keyList = sorted(keys, reverse=
True)
2444 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2445 return os.path.join(directory, filename)
2449 """Match the order of one list to another, padding if necessary
2454 List to be reordered and padded. Elements can be any type.
2455 inputKeys : iterable
2456 Iterable of values to be compared with outputKeys.
2457 Length must match `inputList`
2458 outputKeys : iterable
2459 Iterable of values to be compared with inputKeys.
2461 Any value to be inserted where inputKey not in outputKeys
2466 Copy of inputList reordered per outputKeys and padded with `padWith`
2467 so that the length matches length of outputKeys.
2470 for d
in outputKeys:
2472 outputList.append(inputList[inputKeys.index(d)])
2474 outputList.append(padWith)
def makeDataRefList(self, namespace)
def __init__(self, *config=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def applyAltEdgeMask(self, mask, altMaskList)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def __init__(self, *args, **kwargs)
def detectClip(self, exp, tempExpRefList)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def setInexactPsf(self, mask)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def makeCoaddSuffix(warpType="direct")
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")