40 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41 from .interpImage
import InterpImageTask
42 from .scaleZeroPoint
import ScaleZeroPointTask
43 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
44 from .scaleVariance
import ScaleVarianceTask
45 from .maskStreaks
import MaskStreaksTask
47 from lsst.daf.butler
import DeferredDatasetHandle
49 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
50 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
51 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
55 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap"),
56 defaultTemplates={
"inputCoaddName":
"deep",
57 "outputCoaddName":
"deep",
61 inputWarps = pipeBase.connectionTypes.Input(
62 doc=(
"Input list of warps to be assemebled i.e. stacked."
63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
64 name=
"{inputCoaddName}Coadd_{warpType}Warp",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
70 skyMap = pipeBase.connectionTypes.Input(
71 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
72 name=
"{inputCoaddName}Coadd_skyMap",
73 storageClass=
"SkyMap",
74 dimensions=(
"skymap", ),
76 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
77 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
79 name=
"brightObjectMask",
80 storageClass=
"ObjectMaskCatalog",
81 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
83 coaddExposure = pipeBase.connectionTypes.Output(
84 doc=
"Output coadded exposure, produced by stacking input warps",
85 name=
"{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
86 storageClass=
"ExposureF",
87 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
89 nImage = pipeBase.connectionTypes.Output(
90 doc=
"Output image of number of input images per pixel",
91 name=
"{outputCoaddName}Coadd_nImage",
92 storageClass=
"ImageU",
93 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
96 def __init__(self, *, config=None):
98 config.connections.warpType = config.warpType
102 config.connections.fakesType =
"_fakes"
104 super().__init__(config=config)
106 if not config.doMaskBrightObjects:
107 self.prerequisiteInputs.remove(
"brightObjectMask")
109 if not config.doNImage:
110 self.outputs.remove(
"nImage")
113 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
114 pipelineConnections=AssembleCoaddConnections):
115 """Configuration parameters for the `AssembleCoaddTask`.
119 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
120 only set the bitplane config.brightObjectMaskName. To make this useful you
121 *must* also configure the flags.pixel algorithm, for example by adding
125 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
126 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
128 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
130 warpType = pexConfig.Field(
131 doc=
"Warp name: one of 'direct' or 'psfMatched'",
135 subregionSize = pexConfig.ListField(
137 doc=
"Width, height of stack subregion size; "
138 "make small enough that a full stack of images will fit into memory at once.",
140 default=(2000, 2000),
142 statistic = pexConfig.Field(
144 doc=
"Main stacking statistic for aggregating over the epochs.",
147 doSigmaClip = pexConfig.Field(
149 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
152 sigmaClip = pexConfig.Field(
154 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
157 clipIter = pexConfig.Field(
159 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
162 calcErrorFromInputVariance = pexConfig.Field(
164 doc=
"Calculate coadd variance from input variance by stacking statistic."
165 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
168 scaleZeroPoint = pexConfig.ConfigurableField(
169 target=ScaleZeroPointTask,
170 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
172 doInterp = pexConfig.Field(
173 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
177 interpImage = pexConfig.ConfigurableField(
178 target=InterpImageTask,
179 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
181 doWrite = pexConfig.Field(
182 doc=
"Persist coadd?",
186 doNImage = pexConfig.Field(
187 doc=
"Create image of number of contributing exposures for each pixel",
191 doUsePsfMatchedPolygons = pexConfig.Field(
192 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
196 maskPropagationThresholds = pexConfig.DictField(
199 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
200 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
201 "would have contributed exceeds this value."),
202 default={
"SAT": 0.1},
204 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
205 doc=
"Mask planes to remove before coadding")
206 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
207 doc=
"Set mask and flag bits for bright objects?")
208 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
209 doc=
"Name of mask bit used for bright objects")
210 coaddPsf = pexConfig.ConfigField(
211 doc=
"Configuration for CoaddPsf",
212 dtype=measAlg.CoaddPsfConfig,
214 doAttachTransmissionCurve = pexConfig.Field(
215 dtype=bool, default=
False, optional=
False,
216 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
217 "(requires all input Exposures to have TransmissionCurves).")
219 hasFakes = pexConfig.Field(
222 doc=
"Should be set to True if fake sources have been inserted into the input data."
225 def setDefaults(self):
226 super().setDefaults()
227 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
234 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
235 self.warpType =
'psfMatched'
236 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
237 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
238 self.statistic =
"MEANCLIP"
239 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
240 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
241 "compute and set a non-zero coadd variance estimate." % (self.statistic))
243 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
244 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
245 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
246 if str(k)
not in unstackableStats]
247 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
248 % (self.statistic, stackableStats))
251 class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
252 """Assemble a coadded image from a set of warps (coadded temporary exposures).
254 We want to assemble a coadded image from a set of Warps (also called
255 coadded temporary exposures or ``coaddTempExps``).
256 Each input Warp covers a patch on the sky and corresponds to a single
257 run/visit/exposure of the covered patch. We provide the task with a list
258 of Warps (``selectDataList``) from which it selects Warps that cover the
259 specified patch (pointed at by ``dataRef``).
260 Each Warp that goes into a coadd will typically have an independent
261 photometric zero-point. Therefore, we must scale each Warp to set it to
262 a common photometric zeropoint. WarpType may be one of 'direct' or
263 'psfMatched', and the boolean configs `config.makeDirect` and
264 `config.makePsfMatched` set which of the warp types will be coadded.
265 The coadd is computed as a mean with optional outlier rejection.
266 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
267 Finally, Warps can have bad 'NaN' pixels which received no input from the
268 source calExps. We interpolate over these bad (NaN) pixels.
270 `AssembleCoaddTask` uses several sub-tasks. These are
272 - `ScaleZeroPointTask`
273 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
275 - interpolate across bad pixels (NaN) in the final coadd
277 You can retarget these subtasks if you wish.
281 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
282 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
283 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
284 no debug variables of its own. Some of the subtasks may support debug
285 variables. See the documentation for the subtasks for further information.
289 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
290 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
291 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
292 inputs: a data reference to the tract patch and filter to be coadded, and
293 a list of Warps to attempt to coadd. These are specified using ``--id`` and
294 ``--selectId``, respectively:
298 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
299 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
301 Only the Warps that cover the specified tract and patch will be coadded.
302 A list of the available optional arguments can be obtained by calling
303 ``assembleCoadd.py`` with the ``--help`` command line argument:
307 assembleCoadd.py --help
309 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
310 multi-band processing, we will generate the HSC-I & -R band coadds from
311 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
312 assuming that the lsst stack has been already set up, we must set up the
313 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
314 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
315 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
316 coadds, we must first
319 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
321 - create a skymap that covers the area of the sky present in the raw exposures
323 - warp the individual calibrated exposures to the tangent plane of the coadd
325 We can perform all of these steps by running
329 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
331 This will produce warped exposures for each visit. To coadd the warped
332 data, we call assembleCoadd.py as follows:
336 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
337 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
338 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
339 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
340 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
341 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
342 --selectId visit=903988 ccd=24
344 that will process the HSC-I band data. The results are written in
345 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
347 You may also choose to run:
351 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
352 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
353 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
354 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
355 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
356 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
357 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
358 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
360 to generate the coadd for the HSC-R band if you are interested in
361 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
362 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
363 rather than `AssembleCoaddTask` to make the coadd.
365 ConfigClass = AssembleCoaddConfig
366 _DefaultName =
"assembleCoadd"
368 def __init__(self, *args, **kwargs):
371 argNames = [
"config",
"name",
"parentTask",
"log"]
372 kwargs.update({k: v
for k, v
in zip(argNames, args)})
373 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
374 "PipelineTask will not take positional args" % argNames, FutureWarning)
376 super().__init__(**kwargs)
377 self.makeSubtask(
"interpImage")
378 self.makeSubtask(
"scaleZeroPoint")
380 if self.config.doMaskBrightObjects:
381 mask = afwImage.Mask()
383 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
384 except pexExceptions.LsstCppException:
385 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
386 mask.getMaskPlaneDict().keys())
389 self.warpType = self.config.warpType
391 @utils.inheritDoc(pipeBase.PipelineTask)
392 def runQuantum(self, butlerQC, inputRefs, outputRefs):
397 Assemble a coadd from a set of Warps.
399 PipelineTask (Gen3) entry point to Coadd a set of Warps.
400 Analogous to `runDataRef`, it prepares all the data products to be
401 passed to `run`, and processes the results before returning a struct
402 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
403 Therefore, its inputs are accessed subregion by subregion
404 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
405 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
406 correspond to an update in `runDataRef` while both entry points
409 inputData = butlerQC.get(inputRefs)
413 skyMap = inputData[
"skyMap"]
414 outputDataId = butlerQC.quantum.dataId
417 tractId=outputDataId[
'tract'],
418 patchId=outputDataId[
'patch'])
422 warpRefList = inputData[
'inputWarps']
424 inputs = self.prepareInputs(warpRefList)
425 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
426 self.getTempExpDatasetName(self.warpType))
427 if len(inputs.tempExpRefList) == 0:
428 self.log.warn(
"No coadd temporary exposures found")
431 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
432 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
433 inputs.weightList, supplementaryData=supplementaryData)
435 inputData.setdefault(
'brightObjectMask',
None)
436 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
438 if self.config.doWrite:
439 butlerQC.put(retStruct, outputRefs)
443 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
444 """Assemble a coadd from a set of Warps.
446 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
447 Compute weights to be applied to each Warp and
448 find scalings to match the photometric zeropoint to a reference Warp.
449 Assemble the Warps using `run`. Interpolate over NaNs and
450 optionally write the coadd to disk. Return the coadded exposure.
454 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
455 Data reference defining the patch for coaddition and the
456 reference Warp (if ``config.autoReference=False``).
457 Used to access the following data products:
458 - ``self.config.coaddName + "Coadd_skyMap"``
459 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
460 - ``self.config.coaddName + "Coadd"``
461 selectDataList : `list`
462 List of data references to Calexps. Data to be coadded will be
463 selected from this list based on overlap with the patch defined
464 by dataRef, grouped by visit, and converted to a list of data
467 List of data references to Warps to be coadded.
468 Note: `warpRefList` is just the new name for `tempExpRefList`.
472 retStruct : `lsst.pipe.base.Struct`
473 Result struct with components:
475 - ``coaddExposure``: coadded exposure (``Exposure``).
476 - ``nImage``: exposure count image (``Image``).
478 if selectDataList
and warpRefList:
479 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, "
480 "and which to use is ambiguous. Please pass only one.")
482 skyInfo = self.getSkyInfo(dataRef)
483 if warpRefList
is None:
484 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
485 if len(calExpRefList) == 0:
486 self.log.warn(
"No exposures to coadd")
488 self.log.info(
"Coadding %d exposures", len(calExpRefList))
490 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
492 inputData = self.prepareInputs(warpRefList)
493 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
494 self.getTempExpDatasetName(self.warpType))
495 if len(inputData.tempExpRefList) == 0:
496 self.log.warn(
"No coadd temporary exposures found")
499 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
501 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
502 inputData.weightList, supplementaryData=supplementaryData)
504 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None
505 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
507 if self.config.doWrite:
508 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
509 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
511 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
512 self.log.info(
"Persisting %s" % coaddDatasetName)
513 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
514 if self.config.doNImage
and retStruct.nImage
is not None:
515 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
520 """Interpolate over missing data and mask bright stars.
524 coaddExposure : `lsst.afw.image.Exposure`
525 The coadded exposure to process.
526 dataRef : `lsst.daf.persistence.ButlerDataRef`
527 Butler data reference for supplementary data.
529 if self.config.doInterp:
530 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
532 varArray = coaddExposure.variance.array
533 with numpy.errstate(invalid=
"ignore"):
534 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
536 if self.config.doMaskBrightObjects:
537 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
540 """Make additional inputs to run() specific to subclasses (Gen2)
542 Duplicates interface of `runDataRef` method
543 Available to be implemented by subclasses only if they need the
544 coadd dataRef for performing preliminary processing before
545 assembling the coadd.
549 dataRef : `lsst.daf.persistence.ButlerDataRef`
550 Butler data reference for supplementary data.
551 selectDataList : `list` (optional)
552 Optional List of data references to Calexps.
553 warpRefList : `list` (optional)
554 Optional List of data references to Warps.
556 return pipeBase.Struct()
559 """Make additional inputs to run() specific to subclasses (Gen3)
561 Duplicates interface of `runQuantum` method.
562 Available to be implemented by subclasses only if they need the
563 coadd dataRef for performing preliminary processing before
564 assembling the coadd.
568 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
569 Gen3 Butler object for fetching additional data products before
570 running the Task specialized for quantum being processed
571 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
572 Attributes are the names of the connections describing input dataset types.
573 Values are DatasetRefs that task consumes for corresponding dataset type.
574 DataIds are guaranteed to match data objects in ``inputData``.
575 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
576 Attributes are the names of the connections describing output dataset types.
577 Values are DatasetRefs that task is to produce
578 for corresponding dataset type.
580 return pipeBase.Struct()
583 """Generate list data references corresponding to warped exposures
584 that lie within the patch to be coadded.
589 Data reference for patch.
590 calExpRefList : `list`
591 List of data references for input calexps.
595 tempExpRefList : `list`
596 List of Warp/CoaddTempExp data references.
598 butler = patchRef.getButler()
599 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
600 self.getTempExpDatasetName(self.warpType))
601 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
602 g, groupData.keys)
for
603 g
in groupData.groups.keys()]
604 return tempExpRefList
607 """Prepare the input warps for coaddition by measuring the weight for
608 each warp and the scaling for the photometric zero point.
610 Each Warp has its own photometric zeropoint and background variance.
611 Before coadding these Warps together, compute a scale factor to
612 normalize the photometric zeropoint and compute the weight for each Warp.
617 List of data references to tempExp
621 result : `lsst.pipe.base.Struct`
622 Result struct with components:
624 - ``tempExprefList``: `list` of data references to tempExp.
625 - ``weightList``: `list` of weightings.
626 - ``imageScalerList``: `list` of image scalers.
628 statsCtrl = afwMath.StatisticsControl()
629 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
630 statsCtrl.setNumIter(self.config.clipIter)
631 statsCtrl.setAndMask(self.getBadPixelMask())
632 statsCtrl.setNanSafe(
True)
639 tempExpName = self.getTempExpDatasetName(self.warpType)
640 for tempExpRef
in refList:
643 if not isinstance(tempExpRef, DeferredDatasetHandle):
644 if not tempExpRef.datasetExists(tempExpName):
645 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
648 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
650 if numpy.isnan(tempExp.image.array).all():
652 maskedImage = tempExp.getMaskedImage()
653 imageScaler = self.scaleZeroPoint.computeImageScaler(
658 imageScaler.scaleMaskedImage(maskedImage)
659 except Exception
as e:
660 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
662 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
663 afwMath.MEANCLIP, statsCtrl)
664 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
665 weight = 1.0 / float(meanVar)
666 if not numpy.isfinite(weight):
667 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
669 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
674 tempExpRefList.append(tempExpRef)
675 weightList.append(weight)
676 imageScalerList.append(imageScaler)
678 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
679 imageScalerList=imageScalerList)
682 """Prepare the statistics for coadding images.
686 mask : `int`, optional
687 Bit mask value to exclude from coaddition.
691 stats : `lsst.pipe.base.Struct`
692 Statistics structure with the following fields:
694 - ``statsCtrl``: Statistics control object for coadd
695 (`lsst.afw.math.StatisticsControl`)
696 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
699 mask = self.getBadPixelMask()
700 statsCtrl = afwMath.StatisticsControl()
701 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
702 statsCtrl.setNumIter(self.config.clipIter)
703 statsCtrl.setAndMask(mask)
704 statsCtrl.setNanSafe(
True)
705 statsCtrl.setWeighted(
True)
706 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
707 for plane, threshold
in self.config.maskPropagationThresholds.items():
708 bit = afwImage.Mask.getMaskPlane(plane)
709 statsCtrl.setMaskPropagationThreshold(bit, threshold)
710 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
711 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
714 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
715 altMaskList=None, mask=None, supplementaryData=None):
716 """Assemble a coadd from input warps
718 Assemble the coadd using the provided list of coaddTempExps. Since
719 the full coadd covers a patch (a large area), the assembly is
720 performed over small areas on the image at a time in order to
721 conserve memory usage. Iterate over subregions within the outer
722 bbox of the patch using `assembleSubregion` to stack the corresponding
723 subregions from the coaddTempExps with the statistic specified.
724 Set the edge bits the coadd mask based on the weight map.
728 skyInfo : `lsst.pipe.base.Struct`
729 Struct with geometric information about the patch.
730 tempExpRefList : `list`
731 List of data references to Warps (previously called CoaddTempExps).
732 imageScalerList : `list`
733 List of image scalers.
736 altMaskList : `list`, optional
737 List of alternate masks to use rather than those stored with
739 mask : `int`, optional
740 Bit mask value to exclude from coaddition.
741 supplementaryData : lsst.pipe.base.Struct, optional
742 Struct with additional data products needed to assemble coadd.
743 Only used by subclasses that implement `makeSupplementaryData`
748 result : `lsst.pipe.base.Struct`
749 Result struct with components:
751 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
752 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
753 - ``warpRefList``: input list of refs to the warps (
754 ``lsst.daf.butler.DeferredDatasetHandle`` or
755 ``lsst.daf.persistence.ButlerDataRef``)
757 - ``imageScalerList``: input list of image scalers (unmodified)
758 - ``weightList``: input list of weights (unmodified)
760 tempExpName = self.getTempExpDatasetName(self.warpType)
761 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
762 stats = self.prepareStats(mask=mask)
764 if altMaskList
is None:
765 altMaskList = [
None]*len(tempExpRefList)
767 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
768 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
769 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
770 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
771 coaddMaskedImage = coaddExposure.getMaskedImage()
772 subregionSizeArr = self.config.subregionSize
773 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
775 if self.config.doNImage:
776 nImage = afwImage.ImageU(skyInfo.bbox)
779 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
781 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
782 weightList, altMaskList, stats.flags, stats.ctrl,
784 except Exception
as e:
785 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
787 self.setInexactPsf(coaddMaskedImage.getMask())
790 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
791 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
792 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
793 weightList=weightList)
796 """Set the metadata for the coadd.
798 This basic implementation sets the filter from the first input.
802 coaddExposure : `lsst.afw.image.Exposure`
803 The target exposure for the coadd.
804 tempExpRefList : `list`
805 List of data references to tempExp.
809 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
810 tempExpName = self.getTempExpDatasetName(self.warpType)
816 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
818 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
821 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
822 for tempExpRef
in tempExpRefList]
823 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
825 coaddExposure.setFilter(tempExpList[0].getFilter())
826 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
827 coaddInputs.ccds.reserve(numCcds)
828 coaddInputs.visits.reserve(len(tempExpList))
830 for tempExp, weight
in zip(tempExpList, weightList):
831 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
833 if self.config.doUsePsfMatchedPolygons:
834 self.shrinkValidPolygons(coaddInputs)
836 coaddInputs.visits.sort()
837 if self.warpType ==
"psfMatched":
842 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
843 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
844 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
846 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
847 self.config.coaddPsf.makeControl())
848 coaddExposure.setPsf(psf)
849 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
850 coaddExposure.getWcs())
851 coaddExposure.getInfo().setApCorrMap(apCorrMap)
852 if self.config.doAttachTransmissionCurve:
853 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
854 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
857 altMaskList, statsFlags, statsCtrl, nImage=None):
858 """Assemble the coadd for a sub-region.
860 For each coaddTempExp, check for (and swap in) an alternative mask
861 if one is passed. Remove mask planes listed in
862 `config.removeMaskPlanes`. Finally, stack the actual exposures using
863 `lsst.afw.math.statisticsStack` with the statistic specified by
864 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
865 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
866 an N-sigma clipped mean where N and iterations are specified by
867 statsCtrl. Assign the stacked subregion back to the coadd.
871 coaddExposure : `lsst.afw.image.Exposure`
872 The target exposure for the coadd.
873 bbox : `lsst.geom.Box`
875 tempExpRefList : `list`
876 List of data reference to tempExp.
877 imageScalerList : `list`
878 List of image scalers.
882 List of alternate masks to use rather than those stored with
883 tempExp, or None. Each element is dict with keys = mask plane
884 name to which to add the spans.
885 statsFlags : `lsst.afw.math.Property`
886 Property object for statistic for coadd.
887 statsCtrl : `lsst.afw.math.StatisticsControl`
888 Statistics control object for coadd.
889 nImage : `lsst.afw.image.ImageU`, optional
890 Keeps track of exposure count for each pixel.
892 self.log.debug(
"Computing coadd over %s", bbox)
893 tempExpName = self.getTempExpDatasetName(self.warpType)
894 coaddExposure.mask.addMaskPlane(
"REJECTED")
895 coaddExposure.mask.addMaskPlane(
"CLIPPED")
896 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
897 maskMap = self.setRejectedMaskMapping(statsCtrl)
898 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
900 if nImage
is not None:
901 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
902 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
904 if isinstance(tempExpRef, DeferredDatasetHandle):
906 exposure = tempExpRef.get(parameters={
'bbox': bbox})
909 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
911 maskedImage = exposure.getMaskedImage()
912 mask = maskedImage.getMask()
913 if altMask
is not None:
914 self.applyAltMaskPlanes(mask, altMask)
915 imageScaler.scaleMaskedImage(maskedImage)
919 if nImage
is not None:
920 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
921 if self.config.removeMaskPlanes:
922 self.removeMaskPlanes(maskedImage)
923 maskedImageList.append(maskedImage)
925 with self.timer(
"stack"):
926 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
929 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
930 if nImage
is not None:
931 nImage.assign(subNImage, bbox)
934 """Unset the mask of an image for mask planes specified in the config.
938 maskedImage : `lsst.afw.image.MaskedImage`
939 The masked image to be modified.
941 mask = maskedImage.getMask()
942 for maskPlane
in self.config.removeMaskPlanes:
944 mask &= ~mask.getPlaneBitMask(maskPlane)
945 except pexExceptions.InvalidParameterError:
946 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
950 def setRejectedMaskMapping(statsCtrl):
951 """Map certain mask planes of the warps to new planes for the coadd.
953 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
954 or CLIPPED, set it to REJECTED on the coadd.
955 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
956 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
960 statsCtrl : `lsst.afw.math.StatisticsControl`
961 Statistics control object for coadd
965 maskMap : `list` of `tuple` of `int`
966 A list of mappings of mask planes of the warped exposures to
967 mask planes of the coadd.
969 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
970 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
971 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
972 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
973 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
974 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
979 """Apply in place alt mask formatted as SpanSets to a mask.
983 mask : `lsst.afw.image.Mask`
985 altMaskSpans : `dict`
986 SpanSet lists to apply. Each element contains the new mask
987 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
988 and list of SpanSets to apply to the mask.
992 mask : `lsst.afw.image.Mask`
995 if self.config.doUsePsfMatchedPolygons:
996 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1001 for spanSet
in altMaskSpans[
'NO_DATA']:
1002 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1004 for plane, spanSetList
in altMaskSpans.items():
1005 maskClipValue = mask.addMaskPlane(plane)
1006 for spanSet
in spanSetList:
1007 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1011 """Shrink coaddInputs' ccds' ValidPolygons in place.
1013 Either modify each ccd's validPolygon in place, or if CoaddInputs
1014 does not have a validPolygon, create one from its bbox.
1018 coaddInputs : `lsst.afw.image.coaddInputs`
1022 for ccd
in coaddInputs.ccds:
1023 polyOrig = ccd.getValidPolygon()
1024 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1025 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1027 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1029 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1030 ccd.setValidPolygon(validPolygon)
1033 """Retrieve the bright object masks.
1035 Returns None on failure.
1039 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1044 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1045 Bright object mask from the Butler object, or None if it cannot
1049 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1050 except Exception
as e:
1051 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1055 """Set the bright object masks.
1059 exposure : `lsst.afw.image.Exposure`
1060 Exposure under consideration.
1061 dataId : `lsst.daf.persistence.dataId`
1062 Data identifier dict for patch.
1063 brightObjectMasks : `lsst.afw.table`
1064 Table of bright objects to mask.
1067 if brightObjectMasks
is None:
1068 self.log.warn(
"Unable to apply bright object mask: none supplied")
1070 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1071 mask = exposure.getMaskedImage().getMask()
1072 wcs = exposure.getWcs()
1073 plateScale = wcs.getPixelScale().asArcseconds()
1075 for rec
in brightObjectMasks:
1076 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1077 if rec[
"type"] ==
"box":
1078 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1079 width = rec[
"width"].asArcseconds()/plateScale
1080 height = rec[
"height"].asArcseconds()/plateScale
1083 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1086 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1087 spans = afwGeom.SpanSet(bbox)
1088 elif rec[
"type"] ==
"circle":
1089 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1090 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1092 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
1094 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1097 """Set INEXACT_PSF mask plane.
1099 If any of the input images isn't represented in the coadd (due to
1100 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1105 mask : `lsst.afw.image.Mask`
1106 Coadded exposure's mask, modified in-place.
1108 mask.addMaskPlane(
"INEXACT_PSF")
1109 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1110 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1111 clipped = mask.getPlaneBitMask(
"CLIPPED")
1112 rejected = mask.getPlaneBitMask(
"REJECTED")
1113 array = mask.getArray()
1114 selected = array & (sensorEdge | clipped | rejected) > 0
1115 array[selected] |= inexactPsf
1118 def _makeArgumentParser(cls):
1119 """Create an argument parser.
1121 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1122 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_"
1123 + cls.ConfigClass().warpType +
"Warp",
1124 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1125 ContainerClass=AssembleCoaddDataIdContainer)
1126 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1127 ContainerClass=SelectDataIdContainer)
1131 def _subBBoxIter(bbox, subregionSize):
1132 """Iterate over subregions of a bbox.
1136 bbox : `lsst.geom.Box2I`
1137 Bounding box over which to iterate.
1138 subregionSize: `lsst.geom.Extent2I`
1143 subBBox : `lsst.geom.Box2I`
1144 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1145 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1146 the edges of ``bbox``, but it will never be empty.
1149 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1150 if subregionSize[0] < 1
or subregionSize[1] < 1:
1151 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1153 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1154 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1157 if subBBox.isEmpty():
1158 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1159 "colShift=%s, rowShift=%s" %
1160 (bbox, subregionSize, colShift, rowShift))
1165 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1169 """Make self.refList from self.idList.
1174 Results of parsing command-line (with ``butler`` and ``log`` elements).
1176 datasetType = namespace.config.coaddName +
"Coadd"
1177 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1179 for dataId
in self.idList:
1181 for key
in keysCoadd:
1182 if key
not in dataId:
1183 raise RuntimeError(
"--id must include " + key)
1185 dataRef = namespace.butler.dataRef(
1186 datasetType=datasetType,
1189 self.refList.append(dataRef)
1193 """Function to count the number of pixels with a specific mask in a
1196 Find the intersection of mask & footprint. Count all pixels in the mask
1197 that are in the intersection that have bitmask set but do not have
1198 ignoreMask set. Return the count.
1202 mask : `lsst.afw.image.Mask`
1203 Mask to define intersection region by.
1204 footprint : `lsst.afw.detection.Footprint`
1205 Footprint to define the intersection region by.
1207 Specific mask that we wish to count the number of occurances of.
1209 Pixels to not consider.
1214 Count of number of pixels in footprint with specified mask.
1216 bbox = footprint.getBBox()
1217 bbox.clip(mask.getBBox(afwImage.PARENT))
1218 fp = afwImage.Mask(bbox)
1219 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1220 footprint.spans.setMask(fp, bitmask)
1221 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1222 (subMask.getArray() & ignoreMask) == 0).sum()
1226 """Configuration parameters for the SafeClipAssembleCoaddTask.
1228 assembleMeanCoadd = pexConfig.ConfigurableField(
1229 target=AssembleCoaddTask,
1230 doc=
"Task to assemble an initial Coadd using the MEAN statistic.",
1232 assembleMeanClipCoadd = pexConfig.ConfigurableField(
1233 target=AssembleCoaddTask,
1234 doc=
"Task to assemble an initial Coadd using the MEANCLIP statistic.",
1236 clipDetection = pexConfig.ConfigurableField(
1237 target=SourceDetectionTask,
1238 doc=
"Detect sources on difference between unclipped and clipped coadd")
1239 minClipFootOverlap = pexConfig.Field(
1240 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1244 minClipFootOverlapSingle = pexConfig.Field(
1245 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1246 "clipped when only one visit overlaps",
1250 minClipFootOverlapDouble = pexConfig.Field(
1251 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1252 "clipped when two visits overlap",
1256 maxClipFootOverlapDouble = pexConfig.Field(
1257 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
1258 "considering two visits",
1262 minBigOverlap = pexConfig.Field(
1263 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1264 "when labeling clipped footprints",
1270 """Set default values for clipDetection.
1274 The numeric values for these configuration parameters were
1275 empirically determined, future work may further refine them.
1277 AssembleCoaddConfig.setDefaults(self)
1297 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1298 "Ignoring doSigmaClip.")
1301 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1302 "(%s chosen). Please set statistic to MEAN."
1304 AssembleCoaddTask.ConfigClass.validate(self)
1308 """Assemble a coadded image from a set of coadded temporary exposures,
1309 being careful to clip & flag areas with potential artifacts.
1311 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1312 we clip outliers). The problem with doing this is that when computing the
1313 coadd PSF at a given location, individual visit PSFs from visits with
1314 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1315 In this task, we correct for this behavior by creating a new
1316 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1317 coaddTempExps and the final coadd where
1319 i. difference imaging suggests that there is an outlier and
1320 ii. this outlier appears on only one or two images.
1322 Such regions will not contribute to the final coadd. Furthermore, any
1323 routine to determine the coadd PSF can now be cognizant of clipped regions.
1324 Note that the algorithm implemented by this task is preliminary and works
1325 correctly for HSC data. Parameter modifications and or considerable
1326 redesigning of the algorithm is likley required for other surveys.
1328 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1329 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1330 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1335 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1336 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1337 see `baseDebug` for more about ``debug.py`` files.
1338 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1339 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1340 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1341 for further information.
1345 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1346 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1347 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1349 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1350 and filter to be coadded (specified using
1351 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1352 along with a list of coaddTempExps to attempt to coadd (specified using
1353 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1354 Only the coaddTempExps that cover the specified tract and patch will be
1355 coadded. A list of the available optional arguments can be obtained by
1356 calling assembleCoadd.py with the --help command line argument:
1358 .. code-block:: none
1360 assembleCoadd.py --help
1362 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1363 context of multi-band processing, we will generate the HSC-I & -R band
1364 coadds from HSC engineering test data provided in the ci_hsc package.
1365 To begin, assuming that the lsst stack has been already set up, we must
1366 set up the obs_subaru and ci_hsc packages. This defines the environment
1367 variable $CI_HSC_DIR and points at the location of the package. The raw
1368 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1369 the coadds, we must first
1372 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1374 create a skymap that covers the area of the sky present in the raw exposures
1375 - ``makeCoaddTempExp``
1376 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1378 We can perform all of these steps by running
1380 .. code-block:: none
1382 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1384 This will produce warped coaddTempExps for each visit. To coadd the
1385 warped data, we call ``assembleCoadd.py`` as follows:
1387 .. code-block:: none
1389 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1390 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1391 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1392 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1393 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1394 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1395 --selectId visit=903988 ccd=24
1397 This will process the HSC-I band data. The results are written in
1398 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1400 You may also choose to run:
1402 .. code-block:: none
1404 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1405 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1406 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1407 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1408 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1409 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1410 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1411 --selectId visit=903346 ccd=12
1413 to generate the coadd for the HSC-R band if you are interested in following
1414 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1416 ConfigClass = SafeClipAssembleCoaddConfig
1417 _DefaultName =
"safeClipAssembleCoadd"
1420 AssembleCoaddTask.__init__(self, *args, **kwargs)
1421 schema = afwTable.SourceTable.makeMinimalSchema()
1422 self.makeSubtask(
"clipDetection", schema=schema)
1423 self.makeSubtask(
"assembleMeanClipCoadd")
1424 self.makeSubtask(
"assembleMeanCoadd")
1426 @utils.inheritDoc(AssembleCoaddTask)
1427 @pipeBase.timeMethod
1428 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1429 """Assemble the coadd for a region.
1431 Compute the difference of coadds created with and without outlier
1432 rejection to identify coadd pixels that have outlier values in some
1434 Detect clipped regions on the difference image and mark these regions
1435 on the one or two individual coaddTempExps where they occur if there
1436 is significant overlap between the clipped region and a source. This
1437 leaves us with a set of footprints from the difference image that have
1438 been identified as having occured on just one or two individual visits.
1439 However, these footprints were generated from a difference image. It
1440 is conceivable for a large diffuse source to have become broken up
1441 into multiple footprints acrosss the coadd difference in this process.
1442 Determine the clipped region from all overlapping footprints from the
1443 detected sources in each visit - these are big footprints.
1444 Combine the small and big clipped footprints and mark them on a new
1446 Generate the coadd using `AssembleCoaddTask.run` without outlier
1447 removal. Clipped footprints will no longer make it into the coadd
1448 because they are marked in the new bad mask plane.
1452 args and kwargs are passed but ignored in order to match the call
1453 signature expected by the parent task.
1456 mask = exp.getMaskedImage().getMask()
1457 mask.addMaskPlane(
"CLIPPED")
1459 result = self.
detectClip(exp, tempExpRefList)
1461 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1463 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1464 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1466 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1467 result.detectionFootprints, maskClipValue, maskDetValue,
1470 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1471 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1473 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1474 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1475 maskClip |= maskClipBig
1478 badMaskPlanes = self.config.badMaskPlanes[:]
1479 badMaskPlanes.append(
"CLIPPED")
1480 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1481 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1482 result.clipSpans, mask=badPixelMask)
1485 """Return an exposure that contains the difference between unclipped
1488 Generate a difference image between clipped and unclipped coadds.
1489 Compute the difference image by subtracting an outlier-clipped coadd
1490 from an outlier-unclipped coadd. Return the difference image.
1494 skyInfo : `lsst.pipe.base.Struct`
1495 Patch geometry information, from getSkyInfo
1496 tempExpRefList : `list`
1497 List of data reference to tempExp
1498 imageScalerList : `list`
1499 List of image scalers
1505 exp : `lsst.afw.image.Exposure`
1506 Difference image of unclipped and clipped coadd wrapped in an Exposure
1508 coaddMean = self.assembleMeanCoadd.
run(skyInfo, tempExpRefList,
1509 imageScalerList, weightList).coaddExposure
1511 coaddClip = self.assembleMeanClipCoadd.
run(skyInfo, tempExpRefList,
1512 imageScalerList, weightList).coaddExposure
1514 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1515 coaddDiff -= coaddClip.getMaskedImage()
1516 exp = afwImage.ExposureF(coaddDiff)
1517 exp.setPsf(coaddMean.getPsf())
1521 """Detect clipped regions on an exposure and set the mask on the
1522 individual tempExp masks.
1524 Detect footprints in the difference image after smoothing the
1525 difference image with a Gaussian kernal. Identify footprints that
1526 overlap with one or two input ``coaddTempExps`` by comparing the
1527 computed overlap fraction to thresholds set in the config. A different
1528 threshold is applied depending on the number of overlapping visits
1529 (restricted to one or two). If the overlap exceeds the thresholds,
1530 the footprint is considered "CLIPPED" and is marked as such on the
1531 coaddTempExp. Return a struct with the clipped footprints, the indices
1532 of the ``coaddTempExps`` that end up overlapping with the clipped
1533 footprints, and a list of new masks for the ``coaddTempExps``.
1537 exp : `lsst.afw.image.Exposure`
1538 Exposure to run detection on.
1539 tempExpRefList : `list`
1540 List of data reference to tempExp.
1544 result : `lsst.pipe.base.Struct`
1545 Result struct with components:
1547 - ``clipFootprints``: list of clipped footprints.
1548 - ``clipIndices``: indices for each ``clippedFootprint`` in
1550 - ``clipSpans``: List of dictionaries containing spanSet lists
1551 to clip. Each element contains the new maskplane name
1552 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1553 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1554 compressed into footprints.
1556 mask = exp.getMaskedImage().getMask()
1557 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1558 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1560 fpSet.positive.merge(fpSet.negative)
1561 footprints = fpSet.positive
1562 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1563 ignoreMask = self.getBadPixelMask()
1567 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1570 visitDetectionFootprints = []
1572 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1573 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1574 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1577 for i, warpRef
in enumerate(tempExpRefList):
1578 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1579 immediate=
True).getMaskedImage().getMask()
1580 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1581 afwImage.PARENT,
True)
1582 maskVisitDet &= maskDetValue
1583 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1584 visitDetectionFootprints.append(visitFootprints)
1586 for j, footprint
in enumerate(footprints.getFootprints()):
1591 for j, footprint
in enumerate(footprints.getFootprints()):
1592 nPixel = footprint.getArea()
1595 for i
in range(len(tempExpRefList)):
1596 ignore = ignoreArr[i, j]
1597 overlapDet = overlapDetArr[i, j]
1598 totPixel = nPixel - ignore
1601 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1603 overlap.append(overlapDet/float(totPixel))
1606 overlap = numpy.array(overlap)
1607 if not len(overlap):
1614 if len(overlap) == 1:
1615 if overlap[0] > self.config.minClipFootOverlapSingle:
1620 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1621 if len(clipIndex) == 1:
1623 keepIndex = [clipIndex[0]]
1626 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1627 if len(clipIndex) == 2
and len(overlap) > 3:
1628 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1629 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1631 keepIndex = clipIndex
1636 for index
in keepIndex:
1637 globalIndex = indexList[index]
1638 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1640 clipIndices.append(numpy.array(indexList)[keepIndex])
1641 clipFootprints.append(footprint)
1643 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1644 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1646 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1647 maskClipValue, maskDetValue, coaddBBox):
1648 """Return individual warp footprints for large artifacts and append
1649 them to ``clipList`` in place.
1651 Identify big footprints composed of many sources in the coadd
1652 difference that may have originated in a large diffuse source in the
1653 coadd. We do this by indentifying all clipped footprints that overlap
1654 significantly with each source in all the coaddTempExps.
1659 List of alt mask SpanSets with clipping information. Modified.
1660 clipFootprints : `list`
1661 List of clipped footprints.
1662 clipIndices : `list`
1663 List of which entries in tempExpClipList each footprint belongs to.
1665 Mask value of clipped pixels.
1667 Mask value of detected pixels.
1668 coaddBBox : `lsst.geom.Box`
1669 BBox of the coadd and warps.
1673 bigFootprintsCoadd : `list`
1674 List of big footprints
1676 bigFootprintsCoadd = []
1677 ignoreMask = self.getBadPixelMask()
1678 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1679 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1680 for footprint
in visitFootprints.getFootprints():
1681 footprint.spans.setMask(maskVisitDet, maskDetValue)
1684 clippedFootprintsVisit = []
1685 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1686 if index
not in clipIndex:
1688 clippedFootprintsVisit.append(foot)
1689 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1690 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1692 bigFootprintsVisit = []
1693 for foot
in visitFootprints.getFootprints():
1694 if foot.getArea() < self.config.minBigOverlap:
1697 if nCount > self.config.minBigOverlap:
1698 bigFootprintsVisit.append(foot)
1699 bigFootprintsCoadd.append(foot)
1701 for footprint
in bigFootprintsVisit:
1702 clippedSpans[
"CLIPPED"].append(footprint.spans)
1704 return bigFootprintsCoadd
1708 psfMatchedWarps = pipeBase.connectionTypes.Input(
1709 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1710 "Only PSF-Matched Warps make sense for image subtraction. "
1711 "Therefore, they must be an additional declared input."),
1712 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1713 storageClass=
"ExposureF",
1714 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1718 templateCoadd = pipeBase.connectionTypes.Output(
1719 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1720 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1721 name=
"{fakesType}{outputCoaddName}CoaddPsfMatched",
1722 storageClass=
"ExposureF",
1723 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
1728 if not config.assembleStaticSkyModel.doWrite:
1729 self.outputs.remove(
"templateCoadd")
1734 pipelineConnections=CompareWarpAssembleCoaddConnections):
1735 assembleStaticSkyModel = pexConfig.ConfigurableField(
1736 target=AssembleCoaddTask,
1737 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1738 " naive/first-iteration model of the static sky.",
1740 detect = pexConfig.ConfigurableField(
1741 target=SourceDetectionTask,
1742 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1744 detectTemplate = pexConfig.ConfigurableField(
1745 target=SourceDetectionTask,
1746 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1748 maskStreaks = pexConfig.ConfigurableField(
1749 target=MaskStreaksTask,
1750 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1751 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1754 streakMaskName = pexConfig.Field(
1757 doc=
"Name of mask bit used for streaks"
1759 maxNumEpochs = pexConfig.Field(
1760 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1761 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1762 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1763 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1764 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1765 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1766 "than transient and not masked.",
1770 maxFractionEpochsLow = pexConfig.RangeField(
1771 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1772 "Effective maxNumEpochs = "
1773 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1778 maxFractionEpochsHigh = pexConfig.RangeField(
1779 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1780 "Effective maxNumEpochs = "
1781 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1786 spatialThreshold = pexConfig.RangeField(
1787 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1788 "temporal criteria. If 0, clip all. If 1, clip none.",
1792 inclusiveMin=
True, inclusiveMax=
True
1794 doScaleWarpVariance = pexConfig.Field(
1795 doc=
"Rescale Warp variance plane using empirical noise?",
1799 scaleWarpVariance = pexConfig.ConfigurableField(
1800 target=ScaleVarianceTask,
1801 doc=
"Rescale variance on warps",
1803 doPreserveContainedBySource = pexConfig.Field(
1804 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1805 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1809 doPrefilterArtifacts = pexConfig.Field(
1810 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1811 "because they will be excluded anyway. This prevents them from contributing "
1812 "to the outlier epoch count image and potentially being labeled as persistant."
1813 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1817 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1818 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1820 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1822 prefilterArtifactsRatio = pexConfig.Field(
1823 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1827 doFilterMorphological = pexConfig.Field(
1828 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1835 AssembleCoaddConfig.setDefaults(self)
1841 if "EDGE" in self.badMaskPlanes:
1842 self.badMaskPlanes.remove(
'EDGE')
1843 self.removeMaskPlanes.append(
'EDGE')
1852 self.
detect.doTempLocalBackground =
False
1853 self.
detect.reEstimateBackground =
False
1854 self.
detect.returnOriginalFootprints =
False
1855 self.
detect.thresholdPolarity =
"both"
1856 self.
detect.thresholdValue = 5
1857 self.
detect.minPixels = 4
1858 self.
detect.isotropicGrow =
True
1859 self.
detect.thresholdType =
"pixel_stdev"
1860 self.
detect.nSigmaToGrow = 0.4
1871 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
1872 "Please set assembleStaticSkyModel.doNImage=False")
1875 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1876 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1877 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1882 """Assemble a compareWarp coadded image from a set of warps
1883 by masking artifacts detected by comparing PSF-matched warps.
1885 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1886 we clip outliers). The problem with doing this is that when computing the
1887 coadd PSF at a given location, individual visit PSFs from visits with
1888 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1889 In this task, we correct for this behavior by creating a new badMaskPlane
1890 'CLIPPED' which marks pixels in the individual warps suspected to contain
1891 an artifact. We populate this plane on the input warps by comparing
1892 PSF-matched warps with a PSF-matched median coadd which serves as a
1893 model of the static sky. Any group of pixels that deviates from the
1894 PSF-matched template coadd by more than config.detect.threshold sigma,
1895 is an artifact candidate. The candidates are then filtered to remove
1896 variable sources and sources that are difficult to subtract such as
1897 bright stars. This filter is configured using the config parameters
1898 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1899 the maximum fraction of epochs that the deviation can appear in and still
1900 be considered an artifact. The spatialThreshold is the maximum fraction of
1901 pixels in the footprint of the deviation that appear in other epochs
1902 (where other epochs is defined by the temporalThreshold). If the deviant
1903 region meets this criteria of having a significant percentage of pixels
1904 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1905 set in the mask. These regions will not contribute to the final coadd.
1906 Furthermore, any routine to determine the coadd PSF can now be cognizant
1907 of clipped regions. Note that the algorithm implemented by this task is
1908 preliminary and works correctly for HSC data. Parameter modifications and
1909 or considerable redesigning of the algorithm is likley required for other
1912 ``CompareWarpAssembleCoaddTask`` sub-classes
1913 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1914 as a subtask to generate the TemplateCoadd (the model of the static sky).
1918 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1919 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1920 ``baseDebug`` for more about ``debug.py`` files.
1922 This task supports the following debug variables:
1925 If True then save the Epoch Count Image as a fits file in the `figPath`
1927 Path to save the debug fits images and figures
1929 For example, put something like:
1931 .. code-block:: python
1934 def DebugInfo(name):
1935 di = lsstDebug.getInfo(name)
1936 if name == "lsst.pipe.tasks.assembleCoadd":
1937 di.saveCountIm = True
1938 di.figPath = "/desired/path/to/debugging/output/images"
1940 lsstDebug.Info = DebugInfo
1942 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1943 ``--debug`` flag. Some subtasks may have their own debug variables;
1944 see individual Task documentation.
1948 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1949 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1950 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1951 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1952 and filter to be coadded (specified using
1953 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1954 along with a list of coaddTempExps to attempt to coadd (specified using
1955 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1956 Only the warps that cover the specified tract and patch will be coadded.
1957 A list of the available optional arguments can be obtained by calling
1958 ``assembleCoadd.py`` with the ``--help`` command line argument:
1960 .. code-block:: none
1962 assembleCoadd.py --help
1964 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1965 context of multi-band processing, we will generate the HSC-I & -R band
1966 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1967 To begin, assuming that the lsst stack has been already set up, we must
1968 set up the ``obs_subaru`` and ``ci_hsc`` packages.
1969 This defines the environment variable ``$CI_HSC_DIR`` and points at the
1970 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1971 directory. To begin assembling the coadds, we must first
1974 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1976 create a skymap that covers the area of the sky present in the raw exposures
1978 warp the individual calibrated exposures to the tangent plane of the coadd
1980 We can perform all of these steps by running
1982 .. code-block:: none
1984 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1986 This will produce warped ``coaddTempExps`` for each visit. To coadd the
1987 warped data, we call ``assembleCoadd.py`` as follows:
1989 .. code-block:: none
1991 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1992 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1993 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1994 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1995 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1996 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1997 --selectId visit=903988 ccd=24
1999 This will process the HSC-I band data. The results are written in
2000 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2002 ConfigClass = CompareWarpAssembleCoaddConfig
2003 _DefaultName =
"compareWarpAssembleCoadd"
2006 AssembleCoaddTask.__init__(self, *args, **kwargs)
2007 self.makeSubtask(
"assembleStaticSkyModel")
2008 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2009 self.makeSubtask(
"detect", schema=detectionSchema)
2010 if self.config.doPreserveContainedBySource:
2011 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2012 if self.config.doScaleWarpVariance:
2013 self.makeSubtask(
"scaleWarpVariance")
2014 if self.config.doFilterMorphological:
2015 self.makeSubtask(
"maskStreaks")
2017 @utils.inheritDoc(AssembleCoaddTask)
2020 Generate a templateCoadd to use as a naive model of static sky to
2021 subtract from PSF-Matched warps.
2025 result : `lsst.pipe.base.Struct`
2026 Result struct with components:
2028 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2029 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2032 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2033 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2037 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2038 if self.config.assembleStaticSkyModel.doWrite:
2039 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2042 del outputRefs.templateCoadd
2043 del staticSkyModelOutputRefs.templateCoadd
2046 if 'nImage' in staticSkyModelOutputRefs.keys():
2047 del staticSkyModelOutputRefs.nImage
2049 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2050 staticSkyModelOutputRefs)
2051 if templateCoadd
is None:
2054 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2055 nImage=templateCoadd.nImage,
2056 warpRefList=templateCoadd.warpRefList,
2057 imageScalerList=templateCoadd.imageScalerList,
2058 weightList=templateCoadd.weightList)
2060 @utils.inheritDoc(AssembleCoaddTask)
2063 Generate a templateCoadd to use as a naive model of static sky to
2064 subtract from PSF-Matched warps.
2068 result : `lsst.pipe.base.Struct`
2069 Result struct with components:
2071 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2072 - ``nImage``: N Image (``lsst.afw.image.Image``)
2074 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2075 if templateCoadd
is None:
2078 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2079 nImage=templateCoadd.nImage,
2080 warpRefList=templateCoadd.warpRefList,
2081 imageScalerList=templateCoadd.imageScalerList,
2082 weightList=templateCoadd.weightList)
2084 def _noTemplateMessage(self, warpType):
2085 warpName = (warpType[0].upper() + warpType[1:])
2086 message =
"""No %(warpName)s warps were found to build the template coadd which is
2087 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2088 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2089 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2091 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2092 another algorithm like:
2094 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2095 config.assemble.retarget(SafeClipAssembleCoaddTask)
2096 """ % {
"warpName": warpName}
2099 @utils.inheritDoc(AssembleCoaddTask)
2100 @pipeBase.timeMethod
2101 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2102 supplementaryData, *args, **kwargs):
2103 """Assemble the coadd.
2105 Find artifacts and apply them to the warps' masks creating a list of
2106 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2107 plane. Then pass these alternative masks to the base class's `run`
2110 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2111 that must contain a ``templateCoadd`` that serves as the
2112 model of the static sky.
2118 dataIds = [ref.dataId
for ref
in tempExpRefList]
2119 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
2121 if dataIds != psfMatchedDataIds:
2122 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2123 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2124 psfMatchedDataIds, dataIds)
2125 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2126 psfMatchedDataIds, dataIds)
2129 spanSetMaskList = self.
findArtifacts(supplementaryData.templateCoadd,
2130 supplementaryData.warpRefList,
2131 supplementaryData.imageScalerList)
2133 badMaskPlanes = self.config.badMaskPlanes[:]
2134 badMaskPlanes.append(
"CLIPPED")
2135 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2137 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2138 spanSetMaskList, mask=badPixelMask)
2142 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2146 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2150 mask : `lsst.afw.image.Mask`
2152 altMaskList : `list`
2153 List of Dicts containing ``spanSet`` lists.
2154 Each element contains the new mask plane name (e.g. "CLIPPED
2155 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2158 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2159 for visitMask
in altMaskList:
2160 if "EDGE" in visitMask:
2161 for spanSet
in visitMask[
'EDGE']:
2162 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2167 Loop through warps twice. The first loop builds a map with the count
2168 of how many epochs each pixel deviates from the templateCoadd by more
2169 than ``config.chiThreshold`` sigma. The second loop takes each
2170 difference image and filters the artifacts detected in each using
2171 count map to filter out variable sources and sources that are
2172 difficult to subtract cleanly.
2176 templateCoadd : `lsst.afw.image.Exposure`
2177 Exposure to serve as model of static sky.
2178 tempExpRefList : `list`
2179 List of data references to warps.
2180 imageScalerList : `list`
2181 List of image scalers.
2186 List of dicts containing information about CLIPPED
2187 (i.e., artifacts), NO_DATA, and EDGE pixels.
2190 self.log.debug(
"Generating Count Image, and mask lists.")
2191 coaddBBox = templateCoadd.getBBox()
2192 slateIm = afwImage.ImageU(coaddBBox)
2193 epochCountImage = afwImage.ImageU(coaddBBox)
2194 nImage = afwImage.ImageU(coaddBBox)
2195 spanSetArtifactList = []
2196 spanSetNoDataMaskList = []
2197 spanSetEdgeList = []
2198 spanSetBadMorphoList = []
2199 badPixelMask = self.getBadPixelMask()
2202 templateCoadd.mask.clearAllMaskPlanes()
2204 if self.config.doPreserveContainedBySource:
2205 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2207 templateFootprints =
None
2209 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2211 if warpDiffExp
is not None:
2213 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2214 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2215 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2216 fpSet.positive.merge(fpSet.negative)
2217 footprints = fpSet.positive
2219 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2222 if self.config.doPrefilterArtifacts:
2226 self.detect.clearMask(warpDiffExp.mask)
2227 for spans
in spanSetList:
2228 spans.setImage(slateIm, 1, doClip=
True)
2229 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
2230 epochCountImage += slateIm
2232 if self.config.doFilterMorphological:
2233 maskName = self.config.streakMaskName
2234 _ = self.maskStreaks.
run(warpDiffExp)
2235 streakMask = warpDiffExp.mask
2236 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2237 streakMask.getPlaneBitMask(maskName)).split()
2243 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2244 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2245 nansMask.setXY0(warpDiffExp.getXY0())
2246 edgeMask = warpDiffExp.mask
2247 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2248 edgeMask.getPlaneBitMask(
"EDGE")).split()
2252 nansMask = afwImage.MaskX(coaddBBox, 1)
2254 spanSetEdgeMask = []
2257 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2259 spanSetNoDataMaskList.append(spanSetNoDataMask)
2260 spanSetArtifactList.append(spanSetList)
2261 spanSetEdgeList.append(spanSetEdgeMask)
2262 if self.config.doFilterMorphological:
2263 spanSetBadMorphoList.append(spanSetStreak)
2267 epochCountImage.writeFits(path)
2269 for i, spanSetList
in enumerate(spanSetArtifactList):
2271 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
2273 spanSetArtifactList[i] = filteredSpanSetList
2274 if self.config.doFilterMorphological:
2275 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2278 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2279 altMasks.append({
'CLIPPED': artifacts,
2285 """Remove artifact candidates covered by bad mask plane.
2287 Any future editing of the candidate list that does not depend on
2288 temporal information should go in this method.
2292 spanSetList : `list`
2293 List of SpanSets representing artifact candidates.
2294 exp : `lsst.afw.image.Exposure`
2295 Exposure containing mask planes used to prefilter.
2299 returnSpanSetList : `list`
2300 List of SpanSets with artifacts.
2302 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2303 goodArr = (exp.mask.array & badPixelMask) == 0
2304 returnSpanSetList = []
2305 bbox = exp.getBBox()
2306 x0, y0 = exp.getXY0()
2307 for i, span
in enumerate(spanSetList):
2308 y, x = span.clippedTo(bbox).indices()
2309 yIndexLocal = numpy.array(y) - y0
2310 xIndexLocal = numpy.array(x) - x0
2311 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2312 if goodRatio > self.config.prefilterArtifactsRatio:
2313 returnSpanSetList.append(span)
2314 return returnSpanSetList
2316 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2317 """Filter artifact candidates.
2321 spanSetList : `list`
2322 List of SpanSets representing artifact candidates.
2323 epochCountImage : `lsst.afw.image.Image`
2324 Image of accumulated number of warpDiff detections.
2325 nImage : `lsst.afw.image.Image`
2326 Image of the accumulated number of total epochs contributing.
2330 maskSpanSetList : `list`
2331 List of SpanSets with artifacts.
2334 maskSpanSetList = []
2335 x0, y0 = epochCountImage.getXY0()
2336 for i, span
in enumerate(spanSetList):
2337 y, x = span.indices()
2338 yIdxLocal = [y1 - y0
for y1
in y]
2339 xIdxLocal = [x1 - x0
for x1
in x]
2340 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2341 totalN = nImage.array[yIdxLocal, xIdxLocal]
2344 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2345 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2346 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2347 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2348 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2349 & (outlierN <= effectiveMaxNumEpochs))
2350 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2351 if percentBelowThreshold > self.config.spatialThreshold:
2352 maskSpanSetList.append(span)
2354 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2356 filteredMaskSpanSetList = []
2357 for span
in maskSpanSetList:
2359 for footprint
in footprintsToExclude.positive.getFootprints():
2360 if footprint.spans.contains(span):
2364 filteredMaskSpanSetList.append(span)
2365 maskSpanSetList = filteredMaskSpanSetList
2367 return maskSpanSetList
2369 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2370 """Fetch a warp from the butler and return a warpDiff.
2374 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2375 Butler dataRef for the warp.
2376 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2377 An image scaler object.
2378 templateCoadd : `lsst.afw.image.Exposure`
2379 Exposure to be substracted from the scaled warp.
2383 warp : `lsst.afw.image.Exposure`
2384 Exposure of the image difference between the warp and template.
2392 warpName = self.getTempExpDatasetName(
'psfMatched')
2393 if not isinstance(warpRef, DeferredDatasetHandle):
2394 if not warpRef.datasetExists(warpName):
2395 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2397 warp = warpRef.get(datasetType=warpName, immediate=
True)
2399 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2400 mi = warp.getMaskedImage()
2401 if self.config.doScaleWarpVariance:
2403 self.scaleWarpVariance.
run(mi)
2404 except Exception
as exc:
2405 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2406 mi -= templateCoadd.getMaskedImage()
2409 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2410 """Return a path to which to write debugging output.
2412 Creates a hyphen-delimited string of dataId values for simple filenames.
2417 Prefix for filename.
2418 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2419 Butler dataRef to make the path from.
2420 coaddLevel : `bool`, optional.
2421 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2422 'filter', but no 'visit').
2427 Path for debugging output.
2430 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2432 keys = warpRef.dataId.keys()
2433 keyList = sorted(keys, reverse=
True)
2435 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2436 return os.path.join(directory, filename)
2440 """Match the order of one list to another, padding if necessary
2445 List to be reordered and padded. Elements can be any type.
2446 inputKeys : iterable
2447 Iterable of values to be compared with outputKeys.
2448 Length must match `inputList`
2449 outputKeys : iterable
2450 Iterable of values to be compared with inputKeys.
2452 Any value to be inserted where inputKey not in outputKeys
2457 Copy of inputList reordered per outputKeys and padded with `padWith`
2458 so that the length matches length of outputKeys.
2461 for d
in outputKeys:
2463 outputList.append(inputList[inputKeys.index(d)])
2465 outputList.append(padWith)