26 import lsst.pex.config
as pexConfig
40 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41 from .interpImage
import InterpImageTask
42 from .scaleZeroPoint
import ScaleZeroPointTask
43 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
44 from .scaleVariance
import ScaleVarianceTask
46 from lsst.daf.butler
import DeferredDatasetHandle
48 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
49 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
50 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
54 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap"),
55 defaultTemplates={
"inputCoaddName":
"deep",
56 "outputCoaddName":
"deep",
60 inputWarps = pipeBase.connectionTypes.Input(
61 doc=(
"Input list of warps to be assemebled i.e. stacked."
62 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
63 name=
"{inputCoaddName}Coadd_{warpType}Warp",
64 storageClass=
"ExposureF",
65 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
69 skyMap = pipeBase.connectionTypes.Input(
70 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
71 name=
"{inputCoaddName}Coadd_skyMap",
72 storageClass=
"SkyMap",
73 dimensions=(
"skymap", ),
75 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
76 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
78 name=
"brightObjectMask",
79 storageClass=
"ObjectMaskCatalog",
80 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
82 coaddExposure = pipeBase.connectionTypes.Output(
83 doc=
"Output coadded exposure, produced by stacking input warps",
84 name=
"{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
85 storageClass=
"ExposureF",
86 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
88 nImage = pipeBase.connectionTypes.Output(
89 doc=
"Output image of number of input images per pixel",
90 name=
"{outputCoaddName}Coadd_nImage",
91 storageClass=
"ImageU",
92 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
95 def __init__(self, *, config=None):
97 config.connections.warpType = config.warpType
101 config.connections.fakesType =
"_fakes"
103 super().__init__(config=config)
105 if not config.doMaskBrightObjects:
106 self.prerequisiteInputs.remove(
"brightObjectMask")
108 if not config.doNImage:
109 self.outputs.remove(
"nImage")
112 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
113 pipelineConnections=AssembleCoaddConnections):
114 """Configuration parameters for the `AssembleCoaddTask`.
118 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
119 only set the bitplane config.brightObjectMaskName. To make this useful you
120 *must* also configure the flags.pixel algorithm, for example by adding
124 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
125 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
127 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
129 warpType = pexConfig.Field(
130 doc=
"Warp name: one of 'direct' or 'psfMatched'",
134 subregionSize = pexConfig.ListField(
136 doc=
"Width, height of stack subregion size; "
137 "make small enough that a full stack of images will fit into memory at once.",
139 default=(2000, 2000),
141 statistic = pexConfig.Field(
143 doc=
"Main stacking statistic for aggregating over the epochs.",
146 doSigmaClip = pexConfig.Field(
148 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
151 sigmaClip = pexConfig.Field(
153 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
156 clipIter = pexConfig.Field(
158 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
161 calcErrorFromInputVariance = pexConfig.Field(
163 doc=
"Calculate coadd variance from input variance by stacking statistic."
164 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
167 scaleZeroPoint = pexConfig.ConfigurableField(
168 target=ScaleZeroPointTask,
169 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
171 doInterp = pexConfig.Field(
172 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
176 interpImage = pexConfig.ConfigurableField(
177 target=InterpImageTask,
178 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
180 doWrite = pexConfig.Field(
181 doc=
"Persist coadd?",
185 doNImage = pexConfig.Field(
186 doc=
"Create image of number of contributing exposures for each pixel",
190 doUsePsfMatchedPolygons = pexConfig.Field(
191 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
195 maskPropagationThresholds = pexConfig.DictField(
198 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
199 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
200 "would have contributed exceeds this value."),
201 default={
"SAT": 0.1},
203 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
204 doc=
"Mask planes to remove before coadding")
205 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
206 doc=
"Set mask and flag bits for bright objects?")
207 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
208 doc=
"Name of mask bit used for bright objects")
209 coaddPsf = pexConfig.ConfigField(
210 doc=
"Configuration for CoaddPsf",
211 dtype=measAlg.CoaddPsfConfig,
213 doAttachTransmissionCurve = pexConfig.Field(
214 dtype=bool, default=
False, optional=
False,
215 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
216 "(requires all input Exposures to have TransmissionCurves).")
218 hasFakes = pexConfig.Field(
221 doc=
"Should be set to True if fake sources have been inserted into the input data."
224 def setDefaults(self):
225 super().setDefaults()
226 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
233 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
234 self.warpType =
'psfMatched'
235 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
236 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
237 self.statistic =
"MEANCLIP"
238 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
239 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
240 "compute and set a non-zero coadd variance estimate." % (self.statistic))
242 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
243 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
244 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
245 if str(k)
not in unstackableStats]
246 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
247 % (self.statistic, stackableStats))
250 class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
251 """Assemble a coadded image from a set of warps (coadded temporary exposures).
253 We want to assemble a coadded image from a set of Warps (also called
254 coadded temporary exposures or ``coaddTempExps``).
255 Each input Warp covers a patch on the sky and corresponds to a single
256 run/visit/exposure of the covered patch. We provide the task with a list
257 of Warps (``selectDataList``) from which it selects Warps that cover the
258 specified patch (pointed at by ``dataRef``).
259 Each Warp that goes into a coadd will typically have an independent
260 photometric zero-point. Therefore, we must scale each Warp to set it to
261 a common photometric zeropoint. WarpType may be one of 'direct' or
262 'psfMatched', and the boolean configs `config.makeDirect` and
263 `config.makePsfMatched` set which of the warp types will be coadded.
264 The coadd is computed as a mean with optional outlier rejection.
265 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
266 Finally, Warps can have bad 'NaN' pixels which received no input from the
267 source calExps. We interpolate over these bad (NaN) pixels.
269 `AssembleCoaddTask` uses several sub-tasks. These are
271 - `ScaleZeroPointTask`
272 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
274 - interpolate across bad pixels (NaN) in the final coadd
276 You can retarget these subtasks if you wish.
280 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
281 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
282 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
283 no debug variables of its own. Some of the subtasks may support debug
284 variables. See the documentation for the subtasks for further information.
288 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
289 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
290 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
291 inputs: a data reference to the tract patch and filter to be coadded, and
292 a list of Warps to attempt to coadd. These are specified using ``--id`` and
293 ``--selectId``, respectively:
297 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
298 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
300 Only the Warps that cover the specified tract and patch will be coadded.
301 A list of the available optional arguments can be obtained by calling
302 ``assembleCoadd.py`` with the ``--help`` command line argument:
306 assembleCoadd.py --help
308 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
309 multi-band processing, we will generate the HSC-I & -R band coadds from
310 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
311 assuming that the lsst stack has been already set up, we must set up the
312 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
313 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
314 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
315 coadds, we must first
318 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
320 - create a skymap that covers the area of the sky present in the raw exposures
322 - warp the individual calibrated exposures to the tangent plane of the coadd
324 We can perform all of these steps by running
328 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
330 This will produce warped exposures for each visit. To coadd the warped
331 data, we call assembleCoadd.py as follows:
335 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
336 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
337 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
338 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
339 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
340 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
341 --selectId visit=903988 ccd=24
343 that will process the HSC-I band data. The results are written in
344 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
346 You may also choose to run:
350 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
351 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
352 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
353 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
354 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
355 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
356 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
357 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
359 to generate the coadd for the HSC-R band if you are interested in
360 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
361 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
362 rather than `AssembleCoaddTask` to make the coadd.
364 ConfigClass = AssembleCoaddConfig
365 _DefaultName =
"assembleCoadd"
367 def __init__(self, *args, **kwargs):
370 argNames = [
"config",
"name",
"parentTask",
"log"]
371 kwargs.update({k: v
for k, v
in zip(argNames, args)})
372 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
373 "PipelineTask will not take positional args" % argNames, FutureWarning)
375 super().__init__(**kwargs)
376 self.makeSubtask(
"interpImage")
377 self.makeSubtask(
"scaleZeroPoint")
379 if self.config.doMaskBrightObjects:
380 mask = afwImage.Mask()
382 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
383 except pexExceptions.LsstCppException:
384 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
385 mask.getMaskPlaneDict().keys())
388 self.warpType = self.config.warpType
390 @utils.inheritDoc(pipeBase.PipelineTask)
391 def runQuantum(self, butlerQC, inputRefs, outputRefs):
396 Assemble a coadd from a set of Warps.
398 PipelineTask (Gen3) entry point to Coadd a set of Warps.
399 Analogous to `runDataRef`, it prepares all the data products to be
400 passed to `run`, and processes the results before returning a struct
401 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
402 Therefore, its inputs are accessed subregion by subregion
403 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
404 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
405 correspond to an update in `runDataRef` while both entry points
408 inputData = butlerQC.get(inputRefs)
412 skyMap = inputData[
"skyMap"]
413 outputDataId = butlerQC.quantum.dataId
416 tractId=outputDataId[
'tract'],
417 patchId=outputDataId[
'patch'])
421 warpRefList = inputData[
'inputWarps']
423 inputs = self.prepareInputs(warpRefList)
424 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
425 self.getTempExpDatasetName(self.warpType))
426 if len(inputs.tempExpRefList) == 0:
427 self.log.warn(
"No coadd temporary exposures found")
430 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
431 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
432 inputs.weightList, supplementaryData=supplementaryData)
434 inputData.setdefault(
'brightObjectMask',
None)
435 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
437 if self.config.doWrite:
438 butlerQC.put(retStruct, outputRefs)
442 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
443 """Assemble a coadd from a set of Warps.
445 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
446 Compute weights to be applied to each Warp and
447 find scalings to match the photometric zeropoint to a reference Warp.
448 Assemble the Warps using `run`. Interpolate over NaNs and
449 optionally write the coadd to disk. Return the coadded exposure.
453 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
454 Data reference defining the patch for coaddition and the
455 reference Warp (if ``config.autoReference=False``).
456 Used to access the following data products:
457 - ``self.config.coaddName + "Coadd_skyMap"``
458 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
459 - ``self.config.coaddName + "Coadd"``
460 selectDataList : `list`
461 List of data references to Calexps. Data to be coadded will be
462 selected from this list based on overlap with the patch defined
463 by dataRef, grouped by visit, and converted to a list of data
466 List of data references to Warps to be coadded.
467 Note: `warpRefList` is just the new name for `tempExpRefList`.
471 retStruct : `lsst.pipe.base.Struct`
472 Result struct with components:
474 - ``coaddExposure``: coadded exposure (``Exposure``).
475 - ``nImage``: exposure count image (``Image``).
477 if selectDataList
and warpRefList:
478 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, "
479 "and which to use is ambiguous. Please pass only one.")
481 skyInfo = self.getSkyInfo(dataRef)
482 if warpRefList
is None:
483 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
484 if len(calExpRefList) == 0:
485 self.log.warn(
"No exposures to coadd")
487 self.log.info(
"Coadding %d exposures", len(calExpRefList))
489 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
491 inputData = self.prepareInputs(warpRefList)
492 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
493 self.getTempExpDatasetName(self.warpType))
494 if len(inputData.tempExpRefList) == 0:
495 self.log.warn(
"No coadd temporary exposures found")
498 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
500 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
501 inputData.weightList, supplementaryData=supplementaryData)
503 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None
504 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
506 if self.config.doWrite:
507 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
508 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
510 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
511 self.log.info(
"Persisting %s" % coaddDatasetName)
512 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
513 if self.config.doNImage
and retStruct.nImage
is not None:
514 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
519 """Interpolate over missing data and mask bright stars.
523 coaddExposure : `lsst.afw.image.Exposure`
524 The coadded exposure to process.
525 dataRef : `lsst.daf.persistence.ButlerDataRef`
526 Butler data reference for supplementary data.
528 if self.config.doInterp:
529 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
531 varArray = coaddExposure.variance.array
532 with numpy.errstate(invalid=
"ignore"):
533 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
535 if self.config.doMaskBrightObjects:
536 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
539 """Make additional inputs to run() specific to subclasses (Gen2)
541 Duplicates interface of `runDataRef` method
542 Available to be implemented by subclasses only if they need the
543 coadd dataRef for performing preliminary processing before
544 assembling the coadd.
548 dataRef : `lsst.daf.persistence.ButlerDataRef`
549 Butler data reference for supplementary data.
550 selectDataList : `list` (optional)
551 Optional List of data references to Calexps.
552 warpRefList : `list` (optional)
553 Optional List of data references to Warps.
555 return pipeBase.Struct()
558 """Make additional inputs to run() specific to subclasses (Gen3)
560 Duplicates interface of `runQuantum` method.
561 Available to be implemented by subclasses only if they need the
562 coadd dataRef for performing preliminary processing before
563 assembling the coadd.
567 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
568 Gen3 Butler object for fetching additional data products before
569 running the Task specialized for quantum being processed
570 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
571 Attributes are the names of the connections describing input dataset types.
572 Values are DatasetRefs that task consumes for corresponding dataset type.
573 DataIds are guaranteed to match data objects in ``inputData``.
574 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
575 Attributes are the names of the connections describing output dataset types.
576 Values are DatasetRefs that task is to produce
577 for corresponding dataset type.
579 return pipeBase.Struct()
582 """Generate list data references corresponding to warped exposures
583 that lie within the patch to be coadded.
588 Data reference for patch.
589 calExpRefList : `list`
590 List of data references for input calexps.
594 tempExpRefList : `list`
595 List of Warp/CoaddTempExp data references.
597 butler = patchRef.getButler()
598 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
599 self.getTempExpDatasetName(self.warpType))
600 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
601 g, groupData.keys)
for
602 g
in groupData.groups.keys()]
603 return tempExpRefList
606 """Prepare the input warps for coaddition by measuring the weight for
607 each warp and the scaling for the photometric zero point.
609 Each Warp has its own photometric zeropoint and background variance.
610 Before coadding these Warps together, compute a scale factor to
611 normalize the photometric zeropoint and compute the weight for each Warp.
616 List of data references to tempExp
620 result : `lsst.pipe.base.Struct`
621 Result struct with components:
623 - ``tempExprefList``: `list` of data references to tempExp.
624 - ``weightList``: `list` of weightings.
625 - ``imageScalerList``: `list` of image scalers.
627 statsCtrl = afwMath.StatisticsControl()
628 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
629 statsCtrl.setNumIter(self.config.clipIter)
630 statsCtrl.setAndMask(self.getBadPixelMask())
631 statsCtrl.setNanSafe(
True)
638 tempExpName = self.getTempExpDatasetName(self.warpType)
639 for tempExpRef
in refList:
642 if not isinstance(tempExpRef, DeferredDatasetHandle):
643 if not tempExpRef.datasetExists(tempExpName):
644 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
647 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
649 if numpy.isnan(tempExp.image.array).all():
651 maskedImage = tempExp.getMaskedImage()
652 imageScaler = self.scaleZeroPoint.computeImageScaler(
657 imageScaler.scaleMaskedImage(maskedImage)
658 except Exception
as e:
659 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
661 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
662 afwMath.MEANCLIP, statsCtrl)
663 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
664 weight = 1.0 / float(meanVar)
665 if not numpy.isfinite(weight):
666 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
668 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
673 tempExpRefList.append(tempExpRef)
674 weightList.append(weight)
675 imageScalerList.append(imageScaler)
677 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
678 imageScalerList=imageScalerList)
681 """Prepare the statistics for coadding images.
685 mask : `int`, optional
686 Bit mask value to exclude from coaddition.
690 stats : `lsst.pipe.base.Struct`
691 Statistics structure with the following fields:
693 - ``statsCtrl``: Statistics control object for coadd
694 (`lsst.afw.math.StatisticsControl`)
695 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
698 mask = self.getBadPixelMask()
699 statsCtrl = afwMath.StatisticsControl()
700 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
701 statsCtrl.setNumIter(self.config.clipIter)
702 statsCtrl.setAndMask(mask)
703 statsCtrl.setNanSafe(
True)
704 statsCtrl.setWeighted(
True)
705 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
706 for plane, threshold
in self.config.maskPropagationThresholds.items():
707 bit = afwImage.Mask.getMaskPlane(plane)
708 statsCtrl.setMaskPropagationThreshold(bit, threshold)
709 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
710 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
713 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
714 altMaskList=None, mask=None, supplementaryData=None):
715 """Assemble a coadd from input warps
717 Assemble the coadd using the provided list of coaddTempExps. Since
718 the full coadd covers a patch (a large area), the assembly is
719 performed over small areas on the image at a time in order to
720 conserve memory usage. Iterate over subregions within the outer
721 bbox of the patch using `assembleSubregion` to stack the corresponding
722 subregions from the coaddTempExps with the statistic specified.
723 Set the edge bits the coadd mask based on the weight map.
727 skyInfo : `lsst.pipe.base.Struct`
728 Struct with geometric information about the patch.
729 tempExpRefList : `list`
730 List of data references to Warps (previously called CoaddTempExps).
731 imageScalerList : `list`
732 List of image scalers.
735 altMaskList : `list`, optional
736 List of alternate masks to use rather than those stored with
738 mask : `int`, optional
739 Bit mask value to exclude from coaddition.
740 supplementaryData : lsst.pipe.base.Struct, optional
741 Struct with additional data products needed to assemble coadd.
742 Only used by subclasses that implement `makeSupplementaryData`
747 result : `lsst.pipe.base.Struct`
748 Result struct with components:
750 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
751 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
752 - ``warpRefList``: input list of refs to the warps (
753 ``lsst.daf.butler.DeferredDatasetHandle`` or
754 ``lsst.daf.persistence.ButlerDataRef``)
756 - ``imageScalerList``: input list of image scalers (unmodified)
757 - ``weightList``: input list of weights (unmodified)
759 tempExpName = self.getTempExpDatasetName(self.warpType)
760 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
761 stats = self.prepareStats(mask=mask)
763 if altMaskList
is None:
764 altMaskList = [
None]*len(tempExpRefList)
766 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
767 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
768 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
769 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
770 coaddMaskedImage = coaddExposure.getMaskedImage()
771 subregionSizeArr = self.config.subregionSize
772 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
774 if self.config.doNImage:
775 nImage = afwImage.ImageU(skyInfo.bbox)
778 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
780 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
781 weightList, altMaskList, stats.flags, stats.ctrl,
783 except Exception
as e:
784 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
786 self.setInexactPsf(coaddMaskedImage.getMask())
789 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
790 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
791 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
792 weightList=weightList)
795 """Set the metadata for the coadd.
797 This basic implementation sets the filter from the first input.
801 coaddExposure : `lsst.afw.image.Exposure`
802 The target exposure for the coadd.
803 tempExpRefList : `list`
804 List of data references to tempExp.
808 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
809 tempExpName = self.getTempExpDatasetName(self.warpType)
815 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
817 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
820 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
821 for tempExpRef
in tempExpRefList]
822 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
824 coaddExposure.setFilter(tempExpList[0].getFilter())
825 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
826 coaddInputs.ccds.reserve(numCcds)
827 coaddInputs.visits.reserve(len(tempExpList))
829 for tempExp, weight
in zip(tempExpList, weightList):
830 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
832 if self.config.doUsePsfMatchedPolygons:
833 self.shrinkValidPolygons(coaddInputs)
835 coaddInputs.visits.sort()
836 if self.warpType ==
"psfMatched":
841 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
842 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
843 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
845 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
846 self.config.coaddPsf.makeControl())
847 coaddExposure.setPsf(psf)
848 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
849 coaddExposure.getWcs())
850 coaddExposure.getInfo().setApCorrMap(apCorrMap)
851 if self.config.doAttachTransmissionCurve:
852 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
853 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
856 altMaskList, statsFlags, statsCtrl, nImage=None):
857 """Assemble the coadd for a sub-region.
859 For each coaddTempExp, check for (and swap in) an alternative mask
860 if one is passed. Remove mask planes listed in
861 `config.removeMaskPlanes`. Finally, stack the actual exposures using
862 `lsst.afw.math.statisticsStack` with the statistic specified by
863 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
864 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
865 an N-sigma clipped mean where N and iterations are specified by
866 statsCtrl. Assign the stacked subregion back to the coadd.
870 coaddExposure : `lsst.afw.image.Exposure`
871 The target exposure for the coadd.
872 bbox : `lsst.geom.Box`
874 tempExpRefList : `list`
875 List of data reference to tempExp.
876 imageScalerList : `list`
877 List of image scalers.
881 List of alternate masks to use rather than those stored with
882 tempExp, or None. Each element is dict with keys = mask plane
883 name to which to add the spans.
884 statsFlags : `lsst.afw.math.Property`
885 Property object for statistic for coadd.
886 statsCtrl : `lsst.afw.math.StatisticsControl`
887 Statistics control object for coadd.
888 nImage : `lsst.afw.image.ImageU`, optional
889 Keeps track of exposure count for each pixel.
891 self.log.debug(
"Computing coadd over %s", bbox)
892 tempExpName = self.getTempExpDatasetName(self.warpType)
893 coaddExposure.mask.addMaskPlane(
"REJECTED")
894 coaddExposure.mask.addMaskPlane(
"CLIPPED")
895 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
896 maskMap = self.setRejectedMaskMapping(statsCtrl)
897 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
899 if nImage
is not None:
900 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
901 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
903 if isinstance(tempExpRef, DeferredDatasetHandle):
905 exposure = tempExpRef.get(parameters={
'bbox': bbox})
908 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
910 maskedImage = exposure.getMaskedImage()
911 mask = maskedImage.getMask()
912 if altMask
is not None:
913 self.applyAltMaskPlanes(mask, altMask)
914 imageScaler.scaleMaskedImage(maskedImage)
918 if nImage
is not None:
919 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
920 if self.config.removeMaskPlanes:
921 self.removeMaskPlanes(maskedImage)
922 maskedImageList.append(maskedImage)
924 with self.timer(
"stack"):
925 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
928 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
929 if nImage
is not None:
930 nImage.assign(subNImage, bbox)
933 """Unset the mask of an image for mask planes specified in the config.
937 maskedImage : `lsst.afw.image.MaskedImage`
938 The masked image to be modified.
940 mask = maskedImage.getMask()
941 for maskPlane
in self.config.removeMaskPlanes:
943 mask &= ~mask.getPlaneBitMask(maskPlane)
944 except pexExceptions.InvalidParameterError:
945 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
949 def setRejectedMaskMapping(statsCtrl):
950 """Map certain mask planes of the warps to new planes for the coadd.
952 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
953 or CLIPPED, set it to REJECTED on the coadd.
954 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
955 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
959 statsCtrl : `lsst.afw.math.StatisticsControl`
960 Statistics control object for coadd
964 maskMap : `list` of `tuple` of `int`
965 A list of mappings of mask planes of the warped exposures to
966 mask planes of the coadd.
968 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
969 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
970 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
971 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
972 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
973 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
978 """Apply in place alt mask formatted as SpanSets to a mask.
982 mask : `lsst.afw.image.Mask`
984 altMaskSpans : `dict`
985 SpanSet lists to apply. Each element contains the new mask
986 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
987 and list of SpanSets to apply to the mask.
991 mask : `lsst.afw.image.Mask`
994 if self.config.doUsePsfMatchedPolygons:
995 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1000 for spanSet
in altMaskSpans[
'NO_DATA']:
1001 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1003 for plane, spanSetList
in altMaskSpans.items():
1004 maskClipValue = mask.addMaskPlane(plane)
1005 for spanSet
in spanSetList:
1006 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1010 """Shrink coaddInputs' ccds' ValidPolygons in place.
1012 Either modify each ccd's validPolygon in place, or if CoaddInputs
1013 does not have a validPolygon, create one from its bbox.
1017 coaddInputs : `lsst.afw.image.coaddInputs`
1021 for ccd
in coaddInputs.ccds:
1022 polyOrig = ccd.getValidPolygon()
1023 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1024 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1026 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1028 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1029 ccd.setValidPolygon(validPolygon)
1032 """Retrieve the bright object masks.
1034 Returns None on failure.
1038 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1043 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1044 Bright object mask from the Butler object, or None if it cannot
1048 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1049 except Exception
as e:
1050 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1054 """Set the bright object masks.
1058 exposure : `lsst.afw.image.Exposure`
1059 Exposure under consideration.
1060 dataId : `lsst.daf.persistence.dataId`
1061 Data identifier dict for patch.
1062 brightObjectMasks : `lsst.afw.table`
1063 Table of bright objects to mask.
1066 if brightObjectMasks
is None:
1067 self.log.warn(
"Unable to apply bright object mask: none supplied")
1069 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1070 mask = exposure.getMaskedImage().getMask()
1071 wcs = exposure.getWcs()
1072 plateScale = wcs.getPixelScale().asArcseconds()
1074 for rec
in brightObjectMasks:
1075 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1076 if rec[
"type"] ==
"box":
1077 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1078 width = rec[
"width"].asArcseconds()/plateScale
1079 height = rec[
"height"].asArcseconds()/plateScale
1082 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1085 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1086 spans = afwGeom.SpanSet(bbox)
1087 elif rec[
"type"] ==
"circle":
1088 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1089 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1091 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
1093 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1096 """Set INEXACT_PSF mask plane.
1098 If any of the input images isn't represented in the coadd (due to
1099 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1104 mask : `lsst.afw.image.Mask`
1105 Coadded exposure's mask, modified in-place.
1107 mask.addMaskPlane(
"INEXACT_PSF")
1108 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1109 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1110 clipped = mask.getPlaneBitMask(
"CLIPPED")
1111 rejected = mask.getPlaneBitMask(
"REJECTED")
1112 array = mask.getArray()
1113 selected = array & (sensorEdge | clipped | rejected) > 0
1114 array[selected] |= inexactPsf
1117 def _makeArgumentParser(cls):
1118 """Create an argument parser.
1120 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1121 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_"
1122 + cls.ConfigClass().warpType +
"Warp",
1123 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1124 ContainerClass=AssembleCoaddDataIdContainer)
1125 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1126 ContainerClass=SelectDataIdContainer)
1130 def _subBBoxIter(bbox, subregionSize):
1131 """Iterate over subregions of a bbox.
1135 bbox : `lsst.geom.Box2I`
1136 Bounding box over which to iterate.
1137 subregionSize: `lsst.geom.Extent2I`
1142 subBBox : `lsst.geom.Box2I`
1143 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1144 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1145 the edges of ``bbox``, but it will never be empty.
1148 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1149 if subregionSize[0] < 1
or subregionSize[1] < 1:
1150 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1152 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1153 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1156 if subBBox.isEmpty():
1157 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1158 "colShift=%s, rowShift=%s" %
1159 (bbox, subregionSize, colShift, rowShift))
1164 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1168 """Make self.refList from self.idList.
1173 Results of parsing command-line (with ``butler`` and ``log`` elements).
1175 datasetType = namespace.config.coaddName +
"Coadd"
1176 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1178 for dataId
in self.idList:
1180 for key
in keysCoadd:
1181 if key
not in dataId:
1182 raise RuntimeError(
"--id must include " + key)
1184 dataRef = namespace.butler.dataRef(
1185 datasetType=datasetType,
1188 self.refList.append(dataRef)
1192 """Function to count the number of pixels with a specific mask in a
1195 Find the intersection of mask & footprint. Count all pixels in the mask
1196 that are in the intersection that have bitmask set but do not have
1197 ignoreMask set. Return the count.
1201 mask : `lsst.afw.image.Mask`
1202 Mask to define intersection region by.
1203 footprint : `lsst.afw.detection.Footprint`
1204 Footprint to define the intersection region by.
1206 Specific mask that we wish to count the number of occurances of.
1208 Pixels to not consider.
1213 Count of number of pixels in footprint with specified mask.
1215 bbox = footprint.getBBox()
1216 bbox.clip(mask.getBBox(afwImage.PARENT))
1217 fp = afwImage.Mask(bbox)
1218 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1219 footprint.spans.setMask(fp, bitmask)
1220 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1221 (subMask.getArray() & ignoreMask) == 0).sum()
1225 """Configuration parameters for the SafeClipAssembleCoaddTask.
1227 assembleMeanCoadd = pexConfig.ConfigurableField(
1228 target=AssembleCoaddTask,
1229 doc=
"Task to assemble an initial Coadd using the MEAN statistic.",
1231 assembleMeanClipCoadd = pexConfig.ConfigurableField(
1232 target=AssembleCoaddTask,
1233 doc=
"Task to assemble an initial Coadd using the MEANCLIP statistic.",
1235 clipDetection = pexConfig.ConfigurableField(
1236 target=SourceDetectionTask,
1237 doc=
"Detect sources on difference between unclipped and clipped coadd")
1238 minClipFootOverlap = pexConfig.Field(
1239 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1243 minClipFootOverlapSingle = pexConfig.Field(
1244 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1245 "clipped when only one visit overlaps",
1249 minClipFootOverlapDouble = pexConfig.Field(
1250 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1251 "clipped when two visits overlap",
1255 maxClipFootOverlapDouble = pexConfig.Field(
1256 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
1257 "considering two visits",
1261 minBigOverlap = pexConfig.Field(
1262 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1263 "when labeling clipped footprints",
1269 """Set default values for clipDetection.
1273 The numeric values for these configuration parameters were
1274 empirically determined, future work may further refine them.
1276 AssembleCoaddConfig.setDefaults(self)
1296 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1297 "Ignoring doSigmaClip.")
1300 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1301 "(%s chosen). Please set statistic to MEAN."
1303 AssembleCoaddTask.ConfigClass.validate(self)
1307 """Assemble a coadded image from a set of coadded temporary exposures,
1308 being careful to clip & flag areas with potential artifacts.
1310 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1311 we clip outliers). The problem with doing this is that when computing the
1312 coadd PSF at a given location, individual visit PSFs from visits with
1313 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1314 In this task, we correct for this behavior by creating a new
1315 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1316 coaddTempExps and the final coadd where
1318 i. difference imaging suggests that there is an outlier and
1319 ii. this outlier appears on only one or two images.
1321 Such regions will not contribute to the final coadd. Furthermore, any
1322 routine to determine the coadd PSF can now be cognizant of clipped regions.
1323 Note that the algorithm implemented by this task is preliminary and works
1324 correctly for HSC data. Parameter modifications and or considerable
1325 redesigning of the algorithm is likley required for other surveys.
1327 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1328 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1329 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1334 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1335 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1336 see `baseDebug` for more about ``debug.py`` files.
1337 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1338 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1339 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1340 for further information.
1344 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1345 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1346 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1348 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1349 and filter to be coadded (specified using
1350 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1351 along with a list of coaddTempExps to attempt to coadd (specified using
1352 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1353 Only the coaddTempExps that cover the specified tract and patch will be
1354 coadded. A list of the available optional arguments can be obtained by
1355 calling assembleCoadd.py with the --help command line argument:
1357 .. code-block:: none
1359 assembleCoadd.py --help
1361 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1362 context of multi-band processing, we will generate the HSC-I & -R band
1363 coadds from HSC engineering test data provided in the ci_hsc package.
1364 To begin, assuming that the lsst stack has been already set up, we must
1365 set up the obs_subaru and ci_hsc packages. This defines the environment
1366 variable $CI_HSC_DIR and points at the location of the package. The raw
1367 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1368 the coadds, we must first
1371 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1373 create a skymap that covers the area of the sky present in the raw exposures
1374 - ``makeCoaddTempExp``
1375 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1377 We can perform all of these steps by running
1379 .. code-block:: none
1381 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1383 This will produce warped coaddTempExps for each visit. To coadd the
1384 warped data, we call ``assembleCoadd.py`` as follows:
1386 .. code-block:: none
1388 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1389 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1390 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1391 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1392 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1393 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1394 --selectId visit=903988 ccd=24
1396 This will process the HSC-I band data. The results are written in
1397 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1399 You may also choose to run:
1401 .. code-block:: none
1403 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1404 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1405 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1406 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1407 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1408 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1409 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1410 --selectId visit=903346 ccd=12
1412 to generate the coadd for the HSC-R band if you are interested in following
1413 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1415 ConfigClass = SafeClipAssembleCoaddConfig
1416 _DefaultName =
"safeClipAssembleCoadd"
1419 AssembleCoaddTask.__init__(self, *args, **kwargs)
1420 schema = afwTable.SourceTable.makeMinimalSchema()
1421 self.makeSubtask(
"clipDetection", schema=schema)
1422 self.makeSubtask(
"assembleMeanClipCoadd")
1423 self.makeSubtask(
"assembleMeanCoadd")
1425 @utils.inheritDoc(AssembleCoaddTask)
1426 @pipeBase.timeMethod
1427 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1428 """Assemble the coadd for a region.
1430 Compute the difference of coadds created with and without outlier
1431 rejection to identify coadd pixels that have outlier values in some
1433 Detect clipped regions on the difference image and mark these regions
1434 on the one or two individual coaddTempExps where they occur if there
1435 is significant overlap between the clipped region and a source. This
1436 leaves us with a set of footprints from the difference image that have
1437 been identified as having occured on just one or two individual visits.
1438 However, these footprints were generated from a difference image. It
1439 is conceivable for a large diffuse source to have become broken up
1440 into multiple footprints acrosss the coadd difference in this process.
1441 Determine the clipped region from all overlapping footprints from the
1442 detected sources in each visit - these are big footprints.
1443 Combine the small and big clipped footprints and mark them on a new
1445 Generate the coadd using `AssembleCoaddTask.run` without outlier
1446 removal. Clipped footprints will no longer make it into the coadd
1447 because they are marked in the new bad mask plane.
1451 args and kwargs are passed but ignored in order to match the call
1452 signature expected by the parent task.
1455 mask = exp.getMaskedImage().getMask()
1456 mask.addMaskPlane(
"CLIPPED")
1458 result = self.
detectClip(exp, tempExpRefList)
1460 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1462 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1463 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1465 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1466 result.detectionFootprints, maskClipValue, maskDetValue,
1469 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1470 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1472 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1473 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1474 maskClip |= maskClipBig
1477 badMaskPlanes = self.config.badMaskPlanes[:]
1478 badMaskPlanes.append(
"CLIPPED")
1479 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1480 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1481 result.clipSpans, mask=badPixelMask)
1484 """Return an exposure that contains the difference between unclipped
1487 Generate a difference image between clipped and unclipped coadds.
1488 Compute the difference image by subtracting an outlier-clipped coadd
1489 from an outlier-unclipped coadd. Return the difference image.
1493 skyInfo : `lsst.pipe.base.Struct`
1494 Patch geometry information, from getSkyInfo
1495 tempExpRefList : `list`
1496 List of data reference to tempExp
1497 imageScalerList : `list`
1498 List of image scalers
1504 exp : `lsst.afw.image.Exposure`
1505 Difference image of unclipped and clipped coadd wrapped in an Exposure
1507 coaddMean = self.assembleMeanCoadd.
run(skyInfo, tempExpRefList,
1508 imageScalerList, weightList).coaddExposure
1510 coaddClip = self.assembleMeanClipCoadd.
run(skyInfo, tempExpRefList,
1511 imageScalerList, weightList).coaddExposure
1513 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1514 coaddDiff -= coaddClip.getMaskedImage()
1515 exp = afwImage.ExposureF(coaddDiff)
1516 exp.setPsf(coaddMean.getPsf())
1520 """Detect clipped regions on an exposure and set the mask on the
1521 individual tempExp masks.
1523 Detect footprints in the difference image after smoothing the
1524 difference image with a Gaussian kernal. Identify footprints that
1525 overlap with one or two input ``coaddTempExps`` by comparing the
1526 computed overlap fraction to thresholds set in the config. A different
1527 threshold is applied depending on the number of overlapping visits
1528 (restricted to one or two). If the overlap exceeds the thresholds,
1529 the footprint is considered "CLIPPED" and is marked as such on the
1530 coaddTempExp. Return a struct with the clipped footprints, the indices
1531 of the ``coaddTempExps`` that end up overlapping with the clipped
1532 footprints, and a list of new masks for the ``coaddTempExps``.
1536 exp : `lsst.afw.image.Exposure`
1537 Exposure to run detection on.
1538 tempExpRefList : `list`
1539 List of data reference to tempExp.
1543 result : `lsst.pipe.base.Struct`
1544 Result struct with components:
1546 - ``clipFootprints``: list of clipped footprints.
1547 - ``clipIndices``: indices for each ``clippedFootprint`` in
1549 - ``clipSpans``: List of dictionaries containing spanSet lists
1550 to clip. Each element contains the new maskplane name
1551 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1552 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1553 compressed into footprints.
1555 mask = exp.getMaskedImage().getMask()
1556 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1557 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1559 fpSet.positive.merge(fpSet.negative)
1560 footprints = fpSet.positive
1561 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1562 ignoreMask = self.getBadPixelMask()
1566 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1569 visitDetectionFootprints = []
1571 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1572 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1573 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1576 for i, warpRef
in enumerate(tempExpRefList):
1577 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1578 immediate=
True).getMaskedImage().getMask()
1579 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1580 afwImage.PARENT,
True)
1581 maskVisitDet &= maskDetValue
1582 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1583 visitDetectionFootprints.append(visitFootprints)
1585 for j, footprint
in enumerate(footprints.getFootprints()):
1590 for j, footprint
in enumerate(footprints.getFootprints()):
1591 nPixel = footprint.getArea()
1594 for i
in range(len(tempExpRefList)):
1595 ignore = ignoreArr[i, j]
1596 overlapDet = overlapDetArr[i, j]
1597 totPixel = nPixel - ignore
1600 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1602 overlap.append(overlapDet/float(totPixel))
1605 overlap = numpy.array(overlap)
1606 if not len(overlap):
1613 if len(overlap) == 1:
1614 if overlap[0] > self.config.minClipFootOverlapSingle:
1619 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1620 if len(clipIndex) == 1:
1622 keepIndex = [clipIndex[0]]
1625 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1626 if len(clipIndex) == 2
and len(overlap) > 3:
1627 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1628 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1630 keepIndex = clipIndex
1635 for index
in keepIndex:
1636 globalIndex = indexList[index]
1637 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1639 clipIndices.append(numpy.array(indexList)[keepIndex])
1640 clipFootprints.append(footprint)
1642 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1643 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1645 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1646 maskClipValue, maskDetValue, coaddBBox):
1647 """Return individual warp footprints for large artifacts and append
1648 them to ``clipList`` in place.
1650 Identify big footprints composed of many sources in the coadd
1651 difference that may have originated in a large diffuse source in the
1652 coadd. We do this by indentifying all clipped footprints that overlap
1653 significantly with each source in all the coaddTempExps.
1658 List of alt mask SpanSets with clipping information. Modified.
1659 clipFootprints : `list`
1660 List of clipped footprints.
1661 clipIndices : `list`
1662 List of which entries in tempExpClipList each footprint belongs to.
1664 Mask value of clipped pixels.
1666 Mask value of detected pixels.
1667 coaddBBox : `lsst.geom.Box`
1668 BBox of the coadd and warps.
1672 bigFootprintsCoadd : `list`
1673 List of big footprints
1675 bigFootprintsCoadd = []
1676 ignoreMask = self.getBadPixelMask()
1677 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1678 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1679 for footprint
in visitFootprints.getFootprints():
1680 footprint.spans.setMask(maskVisitDet, maskDetValue)
1683 clippedFootprintsVisit = []
1684 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1685 if index
not in clipIndex:
1687 clippedFootprintsVisit.append(foot)
1688 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1689 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1691 bigFootprintsVisit = []
1692 for foot
in visitFootprints.getFootprints():
1693 if foot.getArea() < self.config.minBigOverlap:
1696 if nCount > self.config.minBigOverlap:
1697 bigFootprintsVisit.append(foot)
1698 bigFootprintsCoadd.append(foot)
1700 for footprint
in bigFootprintsVisit:
1701 clippedSpans[
"CLIPPED"].append(footprint.spans)
1703 return bigFootprintsCoadd
1707 psfMatchedWarps = pipeBase.connectionTypes.Input(
1708 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1709 "Only PSF-Matched Warps make sense for image subtraction. "
1710 "Therefore, they must be an additional declared input."),
1711 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1712 storageClass=
"ExposureF",
1713 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1717 templateCoadd = pipeBase.connectionTypes.Output(
1718 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1719 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1720 name=
"{fakesType}{outputCoaddName}CoaddPsfMatched",
1721 storageClass=
"ExposureF",
1722 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
1727 if not config.assembleStaticSkyModel.doWrite:
1728 self.outputs.remove(
"templateCoadd")
1733 pipelineConnections=CompareWarpAssembleCoaddConnections):
1734 assembleStaticSkyModel = pexConfig.ConfigurableField(
1735 target=AssembleCoaddTask,
1736 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1737 " naive/first-iteration model of the static sky.",
1739 detect = pexConfig.ConfigurableField(
1740 target=SourceDetectionTask,
1741 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1743 detectTemplate = pexConfig.ConfigurableField(
1744 target=SourceDetectionTask,
1745 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1747 maxNumEpochs = pexConfig.Field(
1748 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1749 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1750 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1751 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1752 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1753 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1754 "than transient and not masked.",
1758 maxFractionEpochsLow = pexConfig.RangeField(
1759 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1760 "Effective maxNumEpochs = "
1761 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1766 maxFractionEpochsHigh = pexConfig.RangeField(
1767 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1768 "Effective maxNumEpochs = "
1769 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1774 spatialThreshold = pexConfig.RangeField(
1775 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1776 "temporal criteria. If 0, clip all. If 1, clip none.",
1780 inclusiveMin=
True, inclusiveMax=
True
1782 doScaleWarpVariance = pexConfig.Field(
1783 doc=
"Rescale Warp variance plane using empirical noise?",
1787 scaleWarpVariance = pexConfig.ConfigurableField(
1788 target=ScaleVarianceTask,
1789 doc=
"Rescale variance on warps",
1791 doPreserveContainedBySource = pexConfig.Field(
1792 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1793 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1797 doPrefilterArtifacts = pexConfig.Field(
1798 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1799 "because they will be excluded anyway. This prevents them from contributing "
1800 "to the outlier epoch count image and potentially being labeled as persistant."
1801 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1805 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1806 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1808 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1810 prefilterArtifactsRatio = pexConfig.Field(
1811 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1817 AssembleCoaddConfig.setDefaults(self)
1823 if "EDGE" in self.badMaskPlanes:
1824 self.badMaskPlanes.remove(
'EDGE')
1825 self.removeMaskPlanes.append(
'EDGE')
1834 self.
detect.doTempLocalBackground =
False
1835 self.
detect.reEstimateBackground =
False
1836 self.
detect.returnOriginalFootprints =
False
1837 self.
detect.thresholdPolarity =
"both"
1838 self.
detect.thresholdValue = 5
1839 self.
detect.minPixels = 4
1840 self.
detect.isotropicGrow =
True
1841 self.
detect.thresholdType =
"pixel_stdev"
1842 self.
detect.nSigmaToGrow = 0.4
1853 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
1854 "Please set assembleStaticSkyModel.doNImage=False")
1857 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1858 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1859 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1864 """Assemble a compareWarp coadded image from a set of warps
1865 by masking artifacts detected by comparing PSF-matched warps.
1867 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1868 we clip outliers). The problem with doing this is that when computing the
1869 coadd PSF at a given location, individual visit PSFs from visits with
1870 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1871 In this task, we correct for this behavior by creating a new badMaskPlane
1872 'CLIPPED' which marks pixels in the individual warps suspected to contain
1873 an artifact. We populate this plane on the input warps by comparing
1874 PSF-matched warps with a PSF-matched median coadd which serves as a
1875 model of the static sky. Any group of pixels that deviates from the
1876 PSF-matched template coadd by more than config.detect.threshold sigma,
1877 is an artifact candidate. The candidates are then filtered to remove
1878 variable sources and sources that are difficult to subtract such as
1879 bright stars. This filter is configured using the config parameters
1880 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1881 the maximum fraction of epochs that the deviation can appear in and still
1882 be considered an artifact. The spatialThreshold is the maximum fraction of
1883 pixels in the footprint of the deviation that appear in other epochs
1884 (where other epochs is defined by the temporalThreshold). If the deviant
1885 region meets this criteria of having a significant percentage of pixels
1886 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1887 set in the mask. These regions will not contribute to the final coadd.
1888 Furthermore, any routine to determine the coadd PSF can now be cognizant
1889 of clipped regions. Note that the algorithm implemented by this task is
1890 preliminary and works correctly for HSC data. Parameter modifications and
1891 or considerable redesigning of the algorithm is likley required for other
1894 ``CompareWarpAssembleCoaddTask`` sub-classes
1895 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1896 as a subtask to generate the TemplateCoadd (the model of the static sky).
1900 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1901 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1902 ``baseDebug`` for more about ``debug.py`` files.
1904 This task supports the following debug variables:
1907 If True then save the Epoch Count Image as a fits file in the `figPath`
1909 Path to save the debug fits images and figures
1911 For example, put something like:
1913 .. code-block:: python
1916 def DebugInfo(name):
1917 di = lsstDebug.getInfo(name)
1918 if name == "lsst.pipe.tasks.assembleCoadd":
1919 di.saveCountIm = True
1920 di.figPath = "/desired/path/to/debugging/output/images"
1922 lsstDebug.Info = DebugInfo
1924 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1925 ``--debug`` flag. Some subtasks may have their own debug variables;
1926 see individual Task documentation.
1930 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1931 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1932 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1933 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1934 and filter to be coadded (specified using
1935 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1936 along with a list of coaddTempExps to attempt to coadd (specified using
1937 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1938 Only the warps that cover the specified tract and patch will be coadded.
1939 A list of the available optional arguments can be obtained by calling
1940 ``assembleCoadd.py`` with the ``--help`` command line argument:
1942 .. code-block:: none
1944 assembleCoadd.py --help
1946 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1947 context of multi-band processing, we will generate the HSC-I & -R band
1948 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1949 To begin, assuming that the lsst stack has been already set up, we must
1950 set up the ``obs_subaru`` and ``ci_hsc`` packages.
1951 This defines the environment variable ``$CI_HSC_DIR`` and points at the
1952 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1953 directory. To begin assembling the coadds, we must first
1956 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1958 create a skymap that covers the area of the sky present in the raw exposures
1960 warp the individual calibrated exposures to the tangent plane of the coadd
1962 We can perform all of these steps by running
1964 .. code-block:: none
1966 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1968 This will produce warped ``coaddTempExps`` for each visit. To coadd the
1969 warped data, we call ``assembleCoadd.py`` as follows:
1971 .. code-block:: none
1973 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1974 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1975 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1976 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1977 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1978 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1979 --selectId visit=903988 ccd=24
1981 This will process the HSC-I band data. The results are written in
1982 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1984 ConfigClass = CompareWarpAssembleCoaddConfig
1985 _DefaultName =
"compareWarpAssembleCoadd"
1988 AssembleCoaddTask.__init__(self, *args, **kwargs)
1989 self.makeSubtask(
"assembleStaticSkyModel")
1990 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1991 self.makeSubtask(
"detect", schema=detectionSchema)
1992 if self.config.doPreserveContainedBySource:
1993 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1994 if self.config.doScaleWarpVariance:
1995 self.makeSubtask(
"scaleWarpVariance")
1997 @utils.inheritDoc(AssembleCoaddTask)
2000 Generate a templateCoadd to use as a naive model of static sky to
2001 subtract from PSF-Matched warps.
2005 result : `lsst.pipe.base.Struct`
2006 Result struct with components:
2008 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2009 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2012 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2013 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2017 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2018 if self.config.assembleStaticSkyModel.doWrite:
2019 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2022 del outputRefs.templateCoadd
2023 del staticSkyModelOutputRefs.templateCoadd
2026 if 'nImage' in staticSkyModelOutputRefs.keys():
2027 del staticSkyModelOutputRefs.nImage
2029 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2030 staticSkyModelOutputRefs)
2031 if templateCoadd
is None:
2034 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2035 nImage=templateCoadd.nImage,
2036 warpRefList=templateCoadd.warpRefList,
2037 imageScalerList=templateCoadd.imageScalerList,
2038 weightList=templateCoadd.weightList)
2040 @utils.inheritDoc(AssembleCoaddTask)
2043 Generate a templateCoadd to use as a naive model of static sky to
2044 subtract from PSF-Matched warps.
2048 result : `lsst.pipe.base.Struct`
2049 Result struct with components:
2051 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2052 - ``nImage``: N Image (``lsst.afw.image.Image``)
2054 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2055 if templateCoadd
is None:
2058 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2059 nImage=templateCoadd.nImage,
2060 warpRefList=templateCoadd.warpRefList,
2061 imageScalerList=templateCoadd.imageScalerList,
2062 weightList=templateCoadd.weightList)
2064 def _noTemplateMessage(self, warpType):
2065 warpName = (warpType[0].upper() + warpType[1:])
2066 message =
"""No %(warpName)s warps were found to build the template coadd which is
2067 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2068 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2069 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2071 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2072 another algorithm like:
2074 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2075 config.assemble.retarget(SafeClipAssembleCoaddTask)
2076 """ % {
"warpName": warpName}
2079 @utils.inheritDoc(AssembleCoaddTask)
2080 @pipeBase.timeMethod
2081 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2082 supplementaryData, *args, **kwargs):
2083 """Assemble the coadd.
2085 Find artifacts and apply them to the warps' masks creating a list of
2086 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2087 plane. Then pass these alternative masks to the base class's `run`
2090 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2091 that must contain a ``templateCoadd`` that serves as the
2092 model of the static sky.
2098 dataIds = [ref.dataId
for ref
in tempExpRefList]
2099 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
2101 if dataIds != psfMatchedDataIds:
2102 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2103 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2104 psfMatchedDataIds, dataIds)
2105 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2106 psfMatchedDataIds, dataIds)
2109 spanSetMaskList = self.
findArtifacts(supplementaryData.templateCoadd,
2110 supplementaryData.warpRefList,
2111 supplementaryData.imageScalerList)
2113 badMaskPlanes = self.config.badMaskPlanes[:]
2114 badMaskPlanes.append(
"CLIPPED")
2115 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2117 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2118 spanSetMaskList, mask=badPixelMask)
2122 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2126 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2130 mask : `lsst.afw.image.Mask`
2132 altMaskList : `list`
2133 List of Dicts containing ``spanSet`` lists.
2134 Each element contains the new mask plane name (e.g. "CLIPPED
2135 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2138 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2139 for visitMask
in altMaskList:
2140 if "EDGE" in visitMask:
2141 for spanSet
in visitMask[
'EDGE']:
2142 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2147 Loop through warps twice. The first loop builds a map with the count
2148 of how many epochs each pixel deviates from the templateCoadd by more
2149 than ``config.chiThreshold`` sigma. The second loop takes each
2150 difference image and filters the artifacts detected in each using
2151 count map to filter out variable sources and sources that are
2152 difficult to subtract cleanly.
2156 templateCoadd : `lsst.afw.image.Exposure`
2157 Exposure to serve as model of static sky.
2158 tempExpRefList : `list`
2159 List of data references to warps.
2160 imageScalerList : `list`
2161 List of image scalers.
2166 List of dicts containing information about CLIPPED
2167 (i.e., artifacts), NO_DATA, and EDGE pixels.
2170 self.log.debug(
"Generating Count Image, and mask lists.")
2171 coaddBBox = templateCoadd.getBBox()
2172 slateIm = afwImage.ImageU(coaddBBox)
2173 epochCountImage = afwImage.ImageU(coaddBBox)
2174 nImage = afwImage.ImageU(coaddBBox)
2175 spanSetArtifactList = []
2176 spanSetNoDataMaskList = []
2177 spanSetEdgeList = []
2178 badPixelMask = self.getBadPixelMask()
2181 templateCoadd.mask.clearAllMaskPlanes()
2183 if self.config.doPreserveContainedBySource:
2184 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2186 templateFootprints =
None
2188 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2190 if warpDiffExp
is not None:
2192 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2193 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2194 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2195 fpSet.positive.merge(fpSet.negative)
2196 footprints = fpSet.positive
2198 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2201 if self.config.doPrefilterArtifacts:
2203 for spans
in spanSetList:
2204 spans.setImage(slateIm, 1, doClip=
True)
2205 epochCountImage += slateIm
2211 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2212 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2213 nansMask.setXY0(warpDiffExp.getXY0())
2214 edgeMask = warpDiffExp.mask
2215 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2216 edgeMask.getPlaneBitMask(
"EDGE")).split()
2220 nansMask = afwImage.MaskX(coaddBBox, 1)
2222 spanSetEdgeMask = []
2224 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2226 spanSetNoDataMaskList.append(spanSetNoDataMask)
2227 spanSetArtifactList.append(spanSetList)
2228 spanSetEdgeList.append(spanSetEdgeMask)
2232 epochCountImage.writeFits(path)
2234 for i, spanSetList
in enumerate(spanSetArtifactList):
2236 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
2238 spanSetArtifactList[i] = filteredSpanSetList
2241 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2242 altMasks.append({
'CLIPPED': artifacts,
2248 """Remove artifact candidates covered by bad mask plane.
2250 Any future editing of the candidate list that does not depend on
2251 temporal information should go in this method.
2255 spanSetList : `list`
2256 List of SpanSets representing artifact candidates.
2257 exp : `lsst.afw.image.Exposure`
2258 Exposure containing mask planes used to prefilter.
2262 returnSpanSetList : `list`
2263 List of SpanSets with artifacts.
2265 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2266 goodArr = (exp.mask.array & badPixelMask) == 0
2267 returnSpanSetList = []
2268 bbox = exp.getBBox()
2269 x0, y0 = exp.getXY0()
2270 for i, span
in enumerate(spanSetList):
2271 y, x = span.clippedTo(bbox).indices()
2272 yIndexLocal = numpy.array(y) - y0
2273 xIndexLocal = numpy.array(x) - x0
2274 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2275 if goodRatio > self.config.prefilterArtifactsRatio:
2276 returnSpanSetList.append(span)
2277 return returnSpanSetList
2279 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2280 """Filter artifact candidates.
2284 spanSetList : `list`
2285 List of SpanSets representing artifact candidates.
2286 epochCountImage : `lsst.afw.image.Image`
2287 Image of accumulated number of warpDiff detections.
2288 nImage : `lsst.afw.image.Image`
2289 Image of the accumulated number of total epochs contributing.
2293 maskSpanSetList : `list`
2294 List of SpanSets with artifacts.
2297 maskSpanSetList = []
2298 x0, y0 = epochCountImage.getXY0()
2299 for i, span
in enumerate(spanSetList):
2300 y, x = span.indices()
2301 yIdxLocal = [y1 - y0
for y1
in y]
2302 xIdxLocal = [x1 - x0
for x1
in x]
2303 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2304 totalN = nImage.array[yIdxLocal, xIdxLocal]
2307 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2308 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2309 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2310 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2311 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2312 & (outlierN <= effectiveMaxNumEpochs))
2313 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2314 if percentBelowThreshold > self.config.spatialThreshold:
2315 maskSpanSetList.append(span)
2317 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2319 filteredMaskSpanSetList = []
2320 for span
in maskSpanSetList:
2322 for footprint
in footprintsToExclude.positive.getFootprints():
2323 if footprint.spans.contains(span):
2327 filteredMaskSpanSetList.append(span)
2328 maskSpanSetList = filteredMaskSpanSetList
2330 return maskSpanSetList
2332 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2333 """Fetch a warp from the butler and return a warpDiff.
2337 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2338 Butler dataRef for the warp.
2339 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2340 An image scaler object.
2341 templateCoadd : `lsst.afw.image.Exposure`
2342 Exposure to be substracted from the scaled warp.
2346 warp : `lsst.afw.image.Exposure`
2347 Exposure of the image difference between the warp and template.
2355 warpName = self.getTempExpDatasetName(
'psfMatched')
2356 if not isinstance(warpRef, DeferredDatasetHandle):
2357 if not warpRef.datasetExists(warpName):
2358 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2360 warp = warpRef.get(datasetType=warpName, immediate=
True)
2362 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2363 mi = warp.getMaskedImage()
2364 if self.config.doScaleWarpVariance:
2366 self.scaleWarpVariance.
run(mi)
2367 except Exception
as exc:
2368 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2369 mi -= templateCoadd.getMaskedImage()
2372 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2373 """Return a path to which to write debugging output.
2375 Creates a hyphen-delimited string of dataId values for simple filenames.
2380 Prefix for filename.
2381 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2382 Butler dataRef to make the path from.
2383 coaddLevel : `bool`, optional.
2384 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2385 'filter', but no 'visit').
2390 Path for debugging output.
2393 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2395 keys = warpRef.dataId.keys()
2396 keyList = sorted(keys, reverse=
True)
2398 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2399 return os.path.join(directory, filename)
2403 """Match the order of one list to another, padding if necessary
2408 List to be reordered and padded. Elements can be any type.
2409 inputKeys : iterable
2410 Iterable of values to be compared with outputKeys.
2411 Length must match `inputList`
2412 outputKeys : iterable
2413 Iterable of values to be compared with inputKeys.
2415 Any value to be inserted where inputKey not in outputKeys
2420 Copy of inputList reordered per outputKeys and padded with `padWith`
2421 so that the length matches length of outputKeys.
2424 for d
in outputKeys:
2426 outputList.append(inputList[inputKeys.index(d)])
2428 outputList.append(padWith)