39import lsst.utils
as utils
41from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42from .interpImage
import InterpImageTask
43from .scaleZeroPoint
import ScaleZeroPointTask
44from .coaddHelpers
import groupPatchExposures, getGroupDataRef
45from .maskStreaks
import MaskStreaksTask
46from .healSparseMapping
import HealSparseInputMapTask
48from lsst.daf.butler
import DeferredDatasetHandle
49from lsst.utils.timer
import timeMethod
51__all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
52 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
53 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
55log = logging.getLogger(__name__)
59 dimensions=(
"tract",
"patch",
"band",
"skymap"),
60 defaultTemplates={
"inputCoaddName":
"deep",
61 "outputCoaddName":
"deep",
63 "warpTypeSuffix":
""}):
65 inputWarps = pipeBase.connectionTypes.Input(
66 doc=(
"Input list of warps to be assemebled i.e. stacked."
67 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
68 name=
"{inputCoaddName}Coadd_{warpType}Warp",
69 storageClass=
"ExposureF",
70 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
74 skyMap = pipeBase.connectionTypes.Input(
75 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
77 storageClass=
"SkyMap",
78 dimensions=(
"skymap", ),
80 selectedVisits = pipeBase.connectionTypes.Input(
81 doc=
"Selected visits to be coadded.",
82 name=
"{outputCoaddName}Visits",
83 storageClass=
"StructuredDataDict",
84 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band")
86 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
87 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
89 name=
"brightObjectMask",
90 storageClass=
"ObjectMaskCatalog",
91 dimensions=(
"tract",
"patch",
"skymap",
"band"),
94 coaddExposure = pipeBase.connectionTypes.Output(
95 doc=
"Output coadded exposure, produced by stacking input warps",
96 name=
"{outputCoaddName}Coadd{warpTypeSuffix}",
97 storageClass=
"ExposureF",
98 dimensions=(
"tract",
"patch",
"skymap",
"band"),
100 nImage = pipeBase.connectionTypes.Output(
101 doc=
"Output image of number of input images per pixel",
102 name=
"{outputCoaddName}Coadd_nImage",
103 storageClass=
"ImageU",
104 dimensions=(
"tract",
"patch",
"skymap",
"band"),
106 inputMap = pipeBase.connectionTypes.Output(
107 doc=
"Output healsparse map of input images",
108 name=
"{outputCoaddName}Coadd_inputMap",
109 storageClass=
"HealSparseMap",
110 dimensions=(
"tract",
"patch",
"skymap",
"band"),
113 def __init__(self, *, config=None):
114 super().__init__(config=config)
119 templateValues = {name: getattr(config.connections, name)
for name
in self.defaultTemplates}
120 templateValues[
'warpType'] = config.warpType
122 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
123 for name
in self.allConnections}
124 self._typeNameToVarName = {v: k
for k, v
in self._nameOverrides.items()}
127 if not config.doMaskBrightObjects:
128 self.prerequisiteInputs.remove(
"brightObjectMask")
130 if not config.doSelectVisits:
131 self.inputs.remove(
"selectedVisits")
133 if not config.doNImage:
134 self.outputs.remove(
"nImage")
136 if not self.config.doInputMap:
137 self.outputs.remove(
"inputMap")
140class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
141 pipelineConnections=AssembleCoaddConnections):
142 """Configuration parameters for the `AssembleCoaddTask`.
146 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
147 only set the bitplane config.brightObjectMaskName. To make this useful you
148 *must* also configure the flags.pixel algorithm,
for example by adding
152 config.measurement.plugins[
"base_PixelFlags"].masksFpCenter.append(
"BRIGHT_OBJECT")
153 config.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere.append(
"BRIGHT_OBJECT")
155 to your measureCoaddSources.py
and forcedPhotCoadd.py config overrides.
157 warpType = pexConfig.Field(
158 doc="Warp name: one of 'direct' or 'psfMatched'",
162 subregionSize = pexConfig.ListField(
164 doc=
"Width, height of stack subregion size; "
165 "make small enough that a full stack of images will fit into memory at once.",
167 default=(2000, 2000),
169 statistic = pexConfig.Field(
171 doc=
"Main stacking statistic for aggregating over the epochs.",
174 doOnlineForMean = pexConfig.Field(
176 doc=
"Perform online coaddition when statistic=\"MEAN\" to save memory?",
179 doSigmaClip = pexConfig.Field(
181 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
184 sigmaClip = pexConfig.Field(
186 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
189 clipIter = pexConfig.Field(
191 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
194 calcErrorFromInputVariance = pexConfig.Field(
196 doc=
"Calculate coadd variance from input variance by stacking statistic."
197 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
200 scaleZeroPoint = pexConfig.ConfigurableField(
201 target=ScaleZeroPointTask,
202 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
204 doInterp = pexConfig.Field(
205 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
209 interpImage = pexConfig.ConfigurableField(
210 target=InterpImageTask,
211 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
213 doWrite = pexConfig.Field(
214 doc=
"Persist coadd?",
218 doNImage = pexConfig.Field(
219 doc=
"Create image of number of contributing exposures for each pixel",
223 doUsePsfMatchedPolygons = pexConfig.Field(
224 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
228 maskPropagationThresholds = pexConfig.DictField(
231 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
232 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
233 "would have contributed exceeds this value."),
234 default={
"SAT": 0.1},
236 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
237 doc=
"Mask planes to remove before coadding")
238 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
239 doc=
"Set mask and flag bits for bright objects?")
240 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
241 doc=
"Name of mask bit used for bright objects")
242 coaddPsf = pexConfig.ConfigField(
243 doc=
"Configuration for CoaddPsf",
244 dtype=measAlg.CoaddPsfConfig,
246 doAttachTransmissionCurve = pexConfig.Field(
247 dtype=bool, default=
False, optional=
False,
248 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
249 "(requires all input Exposures to have TransmissionCurves).")
251 hasFakes = pexConfig.Field(
254 doc=
"Should be set to True if fake sources have been inserted into the input data."
256 doSelectVisits = pexConfig.Field(
257 doc=
"Coadd only visits selected by a SelectVisitsTask",
261 doInputMap = pexConfig.Field(
262 doc=
"Create a bitwise map of coadd inputs",
266 inputMapper = pexConfig.ConfigurableField(
267 doc=
"Input map creation subtask.",
268 target=HealSparseInputMapTask,
271 def setDefaults(self):
272 super().setDefaults()
273 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
280 log.warning(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
281 self.warpType =
'psfMatched'
282 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
283 log.warning(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
284 self.statistic =
"MEANCLIP"
285 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
286 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
287 "compute and set a non-zero coadd variance estimate." % (self.statistic))
289 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
290 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
291 stackableStats = [
str(k)
for k
in afwMath.Property.__members__.keys()
292 if str(k)
not in unstackableStats]
293 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
294 % (self.statistic, stackableStats))
297class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
298 """Assemble a coadded image from a set of warps (coadded temporary exposures).
300 We want to assemble a coadded image from a set of Warps (also called
301 coadded temporary exposures
or ``coaddTempExps``).
302 Each input Warp covers a patch on the sky
and corresponds to a single
303 run/visit/exposure of the covered patch. We provide the task
with a list
304 of Warps (``selectDataList``)
from which it selects Warps that cover the
305 specified patch (pointed at by ``dataRef``).
306 Each Warp that goes into a coadd will typically have an independent
307 photometric zero-point. Therefore, we must scale each Warp to set it to
308 a common photometric zeropoint. WarpType may be one of
'direct' or
309 'psfMatched',
and the boolean configs `config.makeDirect`
and
310 `config.makePsfMatched` set which of the warp types will be coadded.
311 The coadd
is computed
as a mean
with optional outlier rejection.
312 Criteria
for outlier rejection are set
in `AssembleCoaddConfig`.
313 Finally, Warps can have bad
'NaN' pixels which received no input
from the
314 source calExps. We interpolate over these bad (NaN) pixels.
316 `AssembleCoaddTask` uses several sub-tasks. These are
318 - `ScaleZeroPointTask`
319 - create
and use an ``imageScaler`` object to scale the photometric zeropoint
for each Warp
321 - interpolate across bad pixels (NaN)
in the final coadd
323 You can retarget these subtasks
if you wish.
327 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
328 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``; see
329 `baseDebug`
for more about ``debug.py`` files. `AssembleCoaddTask` has
330 no debug variables of its own. Some of the subtasks may support debug
331 variables. See the documentation
for the subtasks
for further information.
335 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
336 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
337 with the flag
'--legacyCoadd'. Usage of assembleCoadd.py expects two
338 inputs: a data reference to the tract patch
and filter to be coadded,
and
339 a list of Warps to attempt to coadd. These are specified using ``--id``
and
340 ``--selectId``, respectively:
344 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
345 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
347 Only the Warps that cover the specified tract
and patch will be coadded.
348 A list of the available optional arguments can be obtained by calling
349 ``assembleCoadd.py``
with the ``--help`` command line argument:
353 assembleCoadd.py --help
355 To demonstrate usage of the `AssembleCoaddTask`
in the larger context of
356 multi-band processing, we will generate the HSC-I & -R band coadds
from
357 HSC engineering test data provided
in the ``ci_hsc`` package. To begin,
358 assuming that the lsst stack has been already set up, we must set up the
359 obs_subaru
and ``ci_hsc`` packages. This defines the environment variable
360 ``$CI_HSC_DIR``
and points at the location of the package. The raw HSC
361 data live
in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
362 coadds, we must first
365 - process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
367 - create a skymap that covers the area of the sky present
in the raw exposures
369 - warp the individual calibrated exposures to the tangent plane of the coadd
371 We can perform all of these steps by running
375 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
377 This will produce warped exposures
for each visit. To coadd the warped
378 data, we call assembleCoadd.py
as follows:
382 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
383 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
384 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
385 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
386 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
387 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
388 --selectId visit=903988 ccd=24
390 that will process the HSC-I band data. The results are written
in
391 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
393 You may also choose to run:
397 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
398 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
399 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
400 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
401 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
402 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
403 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
404 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
406 to generate the coadd
for the HSC-R band
if you are interested
in
407 following multiBand Coadd processing
as discussed
in `pipeTasks_multiBand`
408 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
409 rather than `AssembleCoaddTask` to make the coadd.
411 ConfigClass = AssembleCoaddConfig
412 _DefaultName = "assembleCoadd"
414 def __init__(self, *args, **kwargs):
417 argNames = [
"config",
"name",
"parentTask",
"log"]
418 kwargs.update({k: v
for k, v
in zip(argNames, args)})
419 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
420 "PipelineTask will not take positional args" % argNames, FutureWarning)
422 super().__init__(**kwargs)
423 self.makeSubtask(
"interpImage")
424 self.makeSubtask(
"scaleZeroPoint")
426 if self.config.doMaskBrightObjects:
427 mask = afwImage.Mask()
429 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
430 except pexExceptions.LsstCppException:
431 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
432 mask.getMaskPlaneDict().keys())
435 if self.config.doInputMap:
436 self.makeSubtask(
"inputMapper")
438 self.warpType = self.config.warpType
440 @utils.inheritDoc(pipeBase.PipelineTask)
441 def runQuantum(self, butlerQC, inputRefs, outputRefs):
446 Assemble a coadd from a set of Warps.
448 PipelineTask (Gen3) entry point to Coadd a set of Warps.
449 Analogous to `runDataRef`, it prepares all the data products to be
450 passed to `run`,
and processes the results before returning a struct
451 of results to be written out. AssembleCoadd cannot fit all Warps
in memory.
452 Therefore, its inputs are accessed subregion by subregion
453 by the Gen3 `DeferredDatasetHandle` that
is analagous to the Gen2
454 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
455 correspond to an update
in `runDataRef`
while both entry points
458 inputData = butlerQC.get(inputRefs)
462 skyMap = inputData[
"skyMap"]
463 outputDataId = butlerQC.quantum.dataId
466 tractId=outputDataId[
'tract'],
467 patchId=outputDataId[
'patch'])
469 if self.config.doSelectVisits:
470 warpRefList = self.filterWarps(inputData[
'inputWarps'], inputData[
'selectedVisits'])
472 warpRefList = inputData[
'inputWarps']
475 inputs = self.prepareInputs(warpRefList)
476 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
477 self.getTempExpDatasetName(self.warpType))
478 if len(inputs.tempExpRefList) == 0:
479 raise pipeBase.NoWorkFound(
"No coadd temporary exposures found")
481 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
482 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
483 inputs.weightList, supplementaryData=supplementaryData)
485 inputData.setdefault(
'brightObjectMask',
None)
486 if self.config.doMaskBrightObjects
and inputData[
"brightObjectMask"]
is None:
487 log.warning(
"doMaskBrightObjects is set to True, but brightObjectMask not loaded")
488 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
490 if self.config.doWrite:
491 butlerQC.put(retStruct, outputRefs)
495 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
496 """Assemble a coadd from a set of Warps.
498 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
499 Compute weights to be applied to each Warp and
500 find scalings to match the photometric zeropoint to a reference Warp.
501 Assemble the Warps using `run`. Interpolate over NaNs
and
502 optionally write the coadd to disk. Return the coadded exposure.
507 Data reference defining the patch
for coaddition
and the
508 reference Warp (
if ``config.autoReference=
False``).
509 Used to access the following data products:
510 - ``self.config.coaddName +
"Coadd_skyMap"``
511 - ``self.config.coaddName +
"Coadd_ + <warpType> + "Warp
"`` (optionally)
512 - ``self.config.coaddName + "Coadd"``
513 selectDataList : `list`
514 List of data references to Calexps. Data to be coadded will be
515 selected
from this list based on overlap
with the patch defined
516 by dataRef, grouped by visit,
and converted to a list of data
519 List of data references to Warps to be coadded.
520 Note: `warpRefList`
is just the new name
for `tempExpRefList`.
524 retStruct : `lsst.pipe.base.Struct`
525 Result struct
with components:
527 - ``coaddExposure``: coadded exposure (``Exposure``).
528 - ``nImage``: exposure count image (``Image``).
530 if selectDataList
and warpRefList:
531 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, "
532 "and which to use is ambiguous. Please pass only one.")
534 skyInfo = self.getSkyInfo(dataRef)
535 if warpRefList
is None:
536 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
537 if len(calExpRefList) == 0:
538 self.log.warning(
"No exposures to coadd")
540 self.log.info(
"Coadding %d exposures", len(calExpRefList))
542 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
544 inputData = self.prepareInputs(warpRefList)
545 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
546 self.getTempExpDatasetName(self.warpType))
547 if len(inputData.tempExpRefList) == 0:
548 self.log.warning(
"No coadd temporary exposures found")
551 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
553 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
554 inputData.weightList, supplementaryData=supplementaryData)
556 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None
557 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
559 if self.config.doWrite:
560 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
561 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
563 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
564 self.log.info(
"Persisting %s", coaddDatasetName)
565 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
566 if self.config.doNImage
and retStruct.nImage
is not None:
567 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
572 """Interpolate over missing data and mask bright stars.
577 The coadded exposure to process.
578 dataRef : `lsst.daf.persistence.ButlerDataRef`
579 Butler data reference for supplementary data.
581 if self.config.doInterp:
582 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
584 varArray = coaddExposure.variance.array
585 with numpy.errstate(invalid=
"ignore"):
586 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
588 if self.config.doMaskBrightObjects:
589 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
592 """Make additional inputs to run() specific to subclasses (Gen2)
594 Duplicates interface of `runDataRef` method
595 Available to be implemented by subclasses only if they need the
596 coadd dataRef
for performing preliminary processing before
597 assembling the coadd.
601 dataRef : `lsst.daf.persistence.ButlerDataRef`
602 Butler data reference
for supplementary data.
603 selectDataList : `list` (optional)
604 Optional List of data references to Calexps.
605 warpRefList : `list` (optional)
606 Optional List of data references to Warps.
608 return pipeBase.Struct()
611 """Make additional inputs to run() specific to subclasses (Gen3)
613 Duplicates interface of `runQuantum` method.
614 Available to be implemented by subclasses only if they need the
615 coadd dataRef
for performing preliminary processing before
616 assembling the coadd.
620 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
621 Gen3 Butler object
for fetching additional data products before
622 running the Task specialized
for quantum being processed
623 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
624 Attributes are the names of the connections describing input dataset types.
625 Values are DatasetRefs that task consumes
for corresponding dataset type.
626 DataIds are guaranteed to match data objects
in ``inputData``.
627 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
628 Attributes are the names of the connections describing output dataset types.
629 Values are DatasetRefs that task
is to produce
630 for corresponding dataset type.
632 return pipeBase.Struct()
635 """Generate list data references corresponding to warped exposures
636 that lie within the patch to be coadded.
641 Data reference for patch.
642 calExpRefList : `list`
643 List of data references
for input calexps.
647 tempExpRefList : `list`
648 List of Warp/CoaddTempExp data references.
650 butler = patchRef.getButler()
651 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
652 self.getTempExpDatasetName(self.warpType))
653 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
654 g, groupData.keys) for
655 g
in groupData.groups.keys()]
656 return tempExpRefList
659 """Prepare the input warps for coaddition by measuring the weight for
660 each warp and the scaling
for the photometric zero point.
662 Each Warp has its own photometric zeropoint
and background variance.
663 Before coadding these Warps together, compute a scale factor to
664 normalize the photometric zeropoint
and compute the weight
for each Warp.
669 List of data references to tempExp
673 result : `lsst.pipe.base.Struct`
674 Result struct
with components:
676 - ``tempExprefList``: `list` of data references to tempExp.
677 - ``weightList``: `list` of weightings.
678 - ``imageScalerList``: `list` of image scalers.
680 statsCtrl = afwMath.StatisticsControl()
681 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
682 statsCtrl.setNumIter(self.config.clipIter)
683 statsCtrl.setAndMask(self.getBadPixelMask())
684 statsCtrl.setNanSafe(True)
691 tempExpName = self.getTempExpDatasetName(self.warpType)
692 for tempExpRef
in refList:
695 if not isinstance(tempExpRef, DeferredDatasetHandle):
696 if not tempExpRef.datasetExists(tempExpName):
697 self.log.warning(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
700 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
702 if numpy.isnan(tempExp.image.array).all():
704 maskedImage = tempExp.getMaskedImage()
705 imageScaler = self.scaleZeroPoint.computeImageScaler(
710 imageScaler.scaleMaskedImage(maskedImage)
711 except Exception
as e:
712 self.log.warning(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
714 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
715 afwMath.MEANCLIP, statsCtrl)
716 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
717 weight = 1.0 / float(meanVar)
718 if not numpy.isfinite(weight):
719 self.log.warning(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
721 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
726 tempExpRefList.append(tempExpRef)
727 weightList.append(weight)
728 imageScalerList.append(imageScaler)
730 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
731 imageScalerList=imageScalerList)
734 """Prepare the statistics for coadding images.
738 mask : `int`, optional
739 Bit mask value to exclude from coaddition.
743 stats : `lsst.pipe.base.Struct`
744 Statistics structure
with the following fields:
746 - ``statsCtrl``: Statistics control object
for coadd
748 - ``statsFlags``: Statistic
for coadd (`lsst.afw.math.Property`)
751 mask = self.getBadPixelMask()
752 statsCtrl = afwMath.StatisticsControl()
753 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
754 statsCtrl.setNumIter(self.config.clipIter)
755 statsCtrl.setAndMask(mask)
756 statsCtrl.setNanSafe(
True)
757 statsCtrl.setWeighted(
True)
758 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
759 for plane, threshold
in self.config.maskPropagationThresholds.items():
760 bit = afwImage.Mask.getMaskPlane(plane)
761 statsCtrl.setMaskPropagationThreshold(bit, threshold)
762 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
763 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
766 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
767 altMaskList=None, mask=None, supplementaryData=None):
768 """Assemble a coadd from input warps
770 Assemble the coadd using the provided list of coaddTempExps. Since
771 the full coadd covers a patch (a large area), the assembly is
772 performed over small areas on the image at a time
in order to
773 conserve memory usage. Iterate over subregions within the outer
774 bbox of the patch using `assembleSubregion` to stack the corresponding
775 subregions
from the coaddTempExps
with the statistic specified.
776 Set the edge bits the coadd mask based on the weight map.
780 skyInfo : `lsst.pipe.base.Struct`
781 Struct
with geometric information about the patch.
782 tempExpRefList : `list`
783 List of data references to Warps (previously called CoaddTempExps).
784 imageScalerList : `list`
785 List of image scalers.
788 altMaskList : `list`, optional
789 List of alternate masks to use rather than those stored
with
791 mask : `int`, optional
792 Bit mask value to exclude
from coaddition.
793 supplementaryData : lsst.pipe.base.Struct, optional
794 Struct
with additional data products needed to assemble coadd.
795 Only used by subclasses that implement `makeSupplementaryData`
800 result : `lsst.pipe.base.Struct`
801 Result struct
with components:
805 - ``inputMap``: bit-wise map of inputs,
if requested.
806 - ``warpRefList``: input list of refs to the warps (
807 ``lsst.daf.butler.DeferredDatasetHandle``
or
808 ``lsst.daf.persistence.ButlerDataRef``)
810 - ``imageScalerList``: input list of image scalers (unmodified)
811 - ``weightList``: input list of weights (unmodified)
813 tempExpName = self.getTempExpDatasetName(self.warpType)
814 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
815 stats = self.prepareStats(mask=mask)
817 if altMaskList
is None:
818 altMaskList = [
None]*len(tempExpRefList)
820 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
821 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
822 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
823 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
824 coaddMaskedImage = coaddExposure.getMaskedImage()
825 subregionSizeArr = self.config.subregionSize
826 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
828 if self.config.doNImage:
829 nImage = afwImage.ImageU(skyInfo.bbox)
834 if self.config.doInputMap:
835 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
837 coaddExposure.getInfo().getCoaddInputs().ccds)
839 if self.config.doOnlineForMean
and self.config.statistic ==
"MEAN":
841 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
842 weightList, altMaskList, stats.ctrl,
844 except Exception
as e:
845 self.log.exception(
"Cannot compute online coadd %s", e)
848 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
850 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
851 weightList, altMaskList, stats.flags, stats.ctrl,
853 except Exception
as e:
854 self.log.exception(
"Cannot compute coadd %s: %s", subBBox, e)
858 if self.config.doInputMap:
859 self.inputMapper.finalize_ccd_input_map_mask()
860 inputMap = self.inputMapper.ccd_input_map
864 self.setInexactPsf(coaddMaskedImage.getMask())
867 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
868 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
869 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
870 weightList=weightList, inputMap=inputMap)
873 """Set the metadata for the coadd.
875 This basic implementation sets the filter from the first input.
880 The target exposure
for the coadd.
881 tempExpRefList : `list`
882 List of data references to tempExp.
886 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
887 tempExpName = self.getTempExpDatasetName(self.warpType)
893 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
895 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
898 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
899 for tempExpRef
in tempExpRefList]
900 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
904 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
905 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
906 coaddInputs.ccds.reserve(numCcds)
907 coaddInputs.visits.reserve(len(tempExpList))
909 for tempExp, weight
in zip(tempExpList, weightList):
910 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
912 if self.config.doUsePsfMatchedPolygons:
913 self.shrinkValidPolygons(coaddInputs)
915 coaddInputs.visits.sort()
916 coaddInputs.ccds.sort()
917 if self.warpType ==
"psfMatched":
922 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
923 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
924 for modelPsf
in modelPsfList]
925 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
927 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
928 self.config.coaddPsf.makeControl())
929 coaddExposure.setPsf(psf)
930 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
931 coaddExposure.getWcs())
932 coaddExposure.getInfo().setApCorrMap(apCorrMap)
933 if self.config.doAttachTransmissionCurve:
934 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
935 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
938 altMaskList, statsFlags, statsCtrl, nImage=None):
939 """Assemble the coadd for a sub-region.
941 For each coaddTempExp, check for (
and swap
in) an alternative mask
942 if one
is passed. Remove mask planes listed
in
943 `config.removeMaskPlanes`. Finally, stack the actual exposures using
944 `lsst.afw.math.statisticsStack`
with the statistic specified by
945 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
for
946 a mean-stack
or `lsst.afw.math.MEANCLIP`
for outlier rejection using
947 an N-sigma clipped mean where N
and iterations are specified by
948 statsCtrl. Assign the stacked subregion back to the coadd.
953 The target exposure
for the coadd.
954 bbox : `lsst.geom.Box`
956 tempExpRefList : `list`
957 List of data reference to tempExp.
958 imageScalerList : `list`
959 List of image scalers.
963 List of alternate masks to use rather than those stored
with
964 tempExp,
or None. Each element
is dict
with keys = mask plane
965 name to which to add the spans.
966 statsFlags : `lsst.afw.math.Property`
967 Property object
for statistic
for coadd.
969 Statistics control object
for coadd.
970 nImage : `lsst.afw.image.ImageU`, optional
971 Keeps track of exposure count
for each pixel.
973 self.log.debug("Computing coadd over %s", bbox)
974 tempExpName = self.getTempExpDatasetName(self.warpType)
975 coaddExposure.mask.addMaskPlane(
"REJECTED")
976 coaddExposure.mask.addMaskPlane(
"CLIPPED")
977 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
978 maskMap = self.setRejectedMaskMapping(statsCtrl)
979 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
981 if nImage
is not None:
982 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
983 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
985 if isinstance(tempExpRef, DeferredDatasetHandle):
987 exposure = tempExpRef.get(parameters={
'bbox': bbox})
990 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
992 maskedImage = exposure.getMaskedImage()
993 mask = maskedImage.getMask()
994 if altMask
is not None:
995 self.applyAltMaskPlanes(mask, altMask)
996 imageScaler.scaleMaskedImage(maskedImage)
1000 if nImage
is not None:
1001 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
1002 if self.config.removeMaskPlanes:
1003 self.removeMaskPlanes(maskedImage)
1004 maskedImageList.append(maskedImage)
1006 if self.config.doInputMap:
1007 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1008 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1010 with self.timer(
"stack"):
1011 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
1014 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
1015 if nImage
is not None:
1016 nImage.assign(subNImage, bbox)
1019 altMaskList, statsCtrl, nImage=None):
1020 """Assemble the coadd using the "online" method.
1022 This method takes a running sum of images and weights to save memory.
1023 It only works
for MEAN statistics.
1028 The target exposure
for the coadd.
1029 tempExpRefList : `list`
1030 List of data reference to tempExp.
1031 imageScalerList : `list`
1032 List of image scalers.
1035 altMaskList : `list`
1036 List of alternate masks to use rather than those stored
with
1037 tempExp,
or None. Each element
is dict
with keys = mask plane
1038 name to which to add the spans.
1040 Statistics control object
for coadd
1041 nImage : `lsst.afw.image.ImageU`, optional
1042 Keeps track of exposure count
for each pixel.
1044 self.log.debug("Computing online coadd.")
1045 tempExpName = self.getTempExpDatasetName(self.warpType)
1046 coaddExposure.mask.addMaskPlane(
"REJECTED")
1047 coaddExposure.mask.addMaskPlane(
"CLIPPED")
1048 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
1049 maskMap = self.setRejectedMaskMapping(statsCtrl)
1050 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
1052 bbox = coaddExposure.maskedImage.getBBox()
1054 stacker = AccumulatorMeanStack(
1055 coaddExposure.image.array.shape,
1056 statsCtrl.getAndMask(),
1057 mask_threshold_dict=thresholdDict,
1059 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
1060 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
1061 compute_n_image=(nImage
is not None)
1064 for tempExpRef, imageScaler, altMask, weight
in zip(tempExpRefList,
1068 if isinstance(tempExpRef, DeferredDatasetHandle):
1070 exposure = tempExpRef.get()
1073 exposure = tempExpRef.get(tempExpName)
1075 maskedImage = exposure.getMaskedImage()
1076 mask = maskedImage.getMask()
1077 if altMask
is not None:
1078 self.applyAltMaskPlanes(mask, altMask)
1079 imageScaler.scaleMaskedImage(maskedImage)
1080 if self.config.removeMaskPlanes:
1081 self.removeMaskPlanes(maskedImage)
1083 stacker.add_masked_image(maskedImage, weight=weight)
1085 if self.config.doInputMap:
1086 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1087 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1089 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
1091 if nImage
is not None:
1092 nImage.array[:, :] = stacker.n_image
1095 """Unset the mask of an image for mask planes specified in the config.
1100 The masked image to be modified.
1102 mask = maskedImage.getMask()
1103 for maskPlane
in self.config.removeMaskPlanes:
1105 mask &= ~mask.getPlaneBitMask(maskPlane)
1106 except pexExceptions.InvalidParameterError:
1107 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
1111 def setRejectedMaskMapping(statsCtrl):
1112 """Map certain mask planes of the warps to new planes for the coadd.
1114 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1115 or CLIPPED, set it to REJECTED on the coadd.
1116 If a pixel
is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1117 If a pixel
is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1122 Statistics control object
for coadd
1126 maskMap : `list` of `tuple` of `int`
1127 A list of mappings of mask planes of the warped exposures to
1128 mask planes of the coadd.
1130 edge = afwImage.Mask.getPlaneBitMask("EDGE")
1131 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
1132 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
1133 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1134 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
1135 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
1140 """Apply in place alt mask formatted as SpanSets to a mask.
1146 altMaskSpans : `dict`
1147 SpanSet lists to apply. Each element contains the new mask
1148 plane name (e.g. "CLIPPED and/or "NO_DATA
") as the key,
1149 and list of SpanSets to apply to the mask.
1156 if self.config.doUsePsfMatchedPolygons:
1157 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1162 for spanSet
in altMaskSpans[
'NO_DATA']:
1163 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1165 for plane, spanSetList
in altMaskSpans.items():
1166 maskClipValue = mask.addMaskPlane(plane)
1167 for spanSet
in spanSetList:
1168 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1172 """Shrink coaddInputs' ccds' ValidPolygons in place.
1174 Either modify each ccd's validPolygon in place, or if CoaddInputs
1175 does not have a validPolygon, create one
from its bbox.
1179 coaddInputs : `lsst.afw.image.coaddInputs`
1183 for ccd
in coaddInputs.ccds:
1184 polyOrig = ccd.getValidPolygon()
1185 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1186 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1188 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1190 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1191 ccd.setValidPolygon(validPolygon)
1194 """Retrieve the bright object masks.
1196 Returns None on failure.
1206 Bright object mask
from the Butler object,
or None if it cannot
1210 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1211 except Exception
as e:
1212 self.log.warning(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1216 """Set the bright object masks.
1221 Exposure under consideration.
1223 Data identifier dict for patch.
1225 Table of bright objects to mask.
1228 if brightObjectMasks
is None:
1229 self.log.warning(
"Unable to apply bright object mask: none supplied")
1231 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1232 mask = exposure.getMaskedImage().getMask()
1233 wcs = exposure.getWcs()
1234 plateScale = wcs.getPixelScale().asArcseconds()
1236 for rec
in brightObjectMasks:
1237 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1238 if rec[
"type"] ==
"box":
1239 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1240 width = rec[
"width"].asArcseconds()/plateScale
1241 height = rec[
"height"].asArcseconds()/plateScale
1244 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1247 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1248 spans = afwGeom.SpanSet(bbox)
1249 elif rec[
"type"] ==
"circle":
1250 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1251 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1253 self.log.warning(
"Unexpected region type %s at %s", rec[
"type"], center)
1255 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1258 """Set INEXACT_PSF mask plane.
1260 If any of the input images isn't represented in the coadd (due to
1261 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1267 Coadded exposure
's mask, modified in-place.
1269 mask.addMaskPlane("INEXACT_PSF")
1270 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1271 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1272 clipped = mask.getPlaneBitMask(
"CLIPPED")
1273 rejected = mask.getPlaneBitMask(
"REJECTED")
1274 array = mask.getArray()
1275 selected = array & (sensorEdge | clipped | rejected) > 0
1276 array[selected] |= inexactPsf
1279 def _makeArgumentParser(cls):
1280 """Create an argument parser.
1282 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1283 parser.add_id_argument("--id", cls.ConfigClass().coaddName +
"Coadd_"
1284 + cls.ConfigClass().warpType +
"Warp",
1285 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1286 ContainerClass=AssembleCoaddDataIdContainer)
1287 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1288 ContainerClass=SelectDataIdContainer)
1292 def _subBBoxIter(bbox, subregionSize):
1293 """Iterate over subregions of a bbox.
1298 Bounding box over which to iterate.
1305 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1306 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1307 the edges of ``bbox``, but it will never be empty.
1310 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1311 if subregionSize[0] < 1
or subregionSize[1] < 1:
1312 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1314 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1315 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1318 if subBBox.isEmpty():
1319 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1320 "colShift=%s, rowShift=%s" %
1321 (bbox, subregionSize, colShift, rowShift))
1325 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1330 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1332 Dictionary
with good visitIds
as the keys. Value ignored.
1336 filteredInputs : `list`
1337 Filtered
and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1339 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef
for inputRef
in inputs}
1341 for visit
in goodVisits.keys():
1342 if visit
in inputWarpDict:
1343 filteredInputs.append(inputWarpDict[visit])
1344 return filteredInputs
1348 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1352 """Make self.refList from self.idList.
1357 Results of parsing command-line (with ``butler``
and ``log`` elements).
1359 datasetType = namespace.config.coaddName + "Coadd"
1360 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1362 for dataId
in self.idList:
1364 for key
in keysCoadd:
1365 if key
not in dataId:
1366 raise RuntimeError(
"--id must include " + key)
1368 dataRef = namespace.butler.dataRef(
1369 datasetType=datasetType,
1372 self.refList.append(dataRef)
1376 """Function to count the number of pixels with a specific mask in a
1379 Find the intersection of mask & footprint. Count all pixels in the mask
1380 that are
in the intersection that have bitmask set but do
not have
1381 ignoreMask set. Return the count.
1386 Mask to define intersection region by.
1388 Footprint to define the intersection region by.
1390 Specific mask that we wish to count the number of occurances of.
1392 Pixels to
not consider.
1397 Count of number of pixels
in footprint
with specified mask.
1399 bbox = footprint.getBBox()
1400 bbox.clip(mask.getBBox(afwImage.PARENT))
1401 fp = afwImage.Mask(bbox)
1402 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1403 footprint.spans.setMask(fp, bitmask)
1404 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1405 (subMask.getArray() & ignoreMask) == 0).sum()
1409 """Configuration parameters for the SafeClipAssembleCoaddTask.
1411 clipDetection = pexConfig.ConfigurableField(
1412 target=SourceDetectionTask,
1413 doc="Detect sources on difference between unclipped and clipped coadd")
1414 minClipFootOverlap = pexConfig.Field(
1415 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1419 minClipFootOverlapSingle = pexConfig.Field(
1420 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1421 "clipped when only one visit overlaps",
1425 minClipFootOverlapDouble = pexConfig.Field(
1426 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1427 "clipped when two visits overlap",
1431 maxClipFootOverlapDouble = pexConfig.Field(
1432 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
1433 "considering two visits",
1437 minBigOverlap = pexConfig.Field(
1438 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1439 "when labeling clipped footprints",
1445 """Set default values for clipDetection.
1449 The numeric values for these configuration parameters were
1450 empirically determined, future work may further refine them.
1452 AssembleCoaddConfig.setDefaults(self)
1468 log.warning(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1469 "Ignoring doSigmaClip.")
1472 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1473 "(%s chosen). Please set statistic to MEAN."
1475 AssembleCoaddTask.ConfigClass.validate(self)
1479 """Assemble a coadded image from a set of coadded temporary exposures,
1480 being careful to clip & flag areas with potential artifacts.
1482 In ``AssembleCoaddTask``, we compute the coadd
as an clipped mean (i.e.,
1483 we clip outliers). The problem
with doing this
is that when computing the
1484 coadd PSF at a given location, individual visit PSFs
from visits
with
1485 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
1486 In this task, we correct
for this behavior by creating a new
1487 ``badMaskPlane``
'CLIPPED'. We populate this plane on the input
1488 coaddTempExps
and the final coadd where
1490 i. difference imaging suggests that there
is an outlier
and
1491 ii. this outlier appears on only one
or two images.
1493 Such regions will
not contribute to the final coadd. Furthermore, any
1494 routine to determine the coadd PSF can now be cognizant of clipped regions.
1495 Note that the algorithm implemented by this task
is preliminary
and works
1496 correctly
for HSC data. Parameter modifications
and or considerable
1497 redesigning of the algorithm
is likley required
for other surveys.
1499 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1500 "clipDetection" subtask
and also sub-classes ``AssembleCoaddTask``.
1501 You can retarget the ``SourceDetectionTask``
"clipDetection" subtask
1506 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1507 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``;
1508 see `baseDebug`
for more about ``debug.py`` files.
1509 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1510 The ``SourceDetectionTask``
"clipDetection" subtasks may support debug
1511 variables. See the documetation
for `SourceDetectionTask`
"clipDetection"
1512 for further information.
1516 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1517 images into a coadded image. The `SafeClipAssembleCoaddTask`
is invoked by
1518 running assembleCoadd.py *without* the flag
'--legacyCoadd'.
1520 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1521 and filter to be coadded (specified using
1522 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1523 along
with a list of coaddTempExps to attempt to coadd (specified using
1524 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1525 Only the coaddTempExps that cover the specified tract
and patch will be
1526 coadded. A list of the available optional arguments can be obtained by
1527 calling assembleCoadd.py
with the --help command line argument:
1529 .. code-block:: none
1531 assembleCoadd.py --help
1533 To demonstrate usage of the `SafeClipAssembleCoaddTask`
in the larger
1534 context of multi-band processing, we will generate the HSC-I & -R band
1535 coadds
from HSC engineering test data provided
in the ci_hsc package.
1536 To begin, assuming that the lsst stack has been already set up, we must
1537 set up the obs_subaru
and ci_hsc packages. This defines the environment
1538 variable $CI_HSC_DIR
and points at the location of the package. The raw
1539 HSC data live
in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1540 the coadds, we must first
1543 process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
1545 create a skymap that covers the area of the sky present
in the raw exposures
1546 - ``makeCoaddTempExp``
1547 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1549 We can perform all of these steps by running
1551 .. code-block:: none
1553 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1555 This will produce warped coaddTempExps
for each visit. To coadd the
1556 warped data, we call ``assembleCoadd.py``
as follows:
1558 .. code-block:: none
1560 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1561 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1562 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1563 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1564 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1565 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1566 --selectId visit=903988 ccd=24
1568 This will process the HSC-I band data. The results are written
in
1569 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1571 You may also choose to run:
1573 .. code-block:: none
1575 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1576 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1577 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1578 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1579 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1580 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1581 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1582 --selectId visit=903346 ccd=12
1584 to generate the coadd
for the HSC-R band
if you are interested
in following
1585 multiBand Coadd processing
as discussed
in ``pipeTasks_multiBand``.
1587 ConfigClass = SafeClipAssembleCoaddConfig
1588 _DefaultName = "safeClipAssembleCoadd"
1591 AssembleCoaddTask.__init__(self, *args, **kwargs)
1592 schema = afwTable.SourceTable.makeMinimalSchema()
1593 self.makeSubtask(
"clipDetection", schema=schema)
1595 @utils.inheritDoc(AssembleCoaddTask)
1597 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1598 """Assemble the coadd for a region.
1600 Compute the difference of coadds created with and without outlier
1601 rejection to identify coadd pixels that have outlier values
in some
1603 Detect clipped regions on the difference image
and mark these regions
1604 on the one
or two individual coaddTempExps where they occur
if there
1605 is significant overlap between the clipped region
and a source. This
1606 leaves us
with a set of footprints
from the difference image that have
1607 been identified
as having occured on just one
or two individual visits.
1608 However, these footprints were generated
from a difference image. It
1609 is conceivable
for a large diffuse source to have become broken up
1610 into multiple footprints acrosss the coadd difference
in this process.
1611 Determine the clipped region
from all overlapping footprints
from the
1612 detected sources
in each visit - these are big footprints.
1613 Combine the small
and big clipped footprints
and mark them on a new
1615 Generate the coadd using `AssembleCoaddTask.run` without outlier
1616 removal. Clipped footprints will no longer make it into the coadd
1617 because they are marked
in the new bad mask plane.
1621 args
and kwargs are passed but ignored
in order to match the call
1622 signature expected by the parent task.
1625 mask = exp.getMaskedImage().getMask()
1626 mask.addMaskPlane("CLIPPED")
1628 result = self.
detectClip(exp, tempExpRefList)
1630 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1632 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1633 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1635 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1636 result.detectionFootprints, maskClipValue, maskDetValue,
1639 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1640 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1642 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1643 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1644 maskClip |= maskClipBig
1647 badMaskPlanes = self.config.badMaskPlanes[:]
1648 badMaskPlanes.append(
"CLIPPED")
1649 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1650 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1651 result.clipSpans, mask=badPixelMask)
1654 """Return an exposure that contains the difference between unclipped
1657 Generate a difference image between clipped
and unclipped coadds.
1658 Compute the difference image by subtracting an outlier-clipped coadd
1659 from an outlier-unclipped coadd. Return the difference image.
1663 skyInfo : `lsst.pipe.base.Struct`
1664 Patch geometry information,
from getSkyInfo
1665 tempExpRefList : `list`
1666 List of data reference to tempExp
1667 imageScalerList : `list`
1668 List of image scalers
1675 Difference image of unclipped
and clipped coadd wrapped
in an Exposure
1677 config = AssembleCoaddConfig()
1682 configIntersection = {k: getattr(self.config, k)
1683 for k, v
in self.config.toDict().items()
1684 if (k
in config.keys()
and k !=
"connections")}
1685 configIntersection[
'doInputMap'] =
False
1686 configIntersection[
'doNImage'] =
False
1687 config.update(**configIntersection)
1690 config.statistic =
'MEAN'
1691 task = AssembleCoaddTask(config=config)
1692 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1694 config.statistic =
'MEANCLIP'
1695 task = AssembleCoaddTask(config=config)
1696 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1698 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1699 coaddDiff -= coaddClip.getMaskedImage()
1700 exp = afwImage.ExposureF(coaddDiff)
1701 exp.setPsf(coaddMean.getPsf())
1705 """Detect clipped regions on an exposure and set the mask on the
1706 individual tempExp masks.
1708 Detect footprints in the difference image after smoothing the
1709 difference image
with a Gaussian kernal. Identify footprints that
1710 overlap
with one
or two input ``coaddTempExps`` by comparing the
1711 computed overlap fraction to thresholds set
in the config. A different
1712 threshold
is applied depending on the number of overlapping visits
1713 (restricted to one
or two). If the overlap exceeds the thresholds,
1714 the footprint
is considered
"CLIPPED" and is marked
as such on the
1715 coaddTempExp. Return a struct
with the clipped footprints, the indices
1716 of the ``coaddTempExps`` that end up overlapping
with the clipped
1717 footprints,
and a list of new masks
for the ``coaddTempExps``.
1722 Exposure to run detection on.
1723 tempExpRefList : `list`
1724 List of data reference to tempExp.
1728 result : `lsst.pipe.base.Struct`
1729 Result struct
with components:
1731 - ``clipFootprints``: list of clipped footprints.
1732 - ``clipIndices``: indices
for each ``clippedFootprint``
in
1734 - ``clipSpans``: List of dictionaries containing spanSet lists
1735 to clip. Each element contains the new maskplane name
1736 (
"CLIPPED")
as the key
and list of ``SpanSets``
as the value.
1737 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1738 compressed into footprints.
1740 mask = exp.getMaskedImage().getMask()
1741 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1742 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1744 fpSet.positive.merge(fpSet.negative)
1745 footprints = fpSet.positive
1746 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1747 ignoreMask = self.getBadPixelMask()
1751 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1754 visitDetectionFootprints = []
1756 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1757 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1758 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1761 for i, warpRef
in enumerate(tempExpRefList):
1762 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1763 immediate=
True).getMaskedImage().getMask()
1764 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1765 afwImage.PARENT,
True)
1766 maskVisitDet &= maskDetValue
1767 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1768 visitDetectionFootprints.append(visitFootprints)
1770 for j, footprint
in enumerate(footprints.getFootprints()):
1775 for j, footprint
in enumerate(footprints.getFootprints()):
1776 nPixel = footprint.getArea()
1779 for i
in range(len(tempExpRefList)):
1780 ignore = ignoreArr[i, j]
1781 overlapDet = overlapDetArr[i, j]
1782 totPixel = nPixel - ignore
1785 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1787 overlap.append(overlapDet/float(totPixel))
1790 overlap = numpy.array(overlap)
1791 if not len(overlap):
1798 if len(overlap) == 1:
1799 if overlap[0] > self.config.minClipFootOverlapSingle:
1804 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1805 if len(clipIndex) == 1:
1807 keepIndex = [clipIndex[0]]
1810 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1811 if len(clipIndex) == 2
and len(overlap) > 3:
1812 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1813 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1815 keepIndex = clipIndex
1820 for index
in keepIndex:
1821 globalIndex = indexList[index]
1822 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1824 clipIndices.append(numpy.array(indexList)[keepIndex])
1825 clipFootprints.append(footprint)
1827 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1828 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1830 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1831 maskClipValue, maskDetValue, coaddBBox):
1832 """Return individual warp footprints for large artifacts and append
1833 them to ``clipList`` in place.
1835 Identify big footprints composed of many sources
in the coadd
1836 difference that may have originated
in a large diffuse source
in the
1837 coadd. We do this by indentifying all clipped footprints that overlap
1838 significantly
with each source
in all the coaddTempExps.
1843 List of alt mask SpanSets
with clipping information. Modified.
1844 clipFootprints : `list`
1845 List of clipped footprints.
1846 clipIndices : `list`
1847 List of which entries
in tempExpClipList each footprint belongs to.
1849 Mask value of clipped pixels.
1851 Mask value of detected pixels.
1852 coaddBBox : `lsst.geom.Box`
1853 BBox of the coadd
and warps.
1857 bigFootprintsCoadd : `list`
1858 List of big footprints
1860 bigFootprintsCoadd = []
1861 ignoreMask = self.getBadPixelMask()
1862 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1863 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1864 for footprint
in visitFootprints.getFootprints():
1865 footprint.spans.setMask(maskVisitDet, maskDetValue)
1868 clippedFootprintsVisit = []
1869 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1870 if index
not in clipIndex:
1872 clippedFootprintsVisit.append(foot)
1873 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1874 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1876 bigFootprintsVisit = []
1877 for foot
in visitFootprints.getFootprints():
1878 if foot.getArea() < self.config.minBigOverlap:
1881 if nCount > self.config.minBigOverlap:
1882 bigFootprintsVisit.append(foot)
1883 bigFootprintsCoadd.append(foot)
1885 for footprint
in bigFootprintsVisit:
1886 clippedSpans[
"CLIPPED"].append(footprint.spans)
1888 return bigFootprintsCoadd
1892 psfMatchedWarps = pipeBase.connectionTypes.Input(
1893 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1894 "Only PSF-Matched Warps make sense for image subtraction. "
1895 "Therefore, they must be an additional declared input."),
1896 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1897 storageClass=
"ExposureF",
1898 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1902 templateCoadd = pipeBase.connectionTypes.Output(
1903 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1904 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1905 name=
"{outputCoaddName}CoaddPsfMatched",
1906 storageClass=
"ExposureF",
1907 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1912 if not config.assembleStaticSkyModel.doWrite:
1913 self.outputs.remove(
"templateCoadd")
1918 pipelineConnections=CompareWarpAssembleCoaddConnections):
1919 assembleStaticSkyModel = pexConfig.ConfigurableField(
1920 target=AssembleCoaddTask,
1921 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1922 " naive/first-iteration model of the static sky.",
1924 detect = pexConfig.ConfigurableField(
1925 target=SourceDetectionTask,
1926 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1928 detectTemplate = pexConfig.ConfigurableField(
1929 target=SourceDetectionTask,
1930 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1932 maskStreaks = pexConfig.ConfigurableField(
1933 target=MaskStreaksTask,
1934 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1935 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1938 streakMaskName = pexConfig.Field(
1941 doc=
"Name of mask bit used for streaks"
1943 maxNumEpochs = pexConfig.Field(
1944 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1945 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1946 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1947 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1948 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1949 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1950 "than transient and not masked.",
1954 maxFractionEpochsLow = pexConfig.RangeField(
1955 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1956 "Effective maxNumEpochs = "
1957 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1962 maxFractionEpochsHigh = pexConfig.RangeField(
1963 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1964 "Effective maxNumEpochs = "
1965 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1970 spatialThreshold = pexConfig.RangeField(
1971 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1972 "temporal criteria. If 0, clip all. If 1, clip none.",
1976 inclusiveMin=
True, inclusiveMax=
True
1978 doScaleWarpVariance = pexConfig.Field(
1979 doc=
"Rescale Warp variance plane using empirical noise?",
1983 scaleWarpVariance = pexConfig.ConfigurableField(
1984 target=ScaleVarianceTask,
1985 doc=
"Rescale variance on warps",
1987 doPreserveContainedBySource = pexConfig.Field(
1988 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1989 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1993 doPrefilterArtifacts = pexConfig.Field(
1994 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1995 "because they will be excluded anyway. This prevents them from contributing "
1996 "to the outlier epoch count image and potentially being labeled as persistant."
1997 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
2001 prefilterArtifactsMaskPlanes = pexConfig.ListField(
2002 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
2004 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
2006 prefilterArtifactsRatio = pexConfig.Field(
2007 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
2011 doFilterMorphological = pexConfig.Field(
2012 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
2017 growStreakFp = pexConfig.Field(
2018 doc=
"Grow streak footprints by this number multiplied by the PSF width",
2024 AssembleCoaddConfig.setDefaults(self)
2030 if "EDGE" in self.badMaskPlanes:
2031 self.badMaskPlanes.remove(
'EDGE')
2032 self.removeMaskPlanes.append(
'EDGE')
2041 self.
detect.doTempLocalBackground =
False
2042 self.
detect.reEstimateBackground =
False
2043 self.
detect.returnOriginalFootprints =
False
2044 self.
detect.thresholdPolarity =
"both"
2045 self.
detect.thresholdValue = 5
2046 self.
detect.minPixels = 4
2047 self.
detect.isotropicGrow =
True
2048 self.
detect.thresholdType =
"pixel_stdev"
2049 self.
detect.nSigmaToGrow = 0.4
2060 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
2061 "Please set assembleStaticSkyModel.doNImage=False")
2064 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
2065 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
2066 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
2071 """Assemble a compareWarp coadded image from a set of warps
2072 by masking artifacts detected by comparing PSF-matched warps.
2074 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
2075 we clip outliers). The problem
with doing this
is that when computing the
2076 coadd PSF at a given location, individual visit PSFs
from visits
with
2077 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
2078 In this task, we correct
for this behavior by creating a new badMaskPlane
2079 'CLIPPED' which marks pixels
in the individual warps suspected to contain
2080 an artifact. We populate this plane on the input warps by comparing
2081 PSF-matched warps
with a PSF-matched median coadd which serves
as a
2082 model of the static sky. Any group of pixels that deviates
from the
2083 PSF-matched template coadd by more than config.detect.threshold sigma,
2084 is an artifact candidate. The candidates are then filtered to remove
2085 variable sources
and sources that are difficult to subtract such
as
2086 bright stars. This filter
is configured using the config parameters
2087 ``temporalThreshold``
and ``spatialThreshold``. The temporalThreshold
is
2088 the maximum fraction of epochs that the deviation can appear
in and still
2089 be considered an artifact. The spatialThreshold
is the maximum fraction of
2090 pixels
in the footprint of the deviation that appear
in other epochs
2091 (where other epochs
is defined by the temporalThreshold). If the deviant
2092 region meets this criteria of having a significant percentage of pixels
2093 that deviate
in only a few epochs, these pixels have the
'CLIPPED' bit
2094 set
in the mask. These regions will
not contribute to the final coadd.
2095 Furthermore, any routine to determine the coadd PSF can now be cognizant
2096 of clipped regions. Note that the algorithm implemented by this task
is
2097 preliminary
and works correctly
for HSC data. Parameter modifications
and
2098 or considerable redesigning of the algorithm
is likley required
for other
2101 ``CompareWarpAssembleCoaddTask`` sub-classes
2102 ``AssembleCoaddTask``
and instantiates ``AssembleCoaddTask``
2103 as a subtask to generate the TemplateCoadd (the model of the static sky).
2107 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2108 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``; see
2109 ``baseDebug``
for more about ``debug.py`` files.
2111 This task supports the following debug variables:
2114 If
True then save the Epoch Count Image
as a fits file
in the `figPath`
2116 Path to save the debug fits images
and figures
2118 For example, put something like:
2120 .. code-block:: python
2123 def DebugInfo(name):
2125 if name ==
"lsst.pipe.tasks.assembleCoadd":
2126 di.saveCountIm =
True
2127 di.figPath =
"/desired/path/to/debugging/output/images"
2131 into your ``debug.py`` file
and run ``assemebleCoadd.py``
with the
2132 ``--debug`` flag. Some subtasks may have their own debug variables;
2133 see individual Task documentation.
2137 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2138 coadded image. The ``CompareWarpAssembleCoaddTask``
is invoked by running
2139 ``assembleCoadd.py``
with the flag ``--compareWarpCoadd``.
2140 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2141 and filter to be coadded (specified using
2142 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2143 along
with a list of coaddTempExps to attempt to coadd (specified using
2144 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2145 Only the warps that cover the specified tract
and patch will be coadded.
2146 A list of the available optional arguments can be obtained by calling
2147 ``assembleCoadd.py``
with the ``--help`` command line argument:
2149 .. code-block:: none
2151 assembleCoadd.py --help
2153 To demonstrate usage of the ``CompareWarpAssembleCoaddTask``
in the larger
2154 context of multi-band processing, we will generate the HSC-I & -R band
2155 oadds
from HSC engineering test data provided
in the ``ci_hsc`` package.
2156 To begin, assuming that the lsst stack has been already set up, we must
2157 set up the ``obs_subaru``
and ``ci_hsc`` packages.
2158 This defines the environment variable ``$CI_HSC_DIR``
and points at the
2159 location of the package. The raw HSC data live
in the ``$CI_HSC_DIR/raw``
2160 directory. To begin assembling the coadds, we must first
2163 process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
2165 create a skymap that covers the area of the sky present
in the raw exposures
2167 warp the individual calibrated exposures to the tangent plane of the coadd
2169 We can perform all of these steps by running
2171 .. code-block:: none
2173 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2175 This will produce warped ``coaddTempExps``
for each visit. To coadd the
2176 warped data, we call ``assembleCoadd.py``
as follows:
2178 .. code-block:: none
2180 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2181 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2182 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2183 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2184 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2185 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2186 --selectId visit=903988 ccd=24
2188 This will process the HSC-I band data. The results are written
in
2189 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2191 ConfigClass = CompareWarpAssembleCoaddConfig
2192 _DefaultName = "compareWarpAssembleCoadd"
2195 AssembleCoaddTask.__init__(self, *args, **kwargs)
2196 self.makeSubtask(
"assembleStaticSkyModel")
2197 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2198 self.makeSubtask(
"detect", schema=detectionSchema)
2199 if self.config.doPreserveContainedBySource:
2200 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2201 if self.config.doScaleWarpVariance:
2202 self.makeSubtask(
"scaleWarpVariance")
2203 if self.config.doFilterMorphological:
2204 self.makeSubtask(
"maskStreaks")
2206 @utils.inheritDoc(AssembleCoaddTask)
2209 Generate a templateCoadd to use as a naive model of static sky to
2210 subtract
from PSF-Matched warps.
2214 result : `lsst.pipe.base.Struct`
2215 Result struct
with components:
2221 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2222 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2226 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2227 if self.config.assembleStaticSkyModel.doWrite:
2228 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2231 del outputRefs.templateCoadd
2232 del staticSkyModelOutputRefs.templateCoadd
2235 if 'nImage' in staticSkyModelOutputRefs.keys():
2236 del staticSkyModelOutputRefs.nImage
2238 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2239 staticSkyModelOutputRefs)
2240 if templateCoadd
is None:
2243 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2244 nImage=templateCoadd.nImage,
2245 warpRefList=templateCoadd.warpRefList,
2246 imageScalerList=templateCoadd.imageScalerList,
2247 weightList=templateCoadd.weightList)
2249 @utils.inheritDoc(AssembleCoaddTask)
2252 Generate a templateCoadd to use as a naive model of static sky to
2253 subtract
from PSF-Matched warps.
2257 result : `lsst.pipe.base.Struct`
2258 Result struct
with components:
2263 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2264 if templateCoadd
is None:
2267 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2268 nImage=templateCoadd.nImage,
2269 warpRefList=templateCoadd.warpRefList,
2270 imageScalerList=templateCoadd.imageScalerList,
2271 weightList=templateCoadd.weightList)
2273 def _noTemplateMessage(self, warpType):
2274 warpName = (warpType[0].upper() + warpType[1:])
2275 message =
"""No %(warpName)s warps were found to build the template coadd which is
2276 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2277 first either rerun makeCoaddTempExp
with config.make%(warpName)s=
True or
2278 coaddDriver
with config.makeCoadTempExp.make%(warpName)s=
True, before assembleCoadd.
2280 Alternatively, to use another algorithm
with existing warps, retarget the CoaddDriverConfig to
2281 another algorithm like:
2284 config.assemble.retarget(SafeClipAssembleCoaddTask)
2285 """ % {"warpName": warpName}
2288 @utils.inheritDoc(AssembleCoaddTask)
2290 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2291 supplementaryData, *args, **kwargs):
2292 """Assemble the coadd.
2294 Find artifacts and apply them to the warps
' masks creating a list of
2295 alternative masks with a new
"CLIPPED" plane
and updated
"NO_DATA"
2296 plane. Then
pass these alternative masks to the base
class's `run`
2299 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2300 that must contain a ``templateCoadd`` that serves
as the
2301 model of the static sky.
2307 dataIds = [ref.dataId
for ref
in tempExpRefList]
2308 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
2310 if dataIds != psfMatchedDataIds:
2311 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2312 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2313 psfMatchedDataIds, dataIds)
2314 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2315 psfMatchedDataIds, dataIds)
2318 spanSetMaskList = self.
findArtifacts(supplementaryData.templateCoadd,
2319 supplementaryData.warpRefList,
2320 supplementaryData.imageScalerList)
2322 badMaskPlanes = self.config.badMaskPlanes[:]
2323 badMaskPlanes.append(
"CLIPPED")
2324 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2326 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2327 spanSetMaskList, mask=badPixelMask)
2331 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2335 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2341 altMaskList : `list`
2342 List of Dicts containing ``spanSet`` lists.
2343 Each element contains the new mask plane name (e.g. "CLIPPED
2344 and/
or "NO_DATA")
as the key,
and list of ``SpanSets`` to apply to
2347 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE",
"INEXACT_PSF"])
2348 for visitMask
in altMaskList:
2349 if "EDGE" in visitMask:
2350 for spanSet
in visitMask[
'EDGE']:
2351 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2356 Loop through warps twice. The first loop builds a map with the count
2357 of how many epochs each pixel deviates
from the templateCoadd by more
2358 than ``config.chiThreshold`` sigma. The second loop takes each
2359 difference image
and filters the artifacts detected
in each using
2360 count map to filter out variable sources
and sources that are
2361 difficult to subtract cleanly.
2366 Exposure to serve
as model of static sky.
2367 tempExpRefList : `list`
2368 List of data references to warps.
2369 imageScalerList : `list`
2370 List of image scalers.
2375 List of dicts containing information about CLIPPED
2376 (i.e., artifacts), NO_DATA,
and EDGE pixels.
2379 self.log.debug("Generating Count Image, and mask lists.")
2380 coaddBBox = templateCoadd.getBBox()
2381 slateIm = afwImage.ImageU(coaddBBox)
2382 epochCountImage = afwImage.ImageU(coaddBBox)
2383 nImage = afwImage.ImageU(coaddBBox)
2384 spanSetArtifactList = []
2385 spanSetNoDataMaskList = []
2386 spanSetEdgeList = []
2387 spanSetBadMorphoList = []
2388 badPixelMask = self.getBadPixelMask()
2391 templateCoadd.mask.clearAllMaskPlanes()
2393 if self.config.doPreserveContainedBySource:
2394 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2396 templateFootprints =
None
2398 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2400 if warpDiffExp
is not None:
2402 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2403 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2404 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2405 fpSet.positive.merge(fpSet.negative)
2406 footprints = fpSet.positive
2408 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2411 if self.config.doPrefilterArtifacts:
2415 self.detect.clearMask(warpDiffExp.mask)
2416 for spans
in spanSetList:
2417 spans.setImage(slateIm, 1, doClip=
True)
2418 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
2419 epochCountImage += slateIm
2421 if self.config.doFilterMorphological:
2422 maskName = self.config.streakMaskName
2423 _ = self.maskStreaks.
run(warpDiffExp)
2424 streakMask = warpDiffExp.mask
2425 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2426 streakMask.getPlaneBitMask(maskName)).split()
2428 psf = warpDiffExp.getPsf()
2429 for s, sset
in enumerate(spanSetStreak):
2430 psfShape = psf.computeShape(sset.computeCentroid())
2431 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius()
2432 sset_dilated = sset.dilated(int(dilation))
2433 spanSetStreak[s] = sset_dilated
2439 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2440 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2441 nansMask.setXY0(warpDiffExp.getXY0())
2442 edgeMask = warpDiffExp.mask
2443 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2444 edgeMask.getPlaneBitMask(
"EDGE")).split()
2448 nansMask = afwImage.MaskX(coaddBBox, 1)
2450 spanSetEdgeMask = []
2453 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2455 spanSetNoDataMaskList.append(spanSetNoDataMask)
2456 spanSetArtifactList.append(spanSetList)
2457 spanSetEdgeList.append(spanSetEdgeMask)
2458 if self.config.doFilterMorphological:
2459 spanSetBadMorphoList.append(spanSetStreak)
2463 epochCountImage.writeFits(path)
2465 for i, spanSetList
in enumerate(spanSetArtifactList):
2467 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
2469 spanSetArtifactList[i] = filteredSpanSetList
2470 if self.config.doFilterMorphological:
2471 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2474 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2475 altMasks.append({
'CLIPPED': artifacts,
2481 """Remove artifact candidates covered by bad mask plane.
2483 Any future editing of the candidate list that does not depend on
2484 temporal information should go
in this method.
2488 spanSetList : `list`
2489 List of SpanSets representing artifact candidates.
2491 Exposure containing mask planes used to prefilter.
2495 returnSpanSetList : `list`
2496 List of SpanSets
with artifacts.
2498 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2499 goodArr = (exp.mask.array & badPixelMask) == 0
2500 returnSpanSetList = []
2501 bbox = exp.getBBox()
2502 x0, y0 = exp.getXY0()
2503 for i, span
in enumerate(spanSetList):
2504 y, x = span.clippedTo(bbox).indices()
2505 yIndexLocal = numpy.array(y) - y0
2506 xIndexLocal = numpy.array(x) - x0
2507 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2508 if goodRatio > self.config.prefilterArtifactsRatio:
2509 returnSpanSetList.append(span)
2510 return returnSpanSetList
2512 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2513 """Filter artifact candidates.
2517 spanSetList : `list`
2518 List of SpanSets representing artifact candidates.
2520 Image of accumulated number of warpDiff detections.
2522 Image of the accumulated number of total epochs contributing.
2526 maskSpanSetList : `list`
2527 List of SpanSets with artifacts.
2530 maskSpanSetList = []
2531 x0, y0 = epochCountImage.getXY0()
2532 for i, span
in enumerate(spanSetList):
2533 y, x = span.indices()
2534 yIdxLocal = [y1 - y0
for y1
in y]
2535 xIdxLocal = [x1 - x0
for x1
in x]
2536 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2537 totalN = nImage.array[yIdxLocal, xIdxLocal]
2540 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2541 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2542 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2543 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2544 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2545 & (outlierN <= effectiveMaxNumEpochs))
2546 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2547 if percentBelowThreshold > self.config.spatialThreshold:
2548 maskSpanSetList.append(span)
2550 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2552 filteredMaskSpanSetList = []
2553 for span
in maskSpanSetList:
2555 for footprint
in footprintsToExclude.positive.getFootprints():
2556 if footprint.spans.contains(span):
2560 filteredMaskSpanSetList.append(span)
2561 maskSpanSetList = filteredMaskSpanSetList
2563 return maskSpanSetList
2565 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2566 """Fetch a warp from the butler and return a warpDiff.
2571 Butler dataRef for the warp.
2573 An image scaler object.
2575 Exposure to be substracted
from the scaled warp.
2580 Exposure of the image difference between the warp
and template.
2588 warpName = self.getTempExpDatasetName(
'psfMatched')
2589 if not isinstance(warpRef, DeferredDatasetHandle):
2590 if not warpRef.datasetExists(warpName):
2591 self.log.warning(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2593 warp = warpRef.get(datasetType=warpName, immediate=
True)
2595 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2596 mi = warp.getMaskedImage()
2597 if self.config.doScaleWarpVariance:
2599 self.scaleWarpVariance.
run(mi)
2600 except Exception
as exc:
2601 self.log.warning(
"Unable to rescale variance of warp (%s); leaving it as-is", exc)
2602 mi -= templateCoadd.getMaskedImage()
2605 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2606 """Return a path to which to write debugging output.
2608 Creates a hyphen-delimited string of dataId values for simple filenames.
2613 Prefix
for filename.
2615 Butler dataRef to make the path
from.
2616 coaddLevel : `bool`, optional.
2617 If
True, include only coadd-level keys (e.g.,
'tract',
'patch',
2618 'filter', but no
'visit').
2623 Path
for debugging output.
2626 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2628 keys = warpRef.dataId.keys()
2629 keyList = sorted(keys, reverse=
True)
2631 filename =
"%s-%s.fits" % (prefix,
'-'.join([
str(warpRef.dataId[k])
for k
in keyList]))
2632 return os.path.join(directory, filename)
def makeDataRefList(self, namespace)
def __init__(self, *config=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def applyAltEdgeMask(self, mask, altMaskList)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def __init__(self, *args, **kwargs)
def detectClip(self, exp, tempExpRefList)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def setInexactPsf(self, mask)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeCoaddSuffix(warpType="direct")
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")