39import lsst.utils
as utils
41from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42from .interpImage
import InterpImageTask
43from .scaleZeroPoint
import ScaleZeroPointTask
44from .coaddHelpers
import groupPatchExposures, getGroupDataRef
45from .scaleVariance
import ScaleVarianceTask
46from .maskStreaks
import MaskStreaksTask
47from .healSparseMapping
import HealSparseInputMapTask
49from lsst.daf.butler
import DeferredDatasetHandle
50from lsst.utils.timer
import timeMethod
52__all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
53 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
54 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
56log = logging.getLogger(__name__)
60 dimensions=(
"tract",
"patch",
"band",
"skymap"),
61 defaultTemplates={
"inputCoaddName":
"deep",
62 "outputCoaddName":
"deep",
64 "warpTypeSuffix":
""}):
66 inputWarps = pipeBase.connectionTypes.Input(
67 doc=(
"Input list of warps to be assemebled i.e. stacked."
68 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
69 name=
"{inputCoaddName}Coadd_{warpType}Warp",
70 storageClass=
"ExposureF",
71 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
75 skyMap = pipeBase.connectionTypes.Input(
76 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78 storageClass=
"SkyMap",
79 dimensions=(
"skymap", ),
81 selectedVisits = pipeBase.connectionTypes.Input(
82 doc=
"Selected visits to be coadded.",
83 name=
"{outputCoaddName}Visits",
84 storageClass=
"StructuredDataDict",
85 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band")
87 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
88 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
90 name=
"brightObjectMask",
91 storageClass=
"ObjectMaskCatalog",
92 dimensions=(
"tract",
"patch",
"skymap",
"band"),
94 coaddExposure = pipeBase.connectionTypes.Output(
95 doc=
"Output coadded exposure, produced by stacking input warps",
96 name=
"{outputCoaddName}Coadd{warpTypeSuffix}",
97 storageClass=
"ExposureF",
98 dimensions=(
"tract",
"patch",
"skymap",
"band"),
100 nImage = pipeBase.connectionTypes.Output(
101 doc=
"Output image of number of input images per pixel",
102 name=
"{outputCoaddName}Coadd_nImage",
103 storageClass=
"ImageU",
104 dimensions=(
"tract",
"patch",
"skymap",
"band"),
106 inputMap = pipeBase.connectionTypes.Output(
107 doc=
"Output healsparse map of input images",
108 name=
"{outputCoaddName}Coadd_inputMap",
109 storageClass=
"HealSparseMap",
110 dimensions=(
"tract",
"patch",
"skymap",
"band"),
113 def __init__(self, *, config=None):
114 super().__init__(config=config)
119 templateValues = {name: getattr(config.connections, name)
for name
in self.defaultTemplates}
120 templateValues[
'warpType'] = config.warpType
122 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
123 for name
in self.allConnections}
124 self._typeNameToVarName = {v: k
for k, v
in self._nameOverrides.items()}
127 if not config.doMaskBrightObjects:
128 self.prerequisiteInputs.remove(
"brightObjectMask")
130 if not config.doSelectVisits:
131 self.inputs.remove(
"selectedVisits")
133 if not config.doNImage:
134 self.outputs.remove(
"nImage")
136 if not self.config.doInputMap:
137 self.outputs.remove(
"inputMap")
140class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
141 pipelineConnections=AssembleCoaddConnections):
142 """Configuration parameters for the `AssembleCoaddTask`.
146 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
147 only set the bitplane config.brightObjectMaskName. To make this useful you
148 *must* also configure the flags.pixel algorithm,
for example by adding
152 config.measurement.plugins[
"base_PixelFlags"].masksFpCenter.append(
"BRIGHT_OBJECT")
153 config.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere.append(
"BRIGHT_OBJECT")
155 to your measureCoaddSources.py
and forcedPhotCoadd.py config overrides.
157 warpType = pexConfig.Field(
158 doc="Warp name: one of 'direct' or 'psfMatched'",
162 subregionSize = pexConfig.ListField(
164 doc=
"Width, height of stack subregion size; "
165 "make small enough that a full stack of images will fit into memory at once.",
167 default=(2000, 2000),
169 statistic = pexConfig.Field(
171 doc=
"Main stacking statistic for aggregating over the epochs.",
174 doOnlineForMean = pexConfig.Field(
176 doc=
"Perform online coaddition when statistic=\"MEAN\" to save memory?",
179 doSigmaClip = pexConfig.Field(
181 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
184 sigmaClip = pexConfig.Field(
186 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
189 clipIter = pexConfig.Field(
191 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
194 calcErrorFromInputVariance = pexConfig.Field(
196 doc=
"Calculate coadd variance from input variance by stacking statistic."
197 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
200 scaleZeroPoint = pexConfig.ConfigurableField(
201 target=ScaleZeroPointTask,
202 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
204 doInterp = pexConfig.Field(
205 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
209 interpImage = pexConfig.ConfigurableField(
210 target=InterpImageTask,
211 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
213 doWrite = pexConfig.Field(
214 doc=
"Persist coadd?",
218 doNImage = pexConfig.Field(
219 doc=
"Create image of number of contributing exposures for each pixel",
223 doUsePsfMatchedPolygons = pexConfig.Field(
224 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
228 maskPropagationThresholds = pexConfig.DictField(
231 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
232 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
233 "would have contributed exceeds this value."),
234 default={
"SAT": 0.1},
236 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
237 doc=
"Mask planes to remove before coadding")
238 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
239 doc=
"Set mask and flag bits for bright objects?")
240 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
241 doc=
"Name of mask bit used for bright objects")
242 coaddPsf = pexConfig.ConfigField(
243 doc=
"Configuration for CoaddPsf",
244 dtype=measAlg.CoaddPsfConfig,
246 doAttachTransmissionCurve = pexConfig.Field(
247 dtype=bool, default=
False, optional=
False,
248 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
249 "(requires all input Exposures to have TransmissionCurves).")
251 hasFakes = pexConfig.Field(
254 doc=
"Should be set to True if fake sources have been inserted into the input data."
256 doSelectVisits = pexConfig.Field(
257 doc=
"Coadd only visits selected by a SelectVisitsTask",
261 doInputMap = pexConfig.Field(
262 doc=
"Create a bitwise map of coadd inputs",
266 inputMapper = pexConfig.ConfigurableField(
267 doc=
"Input map creation subtask.",
268 target=HealSparseInputMapTask,
271 def setDefaults(self):
272 super().setDefaults()
273 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
280 log.warning(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
281 self.warpType =
'psfMatched'
282 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
283 log.warning(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
284 self.statistic =
"MEANCLIP"
285 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
286 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
287 "compute and set a non-zero coadd variance estimate." % (self.statistic))
289 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
290 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
291 stackableStats = [
str(k)
for k
in afwMath.Property.__members__.keys()
292 if str(k)
not in unstackableStats]
293 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
294 % (self.statistic, stackableStats))
297class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
298 """Assemble a coadded image from a set of warps (coadded temporary exposures).
300 We want to assemble a coadded image from a set of Warps (also called
301 coadded temporary exposures
or ``coaddTempExps``).
302 Each input Warp covers a patch on the sky
and corresponds to a single
303 run/visit/exposure of the covered patch. We provide the task
with a list
304 of Warps (``selectDataList``)
from which it selects Warps that cover the
305 specified patch (pointed at by ``dataRef``).
306 Each Warp that goes into a coadd will typically have an independent
307 photometric zero-point. Therefore, we must scale each Warp to set it to
308 a common photometric zeropoint. WarpType may be one of
'direct' or
309 'psfMatched',
and the boolean configs `config.makeDirect`
and
310 `config.makePsfMatched` set which of the warp types will be coadded.
311 The coadd
is computed
as a mean
with optional outlier rejection.
312 Criteria
for outlier rejection are set
in `AssembleCoaddConfig`.
313 Finally, Warps can have bad
'NaN' pixels which received no input
from the
314 source calExps. We interpolate over these bad (NaN) pixels.
316 `AssembleCoaddTask` uses several sub-tasks. These are
318 - `ScaleZeroPointTask`
319 - create
and use an ``imageScaler`` object to scale the photometric zeropoint
for each Warp
321 - interpolate across bad pixels (NaN)
in the final coadd
323 You can retarget these subtasks
if you wish.
327 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
328 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``; see
329 `baseDebug`
for more about ``debug.py`` files. `AssembleCoaddTask` has
330 no debug variables of its own. Some of the subtasks may support debug
331 variables. See the documentation
for the subtasks
for further information.
335 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
336 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
337 with the flag
'--legacyCoadd'. Usage of assembleCoadd.py expects two
338 inputs: a data reference to the tract patch
and filter to be coadded,
and
339 a list of Warps to attempt to coadd. These are specified using ``--id``
and
340 ``--selectId``, respectively:
344 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
345 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
347 Only the Warps that cover the specified tract
and patch will be coadded.
348 A list of the available optional arguments can be obtained by calling
349 ``assembleCoadd.py``
with the ``--help`` command line argument:
353 assembleCoadd.py --help
355 To demonstrate usage of the `AssembleCoaddTask`
in the larger context of
356 multi-band processing, we will generate the HSC-I & -R band coadds
from
357 HSC engineering test data provided
in the ``ci_hsc`` package. To begin,
358 assuming that the lsst stack has been already set up, we must set up the
359 obs_subaru
and ``ci_hsc`` packages. This defines the environment variable
360 ``$CI_HSC_DIR``
and points at the location of the package. The raw HSC
361 data live
in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
362 coadds, we must first
365 - process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
367 - create a skymap that covers the area of the sky present
in the raw exposures
369 - warp the individual calibrated exposures to the tangent plane of the coadd
371 We can perform all of these steps by running
375 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
377 This will produce warped exposures
for each visit. To coadd the warped
378 data, we call assembleCoadd.py
as follows:
382 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
383 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
384 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
385 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
386 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
387 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
388 --selectId visit=903988 ccd=24
390 that will process the HSC-I band data. The results are written
in
391 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
393 You may also choose to run:
397 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
398 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
399 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
400 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
401 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
402 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
403 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
404 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
406 to generate the coadd
for the HSC-R band
if you are interested
in
407 following multiBand Coadd processing
as discussed
in `pipeTasks_multiBand`
408 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
409 rather than `AssembleCoaddTask` to make the coadd.
411 ConfigClass = AssembleCoaddConfig
412 _DefaultName = "assembleCoadd"
414 def __init__(self, *args, **kwargs):
417 argNames = [
"config",
"name",
"parentTask",
"log"]
418 kwargs.update({k: v
for k, v
in zip(argNames, args)})
419 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
420 "PipelineTask will not take positional args" % argNames, FutureWarning)
422 super().__init__(**kwargs)
423 self.makeSubtask(
"interpImage")
424 self.makeSubtask(
"scaleZeroPoint")
426 if self.config.doMaskBrightObjects:
427 mask = afwImage.Mask()
429 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
430 except pexExceptions.LsstCppException:
431 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
432 mask.getMaskPlaneDict().keys())
435 if self.config.doInputMap:
436 self.makeSubtask(
"inputMapper")
438 self.warpType = self.config.warpType
440 @utils.inheritDoc(pipeBase.PipelineTask)
441 def runQuantum(self, butlerQC, inputRefs, outputRefs):
446 Assemble a coadd from a set of Warps.
448 PipelineTask (Gen3) entry point to Coadd a set of Warps.
449 Analogous to `runDataRef`, it prepares all the data products to be
450 passed to `run`,
and processes the results before returning a struct
451 of results to be written out. AssembleCoadd cannot fit all Warps
in memory.
452 Therefore, its inputs are accessed subregion by subregion
453 by the Gen3 `DeferredDatasetHandle` that
is analagous to the Gen2
454 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
455 correspond to an update
in `runDataRef`
while both entry points
458 inputData = butlerQC.get(inputRefs)
462 skyMap = inputData[
"skyMap"]
463 outputDataId = butlerQC.quantum.dataId
466 tractId=outputDataId[
'tract'],
467 patchId=outputDataId[
'patch'])
469 if self.config.doSelectVisits:
470 warpRefList = self.filterWarps(inputData[
'inputWarps'], inputData[
'selectedVisits'])
472 warpRefList = inputData[
'inputWarps']
475 inputs = self.prepareInputs(warpRefList)
476 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
477 self.getTempExpDatasetName(self.warpType))
478 if len(inputs.tempExpRefList) == 0:
479 raise pipeBase.NoWorkFound(
"No coadd temporary exposures found")
481 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
482 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
483 inputs.weightList, supplementaryData=supplementaryData)
485 inputData.setdefault(
'brightObjectMask',
None)
486 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
488 if self.config.doWrite:
489 butlerQC.put(retStruct, outputRefs)
493 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
494 """Assemble a coadd from a set of Warps.
496 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
497 Compute weights to be applied to each Warp and
498 find scalings to match the photometric zeropoint to a reference Warp.
499 Assemble the Warps using `run`. Interpolate over NaNs
and
500 optionally write the coadd to disk. Return the coadded exposure.
505 Data reference defining the patch
for coaddition
and the
506 reference Warp (
if ``config.autoReference=
False``).
507 Used to access the following data products:
508 - ``self.config.coaddName +
"Coadd_skyMap"``
509 - ``self.config.coaddName +
"Coadd_ + <warpType> + "Warp
"`` (optionally)
510 - ``self.config.coaddName + "Coadd"``
511 selectDataList : `list`
512 List of data references to Calexps. Data to be coadded will be
513 selected
from this list based on overlap
with the patch defined
514 by dataRef, grouped by visit,
and converted to a list of data
517 List of data references to Warps to be coadded.
518 Note: `warpRefList`
is just the new name
for `tempExpRefList`.
522 retStruct : `lsst.pipe.base.Struct`
523 Result struct
with components:
525 - ``coaddExposure``: coadded exposure (``Exposure``).
526 - ``nImage``: exposure count image (``Image``).
528 if selectDataList
and warpRefList:
529 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, "
530 "and which to use is ambiguous. Please pass only one.")
532 skyInfo = self.getSkyInfo(dataRef)
533 if warpRefList
is None:
534 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
535 if len(calExpRefList) == 0:
536 self.log.warning(
"No exposures to coadd")
538 self.log.info(
"Coadding %d exposures", len(calExpRefList))
540 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
542 inputData = self.prepareInputs(warpRefList)
543 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
544 self.getTempExpDatasetName(self.warpType))
545 if len(inputData.tempExpRefList) == 0:
546 self.log.warning(
"No coadd temporary exposures found")
549 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
551 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
552 inputData.weightList, supplementaryData=supplementaryData)
554 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None
555 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
557 if self.config.doWrite:
558 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
559 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
561 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
562 self.log.info(
"Persisting %s", coaddDatasetName)
563 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
564 if self.config.doNImage
and retStruct.nImage
is not None:
565 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
570 """Interpolate over missing data and mask bright stars.
575 The coadded exposure to process.
576 dataRef : `lsst.daf.persistence.ButlerDataRef`
577 Butler data reference for supplementary data.
579 if self.config.doInterp:
580 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
582 varArray = coaddExposure.variance.array
583 with numpy.errstate(invalid=
"ignore"):
584 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
586 if self.config.doMaskBrightObjects:
587 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
590 """Make additional inputs to run() specific to subclasses (Gen2)
592 Duplicates interface of `runDataRef` method
593 Available to be implemented by subclasses only if they need the
594 coadd dataRef
for performing preliminary processing before
595 assembling the coadd.
599 dataRef : `lsst.daf.persistence.ButlerDataRef`
600 Butler data reference
for supplementary data.
601 selectDataList : `list` (optional)
602 Optional List of data references to Calexps.
603 warpRefList : `list` (optional)
604 Optional List of data references to Warps.
606 return pipeBase.Struct()
609 """Make additional inputs to run() specific to subclasses (Gen3)
611 Duplicates interface of `runQuantum` method.
612 Available to be implemented by subclasses only if they need the
613 coadd dataRef
for performing preliminary processing before
614 assembling the coadd.
618 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
619 Gen3 Butler object
for fetching additional data products before
620 running the Task specialized
for quantum being processed
621 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
622 Attributes are the names of the connections describing input dataset types.
623 Values are DatasetRefs that task consumes
for corresponding dataset type.
624 DataIds are guaranteed to match data objects
in ``inputData``.
625 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
626 Attributes are the names of the connections describing output dataset types.
627 Values are DatasetRefs that task
is to produce
628 for corresponding dataset type.
630 return pipeBase.Struct()
633 """Generate list data references corresponding to warped exposures
634 that lie within the patch to be coadded.
639 Data reference for patch.
640 calExpRefList : `list`
641 List of data references
for input calexps.
645 tempExpRefList : `list`
646 List of Warp/CoaddTempExp data references.
648 butler = patchRef.getButler()
649 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
650 self.getTempExpDatasetName(self.warpType))
651 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
652 g, groupData.keys) for
653 g
in groupData.groups.keys()]
654 return tempExpRefList
657 """Prepare the input warps for coaddition by measuring the weight for
658 each warp and the scaling
for the photometric zero point.
660 Each Warp has its own photometric zeropoint
and background variance.
661 Before coadding these Warps together, compute a scale factor to
662 normalize the photometric zeropoint
and compute the weight
for each Warp.
667 List of data references to tempExp
671 result : `lsst.pipe.base.Struct`
672 Result struct
with components:
674 - ``tempExprefList``: `list` of data references to tempExp.
675 - ``weightList``: `list` of weightings.
676 - ``imageScalerList``: `list` of image scalers.
678 statsCtrl = afwMath.StatisticsControl()
679 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
680 statsCtrl.setNumIter(self.config.clipIter)
681 statsCtrl.setAndMask(self.getBadPixelMask())
682 statsCtrl.setNanSafe(True)
689 tempExpName = self.getTempExpDatasetName(self.warpType)
690 for tempExpRef
in refList:
693 if not isinstance(tempExpRef, DeferredDatasetHandle):
694 if not tempExpRef.datasetExists(tempExpName):
695 self.log.warning(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
698 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
700 if numpy.isnan(tempExp.image.array).all():
702 maskedImage = tempExp.getMaskedImage()
703 imageScaler = self.scaleZeroPoint.computeImageScaler(
708 imageScaler.scaleMaskedImage(maskedImage)
709 except Exception
as e:
710 self.log.warning(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
712 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
713 afwMath.MEANCLIP, statsCtrl)
714 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
715 weight = 1.0 / float(meanVar)
716 if not numpy.isfinite(weight):
717 self.log.warning(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
719 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
724 tempExpRefList.append(tempExpRef)
725 weightList.append(weight)
726 imageScalerList.append(imageScaler)
728 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
729 imageScalerList=imageScalerList)
732 """Prepare the statistics for coadding images.
736 mask : `int`, optional
737 Bit mask value to exclude from coaddition.
741 stats : `lsst.pipe.base.Struct`
742 Statistics structure
with the following fields:
744 - ``statsCtrl``: Statistics control object
for coadd
746 - ``statsFlags``: Statistic
for coadd (`lsst.afw.math.Property`)
749 mask = self.getBadPixelMask()
750 statsCtrl = afwMath.StatisticsControl()
751 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
752 statsCtrl.setNumIter(self.config.clipIter)
753 statsCtrl.setAndMask(mask)
754 statsCtrl.setNanSafe(
True)
755 statsCtrl.setWeighted(
True)
756 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
757 for plane, threshold
in self.config.maskPropagationThresholds.items():
758 bit = afwImage.Mask.getMaskPlane(plane)
759 statsCtrl.setMaskPropagationThreshold(bit, threshold)
760 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
761 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
764 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
765 altMaskList=None, mask=None, supplementaryData=None):
766 """Assemble a coadd from input warps
768 Assemble the coadd using the provided list of coaddTempExps. Since
769 the full coadd covers a patch (a large area), the assembly is
770 performed over small areas on the image at a time
in order to
771 conserve memory usage. Iterate over subregions within the outer
772 bbox of the patch using `assembleSubregion` to stack the corresponding
773 subregions
from the coaddTempExps
with the statistic specified.
774 Set the edge bits the coadd mask based on the weight map.
778 skyInfo : `lsst.pipe.base.Struct`
779 Struct
with geometric information about the patch.
780 tempExpRefList : `list`
781 List of data references to Warps (previously called CoaddTempExps).
782 imageScalerList : `list`
783 List of image scalers.
786 altMaskList : `list`, optional
787 List of alternate masks to use rather than those stored
with
789 mask : `int`, optional
790 Bit mask value to exclude
from coaddition.
791 supplementaryData : lsst.pipe.base.Struct, optional
792 Struct
with additional data products needed to assemble coadd.
793 Only used by subclasses that implement `makeSupplementaryData`
798 result : `lsst.pipe.base.Struct`
799 Result struct
with components:
803 - ``inputMap``: bit-wise map of inputs,
if requested.
804 - ``warpRefList``: input list of refs to the warps (
805 ``lsst.daf.butler.DeferredDatasetHandle``
or
806 ``lsst.daf.persistence.ButlerDataRef``)
808 - ``imageScalerList``: input list of image scalers (unmodified)
809 - ``weightList``: input list of weights (unmodified)
811 tempExpName = self.getTempExpDatasetName(self.warpType)
812 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
813 stats = self.prepareStats(mask=mask)
815 if altMaskList
is None:
816 altMaskList = [
None]*len(tempExpRefList)
818 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
819 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
820 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
821 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
822 coaddMaskedImage = coaddExposure.getMaskedImage()
823 subregionSizeArr = self.config.subregionSize
824 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
826 if self.config.doNImage:
827 nImage = afwImage.ImageU(skyInfo.bbox)
832 if self.config.doInputMap:
833 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
835 coaddExposure.getInfo().getCoaddInputs().ccds)
837 if self.config.doOnlineForMean
and self.config.statistic ==
"MEAN":
839 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
840 weightList, altMaskList, stats.ctrl,
842 except Exception
as e:
843 self.log.exception(
"Cannot compute online coadd %s", e)
846 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
848 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
849 weightList, altMaskList, stats.flags, stats.ctrl,
851 except Exception
as e:
852 self.log.exception(
"Cannot compute coadd %s: %s", subBBox, e)
856 if self.config.doInputMap:
857 self.inputMapper.finalize_ccd_input_map_mask()
858 inputMap = self.inputMapper.ccd_input_map
862 self.setInexactPsf(coaddMaskedImage.getMask())
865 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
866 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
867 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
868 weightList=weightList, inputMap=inputMap)
871 """Set the metadata for the coadd.
873 This basic implementation sets the filter from the first input.
878 The target exposure
for the coadd.
879 tempExpRefList : `list`
880 List of data references to tempExp.
884 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
885 tempExpName = self.getTempExpDatasetName(self.warpType)
891 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
893 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
896 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
897 for tempExpRef
in tempExpRefList]
898 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
902 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
903 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
904 coaddInputs.ccds.reserve(numCcds)
905 coaddInputs.visits.reserve(len(tempExpList))
907 for tempExp, weight
in zip(tempExpList, weightList):
908 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
910 if self.config.doUsePsfMatchedPolygons:
911 self.shrinkValidPolygons(coaddInputs)
913 coaddInputs.visits.sort()
914 coaddInputs.ccds.sort()
915 if self.warpType ==
"psfMatched":
920 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
921 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
922 for modelPsf
in modelPsfList]
923 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
925 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
926 self.config.coaddPsf.makeControl())
927 coaddExposure.setPsf(psf)
928 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
929 coaddExposure.getWcs())
930 coaddExposure.getInfo().setApCorrMap(apCorrMap)
931 if self.config.doAttachTransmissionCurve:
932 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
933 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
936 altMaskList, statsFlags, statsCtrl, nImage=None):
937 """Assemble the coadd for a sub-region.
939 For each coaddTempExp, check for (
and swap
in) an alternative mask
940 if one
is passed. Remove mask planes listed
in
941 `config.removeMaskPlanes`. Finally, stack the actual exposures using
942 `lsst.afw.math.statisticsStack`
with the statistic specified by
943 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
for
944 a mean-stack
or `lsst.afw.math.MEANCLIP`
for outlier rejection using
945 an N-sigma clipped mean where N
and iterations are specified by
946 statsCtrl. Assign the stacked subregion back to the coadd.
951 The target exposure
for the coadd.
952 bbox : `lsst.geom.Box`
954 tempExpRefList : `list`
955 List of data reference to tempExp.
956 imageScalerList : `list`
957 List of image scalers.
961 List of alternate masks to use rather than those stored
with
962 tempExp,
or None. Each element
is dict
with keys = mask plane
963 name to which to add the spans.
964 statsFlags : `lsst.afw.math.Property`
965 Property object
for statistic
for coadd.
967 Statistics control object
for coadd.
968 nImage : `lsst.afw.image.ImageU`, optional
969 Keeps track of exposure count
for each pixel.
971 self.log.debug("Computing coadd over %s", bbox)
972 tempExpName = self.getTempExpDatasetName(self.warpType)
973 coaddExposure.mask.addMaskPlane(
"REJECTED")
974 coaddExposure.mask.addMaskPlane(
"CLIPPED")
975 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
976 maskMap = self.setRejectedMaskMapping(statsCtrl)
977 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
979 if nImage
is not None:
980 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
981 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
983 if isinstance(tempExpRef, DeferredDatasetHandle):
985 exposure = tempExpRef.get(parameters={
'bbox': bbox})
988 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
990 maskedImage = exposure.getMaskedImage()
991 mask = maskedImage.getMask()
992 if altMask
is not None:
993 self.applyAltMaskPlanes(mask, altMask)
994 imageScaler.scaleMaskedImage(maskedImage)
998 if nImage
is not None:
999 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
1000 if self.config.removeMaskPlanes:
1001 self.removeMaskPlanes(maskedImage)
1002 maskedImageList.append(maskedImage)
1004 if self.config.doInputMap:
1005 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1006 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1008 with self.timer(
"stack"):
1009 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
1012 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
1013 if nImage
is not None:
1014 nImage.assign(subNImage, bbox)
1017 altMaskList, statsCtrl, nImage=None):
1018 """Assemble the coadd using the "online" method.
1020 This method takes a running sum of images and weights to save memory.
1021 It only works
for MEAN statistics.
1026 The target exposure
for the coadd.
1027 tempExpRefList : `list`
1028 List of data reference to tempExp.
1029 imageScalerList : `list`
1030 List of image scalers.
1033 altMaskList : `list`
1034 List of alternate masks to use rather than those stored
with
1035 tempExp,
or None. Each element
is dict
with keys = mask plane
1036 name to which to add the spans.
1038 Statistics control object
for coadd
1039 nImage : `lsst.afw.image.ImageU`, optional
1040 Keeps track of exposure count
for each pixel.
1042 self.log.debug("Computing online coadd.")
1043 tempExpName = self.getTempExpDatasetName(self.warpType)
1044 coaddExposure.mask.addMaskPlane(
"REJECTED")
1045 coaddExposure.mask.addMaskPlane(
"CLIPPED")
1046 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
1047 maskMap = self.setRejectedMaskMapping(statsCtrl)
1048 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
1050 bbox = coaddExposure.maskedImage.getBBox()
1052 stacker = AccumulatorMeanStack(
1053 coaddExposure.image.array.shape,
1054 statsCtrl.getAndMask(),
1055 mask_threshold_dict=thresholdDict,
1057 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
1058 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
1059 compute_n_image=(nImage
is not None)
1062 for tempExpRef, imageScaler, altMask, weight
in zip(tempExpRefList,
1066 if isinstance(tempExpRef, DeferredDatasetHandle):
1068 exposure = tempExpRef.get()
1071 exposure = tempExpRef.get(tempExpName)
1073 maskedImage = exposure.getMaskedImage()
1074 mask = maskedImage.getMask()
1075 if altMask
is not None:
1076 self.applyAltMaskPlanes(mask, altMask)
1077 imageScaler.scaleMaskedImage(maskedImage)
1078 if self.config.removeMaskPlanes:
1079 self.removeMaskPlanes(maskedImage)
1081 stacker.add_masked_image(maskedImage, weight=weight)
1083 if self.config.doInputMap:
1084 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1085 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1087 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
1089 if nImage
is not None:
1090 nImage.array[:, :] = stacker.n_image
1093 """Unset the mask of an image for mask planes specified in the config.
1098 The masked image to be modified.
1100 mask = maskedImage.getMask()
1101 for maskPlane
in self.config.removeMaskPlanes:
1103 mask &= ~mask.getPlaneBitMask(maskPlane)
1104 except pexExceptions.InvalidParameterError:
1105 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
1109 def setRejectedMaskMapping(statsCtrl):
1110 """Map certain mask planes of the warps to new planes for the coadd.
1112 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1113 or CLIPPED, set it to REJECTED on the coadd.
1114 If a pixel
is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1115 If a pixel
is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1120 Statistics control object
for coadd
1124 maskMap : `list` of `tuple` of `int`
1125 A list of mappings of mask planes of the warped exposures to
1126 mask planes of the coadd.
1128 edge = afwImage.Mask.getPlaneBitMask("EDGE")
1129 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
1130 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
1131 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1132 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
1133 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
1138 """Apply in place alt mask formatted as SpanSets to a mask.
1144 altMaskSpans : `dict`
1145 SpanSet lists to apply. Each element contains the new mask
1146 plane name (e.g. "CLIPPED and/or "NO_DATA
") as the key,
1147 and list of SpanSets to apply to the mask.
1154 if self.config.doUsePsfMatchedPolygons:
1155 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1160 for spanSet
in altMaskSpans[
'NO_DATA']:
1161 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1163 for plane, spanSetList
in altMaskSpans.items():
1164 maskClipValue = mask.addMaskPlane(plane)
1165 for spanSet
in spanSetList:
1166 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1170 """Shrink coaddInputs' ccds' ValidPolygons in place.
1172 Either modify each ccd's validPolygon in place, or if CoaddInputs
1173 does not have a validPolygon, create one
from its bbox.
1177 coaddInputs : `lsst.afw.image.coaddInputs`
1181 for ccd
in coaddInputs.ccds:
1182 polyOrig = ccd.getValidPolygon()
1183 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1184 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1186 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1188 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1189 ccd.setValidPolygon(validPolygon)
1192 """Retrieve the bright object masks.
1194 Returns None on failure.
1204 Bright object mask
from the Butler object,
or None if it cannot
1208 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1209 except Exception
as e:
1210 self.log.warning(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1214 """Set the bright object masks.
1219 Exposure under consideration.
1221 Data identifier dict for patch.
1223 Table of bright objects to mask.
1226 if brightObjectMasks
is None:
1227 self.log.warning(
"Unable to apply bright object mask: none supplied")
1229 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1230 mask = exposure.getMaskedImage().getMask()
1231 wcs = exposure.getWcs()
1232 plateScale = wcs.getPixelScale().asArcseconds()
1234 for rec
in brightObjectMasks:
1235 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1236 if rec[
"type"] ==
"box":
1237 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1238 width = rec[
"width"].asArcseconds()/plateScale
1239 height = rec[
"height"].asArcseconds()/plateScale
1242 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1245 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1246 spans = afwGeom.SpanSet(bbox)
1247 elif rec[
"type"] ==
"circle":
1248 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1249 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1251 self.log.warning(
"Unexpected region type %s at %s", rec[
"type"], center)
1253 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1256 """Set INEXACT_PSF mask plane.
1258 If any of the input images isn't represented in the coadd (due to
1259 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1265 Coadded exposure
's mask, modified in-place.
1267 mask.addMaskPlane("INEXACT_PSF")
1268 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1269 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1270 clipped = mask.getPlaneBitMask(
"CLIPPED")
1271 rejected = mask.getPlaneBitMask(
"REJECTED")
1272 array = mask.getArray()
1273 selected = array & (sensorEdge | clipped | rejected) > 0
1274 array[selected] |= inexactPsf
1277 def _makeArgumentParser(cls):
1278 """Create an argument parser.
1280 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1281 parser.add_id_argument("--id", cls.ConfigClass().coaddName +
"Coadd_"
1282 + cls.ConfigClass().warpType +
"Warp",
1283 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1284 ContainerClass=AssembleCoaddDataIdContainer)
1285 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1286 ContainerClass=SelectDataIdContainer)
1290 def _subBBoxIter(bbox, subregionSize):
1291 """Iterate over subregions of a bbox.
1296 Bounding box over which to iterate.
1303 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1304 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1305 the edges of ``bbox``, but it will never be empty.
1308 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1309 if subregionSize[0] < 1
or subregionSize[1] < 1:
1310 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1312 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1313 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1316 if subBBox.isEmpty():
1317 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1318 "colShift=%s, rowShift=%s" %
1319 (bbox, subregionSize, colShift, rowShift))
1323 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1328 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1330 Dictionary
with good visitIds
as the keys. Value ignored.
1334 filteredInputs : `list`
1335 Filtered
and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1337 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef
for inputRef
in inputs}
1339 for visit
in goodVisits.keys():
1340 if visit
in inputWarpDict:
1341 filteredInputs.append(inputWarpDict[visit])
1342 return filteredInputs
1346 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1350 """Make self.refList from self.idList.
1355 Results of parsing command-line (with ``butler``
and ``log`` elements).
1357 datasetType = namespace.config.coaddName + "Coadd"
1358 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1360 for dataId
in self.idList:
1362 for key
in keysCoadd:
1363 if key
not in dataId:
1364 raise RuntimeError(
"--id must include " + key)
1366 dataRef = namespace.butler.dataRef(
1367 datasetType=datasetType,
1370 self.refList.append(dataRef)
1374 """Function to count the number of pixels with a specific mask in a
1377 Find the intersection of mask & footprint. Count all pixels in the mask
1378 that are
in the intersection that have bitmask set but do
not have
1379 ignoreMask set. Return the count.
1384 Mask to define intersection region by.
1386 Footprint to define the intersection region by.
1388 Specific mask that we wish to count the number of occurances of.
1390 Pixels to
not consider.
1395 Count of number of pixels
in footprint
with specified mask.
1397 bbox = footprint.getBBox()
1398 bbox.clip(mask.getBBox(afwImage.PARENT))
1399 fp = afwImage.Mask(bbox)
1400 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1401 footprint.spans.setMask(fp, bitmask)
1402 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1403 (subMask.getArray() & ignoreMask) == 0).sum()
1407 """Configuration parameters for the SafeClipAssembleCoaddTask.
1409 clipDetection = pexConfig.ConfigurableField(
1410 target=SourceDetectionTask,
1411 doc="Detect sources on difference between unclipped and clipped coadd")
1412 minClipFootOverlap = pexConfig.Field(
1413 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1417 minClipFootOverlapSingle = pexConfig.Field(
1418 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1419 "clipped when only one visit overlaps",
1423 minClipFootOverlapDouble = pexConfig.Field(
1424 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1425 "clipped when two visits overlap",
1429 maxClipFootOverlapDouble = pexConfig.Field(
1430 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
1431 "considering two visits",
1435 minBigOverlap = pexConfig.Field(
1436 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1437 "when labeling clipped footprints",
1443 """Set default values for clipDetection.
1447 The numeric values for these configuration parameters were
1448 empirically determined, future work may further refine them.
1450 AssembleCoaddConfig.setDefaults(self)
1451 self.clipDetectionclipDetection.doTempLocalBackground = False
1452 self.
clipDetectionclipDetection.reEstimateBackground =
False
1453 self.
clipDetectionclipDetection.returnOriginalFootprints =
False
1459 self.
clipDetectionclipDetection.thresholdType =
"pixel_stdev"
1466 log.warning(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1467 "Ignoring doSigmaClip.")
1470 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1471 "(%s chosen). Please set statistic to MEAN."
1473 AssembleCoaddTask.ConfigClass.validate(self)
1477 """Assemble a coadded image from a set of coadded temporary exposures,
1478 being careful to clip & flag areas with potential artifacts.
1480 In ``AssembleCoaddTask``, we compute the coadd
as an clipped mean (i.e.,
1481 we clip outliers). The problem
with doing this
is that when computing the
1482 coadd PSF at a given location, individual visit PSFs
from visits
with
1483 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
1484 In this task, we correct
for this behavior by creating a new
1485 ``badMaskPlane``
'CLIPPED'. We populate this plane on the input
1486 coaddTempExps
and the final coadd where
1488 i. difference imaging suggests that there
is an outlier
and
1489 ii. this outlier appears on only one
or two images.
1491 Such regions will
not contribute to the final coadd. Furthermore, any
1492 routine to determine the coadd PSF can now be cognizant of clipped regions.
1493 Note that the algorithm implemented by this task
is preliminary
and works
1494 correctly
for HSC data. Parameter modifications
and or considerable
1495 redesigning of the algorithm
is likley required
for other surveys.
1497 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1498 "clipDetection" subtask
and also sub-classes ``AssembleCoaddTask``.
1499 You can retarget the ``SourceDetectionTask``
"clipDetection" subtask
1504 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1505 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``;
1506 see `baseDebug`
for more about ``debug.py`` files.
1507 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1508 The ``SourceDetectionTask``
"clipDetection" subtasks may support debug
1509 variables. See the documetation
for `SourceDetectionTask`
"clipDetection"
1510 for further information.
1514 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1515 images into a coadded image. The `SafeClipAssembleCoaddTask`
is invoked by
1516 running assembleCoadd.py *without* the flag
'--legacyCoadd'.
1518 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1519 and filter to be coadded (specified using
1520 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1521 along
with a list of coaddTempExps to attempt to coadd (specified using
1522 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1523 Only the coaddTempExps that cover the specified tract
and patch will be
1524 coadded. A list of the available optional arguments can be obtained by
1525 calling assembleCoadd.py
with the --help command line argument:
1527 .. code-block:: none
1529 assembleCoadd.py --help
1531 To demonstrate usage of the `SafeClipAssembleCoaddTask`
in the larger
1532 context of multi-band processing, we will generate the HSC-I & -R band
1533 coadds
from HSC engineering test data provided
in the ci_hsc package.
1534 To begin, assuming that the lsst stack has been already set up, we must
1535 set up the obs_subaru
and ci_hsc packages. This defines the environment
1536 variable $CI_HSC_DIR
and points at the location of the package. The raw
1537 HSC data live
in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1538 the coadds, we must first
1541 process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
1543 create a skymap that covers the area of the sky present
in the raw exposures
1544 - ``makeCoaddTempExp``
1545 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1547 We can perform all of these steps by running
1549 .. code-block:: none
1551 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1553 This will produce warped coaddTempExps
for each visit. To coadd the
1554 warped data, we call ``assembleCoadd.py``
as follows:
1556 .. code-block:: none
1558 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1559 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1560 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1561 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1562 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1563 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1564 --selectId visit=903988 ccd=24
1566 This will process the HSC-I band data. The results are written
in
1567 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1569 You may also choose to run:
1571 .. code-block:: none
1573 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1574 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1575 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1576 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1577 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1578 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1579 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1580 --selectId visit=903346 ccd=12
1582 to generate the coadd
for the HSC-R band
if you are interested
in following
1583 multiBand Coadd processing
as discussed
in ``pipeTasks_multiBand``.
1585 ConfigClass = SafeClipAssembleCoaddConfig
1586 _DefaultName = "safeClipAssembleCoadd"
1589 AssembleCoaddTask.__init__(self, *args, **kwargs)
1590 schema = afwTable.SourceTable.makeMinimalSchema()
1591 self.makeSubtask(
"clipDetection", schema=schema)
1593 @utils.inheritDoc(AssembleCoaddTask)
1595 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1596 """Assemble the coadd for a region.
1598 Compute the difference of coadds created with and without outlier
1599 rejection to identify coadd pixels that have outlier values
in some
1601 Detect clipped regions on the difference image
and mark these regions
1602 on the one
or two individual coaddTempExps where they occur
if there
1603 is significant overlap between the clipped region
and a source. This
1604 leaves us
with a set of footprints
from the difference image that have
1605 been identified
as having occured on just one
or two individual visits.
1606 However, these footprints were generated
from a difference image. It
1607 is conceivable
for a large diffuse source to have become broken up
1608 into multiple footprints acrosss the coadd difference
in this process.
1609 Determine the clipped region
from all overlapping footprints
from the
1610 detected sources
in each visit - these are big footprints.
1611 Combine the small
and big clipped footprints
and mark them on a new
1613 Generate the coadd using `AssembleCoaddTask.run` without outlier
1614 removal. Clipped footprints will no longer make it into the coadd
1615 because they are marked
in the new bad mask plane.
1619 args
and kwargs are passed but ignored
in order to match the call
1620 signature expected by the parent task.
1622 exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1623 mask = exp.getMaskedImage().getMask()
1624 mask.addMaskPlane("CLIPPED")
1626 result = self.
detectClipdetectClip(exp, tempExpRefList)
1628 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1630 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1631 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1633 bigFootprints = self.
detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1634 result.detectionFootprints, maskClipValue, maskDetValue,
1637 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1638 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1640 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1641 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1642 maskClip |= maskClipBig
1645 badMaskPlanes = self.config.badMaskPlanes[:]
1646 badMaskPlanes.append(
"CLIPPED")
1647 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1648 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1649 result.clipSpans, mask=badPixelMask)
1652 """Return an exposure that contains the difference between unclipped
1655 Generate a difference image between clipped
and unclipped coadds.
1656 Compute the difference image by subtracting an outlier-clipped coadd
1657 from an outlier-unclipped coadd. Return the difference image.
1661 skyInfo : `lsst.pipe.base.Struct`
1662 Patch geometry information,
from getSkyInfo
1663 tempExpRefList : `list`
1664 List of data reference to tempExp
1665 imageScalerList : `list`
1666 List of image scalers
1673 Difference image of unclipped
and clipped coadd wrapped
in an Exposure
1675 config = AssembleCoaddConfig()
1680 configIntersection = {k: getattr(self.config, k)
1681 for k, v
in self.config.toDict().items()
1682 if (k
in config.keys()
and k !=
"connections")}
1683 configIntersection[
'doInputMap'] =
False
1684 configIntersection[
'doNImage'] =
False
1685 config.update(**configIntersection)
1688 config.statistic =
'MEAN'
1689 task = AssembleCoaddTask(config=config)
1690 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1692 config.statistic =
'MEANCLIP'
1693 task = AssembleCoaddTask(config=config)
1694 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1696 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1697 coaddDiff -= coaddClip.getMaskedImage()
1698 exp = afwImage.ExposureF(coaddDiff)
1699 exp.setPsf(coaddMean.getPsf())
1703 """Detect clipped regions on an exposure and set the mask on the
1704 individual tempExp masks.
1706 Detect footprints in the difference image after smoothing the
1707 difference image
with a Gaussian kernal. Identify footprints that
1708 overlap
with one
or two input ``coaddTempExps`` by comparing the
1709 computed overlap fraction to thresholds set
in the config. A different
1710 threshold
is applied depending on the number of overlapping visits
1711 (restricted to one
or two). If the overlap exceeds the thresholds,
1712 the footprint
is considered
"CLIPPED" and is marked
as such on the
1713 coaddTempExp. Return a struct
with the clipped footprints, the indices
1714 of the ``coaddTempExps`` that end up overlapping
with the clipped
1715 footprints,
and a list of new masks
for the ``coaddTempExps``.
1720 Exposure to run detection on.
1721 tempExpRefList : `list`
1722 List of data reference to tempExp.
1726 result : `lsst.pipe.base.Struct`
1727 Result struct
with components:
1729 - ``clipFootprints``: list of clipped footprints.
1730 - ``clipIndices``: indices
for each ``clippedFootprint``
in
1732 - ``clipSpans``: List of dictionaries containing spanSet lists
1733 to clip. Each element contains the new maskplane name
1734 (
"CLIPPED")
as the key
and list of ``SpanSets``
as the value.
1735 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1736 compressed into footprints.
1738 mask = exp.getMaskedImage().getMask()
1739 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1740 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1742 fpSet.positive.merge(fpSet.negative)
1743 footprints = fpSet.positive
1744 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1745 ignoreMask = self.getBadPixelMask()
1749 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1752 visitDetectionFootprints = []
1754 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1755 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1756 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1759 for i, warpRef
in enumerate(tempExpRefList):
1760 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1761 immediate=
True).getMaskedImage().getMask()
1762 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1763 afwImage.PARENT,
True)
1764 maskVisitDet &= maskDetValue
1765 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1766 visitDetectionFootprints.append(visitFootprints)
1768 for j, footprint
in enumerate(footprints.getFootprints()):
1773 for j, footprint
in enumerate(footprints.getFootprints()):
1774 nPixel = footprint.getArea()
1777 for i
in range(len(tempExpRefList)):
1778 ignore = ignoreArr[i, j]
1779 overlapDet = overlapDetArr[i, j]
1780 totPixel = nPixel - ignore
1783 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1785 overlap.append(overlapDet/float(totPixel))
1788 overlap = numpy.array(overlap)
1789 if not len(overlap):
1796 if len(overlap) == 1:
1797 if overlap[0] > self.config.minClipFootOverlapSingle:
1802 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1803 if len(clipIndex) == 1:
1805 keepIndex = [clipIndex[0]]
1808 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1809 if len(clipIndex) == 2
and len(overlap) > 3:
1810 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1811 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1813 keepIndex = clipIndex
1818 for index
in keepIndex:
1819 globalIndex = indexList[index]
1820 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1822 clipIndices.append(numpy.array(indexList)[keepIndex])
1823 clipFootprints.append(footprint)
1825 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1826 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1828 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1829 maskClipValue, maskDetValue, coaddBBox):
1830 """Return individual warp footprints for large artifacts and append
1831 them to ``clipList`` in place.
1833 Identify big footprints composed of many sources
in the coadd
1834 difference that may have originated
in a large diffuse source
in the
1835 coadd. We do this by indentifying all clipped footprints that overlap
1836 significantly
with each source
in all the coaddTempExps.
1841 List of alt mask SpanSets
with clipping information. Modified.
1842 clipFootprints : `list`
1843 List of clipped footprints.
1844 clipIndices : `list`
1845 List of which entries
in tempExpClipList each footprint belongs to.
1847 Mask value of clipped pixels.
1849 Mask value of detected pixels.
1850 coaddBBox : `lsst.geom.Box`
1851 BBox of the coadd
and warps.
1855 bigFootprintsCoadd : `list`
1856 List of big footprints
1858 bigFootprintsCoadd = []
1859 ignoreMask = self.getBadPixelMask()
1860 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1861 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1862 for footprint
in visitFootprints.getFootprints():
1863 footprint.spans.setMask(maskVisitDet, maskDetValue)
1866 clippedFootprintsVisit = []
1867 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1868 if index
not in clipIndex:
1870 clippedFootprintsVisit.append(foot)
1871 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1872 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1874 bigFootprintsVisit = []
1875 for foot
in visitFootprints.getFootprints():
1876 if foot.getArea() < self.config.minBigOverlap:
1879 if nCount > self.config.minBigOverlap:
1880 bigFootprintsVisit.append(foot)
1881 bigFootprintsCoadd.append(foot)
1883 for footprint
in bigFootprintsVisit:
1884 clippedSpans[
"CLIPPED"].append(footprint.spans)
1886 return bigFootprintsCoadd
1890 psfMatchedWarps = pipeBase.connectionTypes.Input(
1891 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1892 "Only PSF-Matched Warps make sense for image subtraction. "
1893 "Therefore, they must be an additional declared input."),
1894 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1895 storageClass=
"ExposureF",
1896 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1900 templateCoadd = pipeBase.connectionTypes.Output(
1901 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1902 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1903 name=
"{outputCoaddName}CoaddPsfMatched",
1904 storageClass=
"ExposureF",
1905 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1910 if not config.assembleStaticSkyModel.doWrite:
1911 self.outputs.remove(
"templateCoadd")
1916 pipelineConnections=CompareWarpAssembleCoaddConnections):
1917 assembleStaticSkyModel = pexConfig.ConfigurableField(
1918 target=AssembleCoaddTask,
1919 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1920 " naive/first-iteration model of the static sky.",
1922 detect = pexConfig.ConfigurableField(
1923 target=SourceDetectionTask,
1924 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1926 detectTemplate = pexConfig.ConfigurableField(
1927 target=SourceDetectionTask,
1928 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1930 maskStreaks = pexConfig.ConfigurableField(
1931 target=MaskStreaksTask,
1932 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1933 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1936 streakMaskName = pexConfig.Field(
1939 doc=
"Name of mask bit used for streaks"
1941 maxNumEpochs = pexConfig.Field(
1942 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1943 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1944 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1945 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1946 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1947 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1948 "than transient and not masked.",
1952 maxFractionEpochsLow = pexConfig.RangeField(
1953 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1954 "Effective maxNumEpochs = "
1955 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1960 maxFractionEpochsHigh = pexConfig.RangeField(
1961 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1962 "Effective maxNumEpochs = "
1963 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1968 spatialThreshold = pexConfig.RangeField(
1969 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1970 "temporal criteria. If 0, clip all. If 1, clip none.",
1974 inclusiveMin=
True, inclusiveMax=
True
1976 doScaleWarpVariance = pexConfig.Field(
1977 doc=
"Rescale Warp variance plane using empirical noise?",
1981 scaleWarpVariance = pexConfig.ConfigurableField(
1982 target=ScaleVarianceTask,
1983 doc=
"Rescale variance on warps",
1985 doPreserveContainedBySource = pexConfig.Field(
1986 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1987 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1991 doPrefilterArtifacts = pexConfig.Field(
1992 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1993 "because they will be excluded anyway. This prevents them from contributing "
1994 "to the outlier epoch count image and potentially being labeled as persistant."
1995 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1999 prefilterArtifactsMaskPlanes = pexConfig.ListField(
2000 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
2002 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
2004 prefilterArtifactsRatio = pexConfig.Field(
2005 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
2009 doFilterMorphological = pexConfig.Field(
2010 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
2017 AssembleCoaddConfig.setDefaults(self)
2023 if "EDGE" in self.badMaskPlanes:
2024 self.badMaskPlanes.remove(
'EDGE')
2025 self.removeMaskPlanes.append(
'EDGE')
2034 self.
detectdetect.doTempLocalBackground =
False
2035 self.
detectdetect.reEstimateBackground =
False
2036 self.
detectdetect.returnOriginalFootprints =
False
2037 self.
detectdetect.thresholdPolarity =
"both"
2038 self.
detectdetect.thresholdValue = 5
2039 self.
detectdetect.minPixels = 4
2040 self.
detectdetect.isotropicGrow =
True
2041 self.
detectdetect.thresholdType =
"pixel_stdev"
2042 self.
detectdetect.nSigmaToGrow = 0.4
2048 self.
detectTemplatedetectTemplate.returnOriginalFootprints =
False
2053 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
2054 "Please set assembleStaticSkyModel.doNImage=False")
2057 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
2058 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
2059 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
2064 """Assemble a compareWarp coadded image from a set of warps
2065 by masking artifacts detected by comparing PSF-matched warps.
2067 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
2068 we clip outliers). The problem
with doing this
is that when computing the
2069 coadd PSF at a given location, individual visit PSFs
from visits
with
2070 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
2071 In this task, we correct
for this behavior by creating a new badMaskPlane
2072 'CLIPPED' which marks pixels
in the individual warps suspected to contain
2073 an artifact. We populate this plane on the input warps by comparing
2074 PSF-matched warps
with a PSF-matched median coadd which serves
as a
2075 model of the static sky. Any group of pixels that deviates
from the
2076 PSF-matched template coadd by more than config.detect.threshold sigma,
2077 is an artifact candidate. The candidates are then filtered to remove
2078 variable sources
and sources that are difficult to subtract such
as
2079 bright stars. This filter
is configured using the config parameters
2080 ``temporalThreshold``
and ``spatialThreshold``. The temporalThreshold
is
2081 the maximum fraction of epochs that the deviation can appear
in and still
2082 be considered an artifact. The spatialThreshold
is the maximum fraction of
2083 pixels
in the footprint of the deviation that appear
in other epochs
2084 (where other epochs
is defined by the temporalThreshold). If the deviant
2085 region meets this criteria of having a significant percentage of pixels
2086 that deviate
in only a few epochs, these pixels have the
'CLIPPED' bit
2087 set
in the mask. These regions will
not contribute to the final coadd.
2088 Furthermore, any routine to determine the coadd PSF can now be cognizant
2089 of clipped regions. Note that the algorithm implemented by this task
is
2090 preliminary
and works correctly
for HSC data. Parameter modifications
and
2091 or considerable redesigning of the algorithm
is likley required
for other
2094 ``CompareWarpAssembleCoaddTask`` sub-classes
2095 ``AssembleCoaddTask``
and instantiates ``AssembleCoaddTask``
2096 as a subtask to generate the TemplateCoadd (the model of the static sky).
2100 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2101 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``; see
2102 ``baseDebug``
for more about ``debug.py`` files.
2104 This task supports the following debug variables:
2107 If
True then save the Epoch Count Image
as a fits file
in the `figPath`
2109 Path to save the debug fits images
and figures
2111 For example, put something like:
2113 .. code-block:: python
2116 def DebugInfo(name):
2118 if name ==
"lsst.pipe.tasks.assembleCoadd":
2119 di.saveCountIm =
True
2120 di.figPath =
"/desired/path/to/debugging/output/images"
2124 into your ``debug.py`` file
and run ``assemebleCoadd.py``
with the
2125 ``--debug`` flag. Some subtasks may have their own debug variables;
2126 see individual Task documentation.
2130 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2131 coadded image. The ``CompareWarpAssembleCoaddTask``
is invoked by running
2132 ``assembleCoadd.py``
with the flag ``--compareWarpCoadd``.
2133 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2134 and filter to be coadded (specified using
2135 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2136 along
with a list of coaddTempExps to attempt to coadd (specified using
2137 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2138 Only the warps that cover the specified tract
and patch will be coadded.
2139 A list of the available optional arguments can be obtained by calling
2140 ``assembleCoadd.py``
with the ``--help`` command line argument:
2142 .. code-block:: none
2144 assembleCoadd.py --help
2146 To demonstrate usage of the ``CompareWarpAssembleCoaddTask``
in the larger
2147 context of multi-band processing, we will generate the HSC-I & -R band
2148 oadds
from HSC engineering test data provided
in the ``ci_hsc`` package.
2149 To begin, assuming that the lsst stack has been already set up, we must
2150 set up the ``obs_subaru``
and ``ci_hsc`` packages.
2151 This defines the environment variable ``$CI_HSC_DIR``
and points at the
2152 location of the package. The raw HSC data live
in the ``$CI_HSC_DIR/raw``
2153 directory. To begin assembling the coadds, we must first
2156 process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
2158 create a skymap that covers the area of the sky present
in the raw exposures
2160 warp the individual calibrated exposures to the tangent plane of the coadd
2162 We can perform all of these steps by running
2164 .. code-block:: none
2166 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2168 This will produce warped ``coaddTempExps``
for each visit. To coadd the
2169 warped data, we call ``assembleCoadd.py``
as follows:
2171 .. code-block:: none
2173 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2174 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2175 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2176 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2177 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2178 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2179 --selectId visit=903988 ccd=24
2181 This will process the HSC-I band data. The results are written
in
2182 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2184 ConfigClass = CompareWarpAssembleCoaddConfig
2185 _DefaultName = "compareWarpAssembleCoadd"
2188 AssembleCoaddTask.__init__(self, *args, **kwargs)
2189 self.makeSubtask(
"assembleStaticSkyModel")
2190 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2191 self.makeSubtask(
"detect", schema=detectionSchema)
2192 if self.config.doPreserveContainedBySource:
2193 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2194 if self.config.doScaleWarpVariance:
2195 self.makeSubtask(
"scaleWarpVariance")
2196 if self.config.doFilterMorphological:
2197 self.makeSubtask(
"maskStreaks")
2199 @utils.inheritDoc(AssembleCoaddTask)
2202 Generate a templateCoadd to use as a naive model of static sky to
2203 subtract
from PSF-Matched warps.
2207 result : `lsst.pipe.base.Struct`
2208 Result struct
with components:
2214 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2215 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2219 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2220 if self.config.assembleStaticSkyModel.doWrite:
2221 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2224 del outputRefs.templateCoadd
2225 del staticSkyModelOutputRefs.templateCoadd
2228 if 'nImage' in staticSkyModelOutputRefs.keys():
2229 del staticSkyModelOutputRefs.nImage
2231 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2232 staticSkyModelOutputRefs)
2233 if templateCoadd
is None:
2234 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2236 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2237 nImage=templateCoadd.nImage,
2238 warpRefList=templateCoadd.warpRefList,
2239 imageScalerList=templateCoadd.imageScalerList,
2240 weightList=templateCoadd.weightList)
2242 @utils.inheritDoc(AssembleCoaddTask)
2245 Generate a templateCoadd to use as a naive model of static sky to
2246 subtract
from PSF-Matched warps.
2250 result : `lsst.pipe.base.Struct`
2251 Result struct
with components:
2256 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2257 if templateCoadd
is None:
2258 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2260 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2261 nImage=templateCoadd.nImage,
2262 warpRefList=templateCoadd.warpRefList,
2263 imageScalerList=templateCoadd.imageScalerList,
2264 weightList=templateCoadd.weightList)
2266 def _noTemplateMessage(self, warpType):
2267 warpName = (warpType[0].upper() + warpType[1:])
2268 message =
"""No %(warpName)s warps were found to build the template coadd which is
2269 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2270 first either rerun makeCoaddTempExp
with config.make%(warpName)s=
True or
2271 coaddDriver
with config.makeCoadTempExp.make%(warpName)s=
True, before assembleCoadd.
2273 Alternatively, to use another algorithm
with existing warps, retarget the CoaddDriverConfig to
2274 another algorithm like:
2277 config.assemble.retarget(SafeClipAssembleCoaddTask)
2278 """ % {"warpName": warpName}
2281 @utils.inheritDoc(AssembleCoaddTask)
2283 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2284 supplementaryData, *args, **kwargs):
2285 """Assemble the coadd.
2287 Find artifacts and apply them to the warps
' masks creating a list of
2288 alternative masks with a new
"CLIPPED" plane
and updated
"NO_DATA"
2289 plane. Then
pass these alternative masks to the base
class's `run`
2292 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2293 that must contain a ``templateCoadd`` that serves
as the
2294 model of the static sky.
2300 dataIds = [ref.dataId
for ref
in tempExpRefList]
2301 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
2303 if dataIds != psfMatchedDataIds:
2304 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2305 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2306 psfMatchedDataIds, dataIds)
2307 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2308 psfMatchedDataIds, dataIds)
2311 spanSetMaskList = self.
findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2312 supplementaryData.warpRefList,
2313 supplementaryData.imageScalerList)
2315 badMaskPlanes = self.config.badMaskPlanes[:]
2316 badMaskPlanes.append(
"CLIPPED")
2317 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2319 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2320 spanSetMaskList, mask=badPixelMask)
2324 self.
applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2328 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2334 altMaskList : `list`
2335 List of Dicts containing ``spanSet`` lists.
2336 Each element contains the new mask plane name (e.g. "CLIPPED
2337 and/
or "NO_DATA")
as the key,
and list of ``SpanSets`` to apply to
2340 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE",
"INEXACT_PSF"])
2341 for visitMask
in altMaskList:
2342 if "EDGE" in visitMask:
2343 for spanSet
in visitMask[
'EDGE']:
2344 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2349 Loop through warps twice. The first loop builds a map with the count
2350 of how many epochs each pixel deviates
from the templateCoadd by more
2351 than ``config.chiThreshold`` sigma. The second loop takes each
2352 difference image
and filters the artifacts detected
in each using
2353 count map to filter out variable sources
and sources that are
2354 difficult to subtract cleanly.
2359 Exposure to serve
as model of static sky.
2360 tempExpRefList : `list`
2361 List of data references to warps.
2362 imageScalerList : `list`
2363 List of image scalers.
2368 List of dicts containing information about CLIPPED
2369 (i.e., artifacts), NO_DATA,
and EDGE pixels.
2372 self.log.debug("Generating Count Image, and mask lists.")
2373 coaddBBox = templateCoadd.getBBox()
2374 slateIm = afwImage.ImageU(coaddBBox)
2375 epochCountImage = afwImage.ImageU(coaddBBox)
2376 nImage = afwImage.ImageU(coaddBBox)
2377 spanSetArtifactList = []
2378 spanSetNoDataMaskList = []
2379 spanSetEdgeList = []
2380 spanSetBadMorphoList = []
2381 badPixelMask = self.getBadPixelMask()
2384 templateCoadd.mask.clearAllMaskPlanes()
2386 if self.config.doPreserveContainedBySource:
2387 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2389 templateFootprints =
None
2391 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2393 if warpDiffExp
is not None:
2395 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2396 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2397 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2398 fpSet.positive.merge(fpSet.negative)
2399 footprints = fpSet.positive
2401 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2404 if self.config.doPrefilterArtifacts:
2408 self.detect.clearMask(warpDiffExp.mask)
2409 for spans
in spanSetList:
2410 spans.setImage(slateIm, 1, doClip=
True)
2411 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
2412 epochCountImage += slateIm
2414 if self.config.doFilterMorphological:
2415 maskName = self.config.streakMaskName
2416 _ = self.maskStreaks.
run(warpDiffExp)
2417 streakMask = warpDiffExp.mask
2418 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2419 streakMask.getPlaneBitMask(maskName)).split()
2425 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2426 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2427 nansMask.setXY0(warpDiffExp.getXY0())
2428 edgeMask = warpDiffExp.mask
2429 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2430 edgeMask.getPlaneBitMask(
"EDGE")).split()
2434 nansMask = afwImage.MaskX(coaddBBox, 1)
2436 spanSetEdgeMask = []
2439 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2441 spanSetNoDataMaskList.append(spanSetNoDataMask)
2442 spanSetArtifactList.append(spanSetList)
2443 spanSetEdgeList.append(spanSetEdgeMask)
2444 if self.config.doFilterMorphological:
2445 spanSetBadMorphoList.append(spanSetStreak)
2448 path = self.
_dataRef2DebugPath_dataRef2DebugPath(
"epochCountIm", tempExpRefList[0], coaddLevel=
True)
2449 epochCountImage.writeFits(path)
2451 for i, spanSetList
in enumerate(spanSetArtifactList):
2453 filteredSpanSetList = self.
filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2455 spanSetArtifactList[i] = filteredSpanSetList
2456 if self.config.doFilterMorphological:
2457 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2460 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2461 altMasks.append({
'CLIPPED': artifacts,
2467 """Remove artifact candidates covered by bad mask plane.
2469 Any future editing of the candidate list that does not depend on
2470 temporal information should go
in this method.
2474 spanSetList : `list`
2475 List of SpanSets representing artifact candidates.
2477 Exposure containing mask planes used to prefilter.
2481 returnSpanSetList : `list`
2482 List of SpanSets
with artifacts.
2484 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2485 goodArr = (exp.mask.array & badPixelMask) == 0
2486 returnSpanSetList = []
2487 bbox = exp.getBBox()
2488 x0, y0 = exp.getXY0()
2489 for i, span
in enumerate(spanSetList):
2490 y, x = span.clippedTo(bbox).indices()
2491 yIndexLocal = numpy.array(y) - y0
2492 xIndexLocal = numpy.array(x) - x0
2493 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2494 if goodRatio > self.config.prefilterArtifactsRatio:
2495 returnSpanSetList.append(span)
2496 return returnSpanSetList
2498 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2499 """Filter artifact candidates.
2503 spanSetList : `list`
2504 List of SpanSets representing artifact candidates.
2506 Image of accumulated number of warpDiff detections.
2508 Image of the accumulated number of total epochs contributing.
2512 maskSpanSetList : `list`
2513 List of SpanSets with artifacts.
2516 maskSpanSetList = []
2517 x0, y0 = epochCountImage.getXY0()
2518 for i, span
in enumerate(spanSetList):
2519 y, x = span.indices()
2520 yIdxLocal = [y1 - y0
for y1
in y]
2521 xIdxLocal = [x1 - x0
for x1
in x]
2522 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2523 totalN = nImage.array[yIdxLocal, xIdxLocal]
2526 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2527 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2528 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2529 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2530 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2531 & (outlierN <= effectiveMaxNumEpochs))
2532 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2533 if percentBelowThreshold > self.config.spatialThreshold:
2534 maskSpanSetList.append(span)
2536 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2538 filteredMaskSpanSetList = []
2539 for span
in maskSpanSetList:
2541 for footprint
in footprintsToExclude.positive.getFootprints():
2542 if footprint.spans.contains(span):
2546 filteredMaskSpanSetList.append(span)
2547 maskSpanSetList = filteredMaskSpanSetList
2549 return maskSpanSetList
2551 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2552 """Fetch a warp from the butler and return a warpDiff.
2557 Butler dataRef for the warp.
2559 An image scaler object.
2561 Exposure to be substracted
from the scaled warp.
2566 Exposure of the image difference between the warp
and template.
2574 warpName = self.getTempExpDatasetName(
'psfMatched')
2575 if not isinstance(warpRef, DeferredDatasetHandle):
2576 if not warpRef.datasetExists(warpName):
2577 self.log.warning(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2579 warp = warpRef.get(datasetType=warpName, immediate=
True)
2581 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2582 mi = warp.getMaskedImage()
2583 if self.config.doScaleWarpVariance:
2585 self.scaleWarpVariance.
run(mi)
2586 except Exception
as exc:
2587 self.log.warning(
"Unable to rescale variance of warp (%s); leaving it as-is", exc)
2588 mi -= templateCoadd.getMaskedImage()
2591 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2592 """Return a path to which to write debugging output.
2594 Creates a hyphen-delimited string of dataId values for simple filenames.
2599 Prefix
for filename.
2601 Butler dataRef to make the path
from.
2602 coaddLevel : `bool`, optional.
2603 If
True, include only coadd-level keys (e.g.,
'tract',
'patch',
2604 'filter', but no
'visit').
2609 Path
for debugging output.
2612 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2614 keys = warpRef.dataId.keys()
2615 keyList = sorted(keys, reverse=
True)
2617 filename =
"%s-%s.fits" % (prefix,
'-'.join([
str(warpRef.dataId[k])
for k
in keyList]))
2618 return os.path.join(directory, filename)
def makeDataRefList(self, namespace)
def __init__(self, *config=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def applyAltEdgeMask(self, mask, altMaskList)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def __init__(self, *args, **kwargs)
def detectClip(self, exp, tempExpRefList)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def setInexactPsf(self, mask)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeCoaddSuffix(warpType="direct")
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")