39import lsst.utils
as utils
41from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42from .interpImage
import InterpImageTask
43from .scaleZeroPoint
import ScaleZeroPointTask
44from .coaddHelpers
import groupPatchExposures, getGroupDataRef
45from .scaleVariance
import ScaleVarianceTask
46from .maskStreaks
import MaskStreaksTask
47from .healSparseMapping
import HealSparseInputMapTask
49from lsst.daf.butler
import DeferredDatasetHandle
50from lsst.utils.timer
import timeMethod
52__all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
53 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
54 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
56log = logging.getLogger(__name__)
60 dimensions=(
"tract",
"patch",
"band",
"skymap"),
61 defaultTemplates={
"inputCoaddName":
"deep",
62 "outputCoaddName":
"deep",
64 "warpTypeSuffix":
""}):
66 inputWarps = pipeBase.connectionTypes.Input(
67 doc=(
"Input list of warps to be assemebled i.e. stacked."
68 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
69 name=
"{inputCoaddName}Coadd_{warpType}Warp",
70 storageClass=
"ExposureF",
71 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
75 skyMap = pipeBase.connectionTypes.Input(
76 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78 storageClass=
"SkyMap",
79 dimensions=(
"skymap", ),
81 selectedVisits = pipeBase.connectionTypes.Input(
82 doc=
"Selected visits to be coadded.",
83 name=
"{outputCoaddName}Visits",
84 storageClass=
"StructuredDataDict",
85 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band")
87 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
88 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
90 name=
"brightObjectMask",
91 storageClass=
"ObjectMaskCatalog",
92 dimensions=(
"tract",
"patch",
"skymap",
"band"),
94 coaddExposure = pipeBase.connectionTypes.Output(
95 doc=
"Output coadded exposure, produced by stacking input warps",
96 name=
"{outputCoaddName}Coadd{warpTypeSuffix}",
97 storageClass=
"ExposureF",
98 dimensions=(
"tract",
"patch",
"skymap",
"band"),
100 nImage = pipeBase.connectionTypes.Output(
101 doc=
"Output image of number of input images per pixel",
102 name=
"{outputCoaddName}Coadd_nImage",
103 storageClass=
"ImageU",
104 dimensions=(
"tract",
"patch",
"skymap",
"band"),
106 inputMap = pipeBase.connectionTypes.Output(
107 doc=
"Output healsparse map of input images",
108 name=
"{outputCoaddName}Coadd_inputMap",
109 storageClass=
"HealSparseMap",
110 dimensions=(
"tract",
"patch",
"skymap",
"band"),
113 def __init__(self, *, config=None):
114 super().__init__(config=config)
119 templateValues = {name: getattr(config.connections, name)
for name
in self.defaultTemplates}
120 templateValues[
'warpType'] = config.warpType
122 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
123 for name
in self.allConnections}
124 self._typeNameToVarName = {v: k
for k, v
in self._nameOverrides.items()}
127 if not config.doMaskBrightObjects:
128 self.prerequisiteInputs.remove(
"brightObjectMask")
130 if not config.doSelectVisits:
131 self.inputs.remove(
"selectedVisits")
133 if not config.doNImage:
134 self.outputs.remove(
"nImage")
136 if not self.config.doInputMap:
137 self.outputs.remove(
"inputMap")
140class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
141 pipelineConnections=AssembleCoaddConnections):
142 """Configuration parameters for the `AssembleCoaddTask`.
146 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
147 only set the bitplane config.brightObjectMaskName. To make this useful you
148 *must* also configure the flags.pixel algorithm,
for example by adding
152 config.measurement.plugins[
"base_PixelFlags"].masksFpCenter.append(
"BRIGHT_OBJECT")
153 config.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere.append(
"BRIGHT_OBJECT")
155 to your measureCoaddSources.py
and forcedPhotCoadd.py config overrides.
157 warpType = pexConfig.Field(
158 doc="Warp name: one of 'direct' or 'psfMatched'",
162 subregionSize = pexConfig.ListField(
164 doc=
"Width, height of stack subregion size; "
165 "make small enough that a full stack of images will fit into memory at once.",
167 default=(2000, 2000),
169 statistic = pexConfig.Field(
171 doc=
"Main stacking statistic for aggregating over the epochs.",
174 doOnlineForMean = pexConfig.Field(
176 doc=
"Perform online coaddition when statistic=\"MEAN\" to save memory?",
179 doSigmaClip = pexConfig.Field(
181 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
184 sigmaClip = pexConfig.Field(
186 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
189 clipIter = pexConfig.Field(
191 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
194 calcErrorFromInputVariance = pexConfig.Field(
196 doc=
"Calculate coadd variance from input variance by stacking statistic."
197 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
200 scaleZeroPoint = pexConfig.ConfigurableField(
201 target=ScaleZeroPointTask,
202 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
204 doInterp = pexConfig.Field(
205 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
209 interpImage = pexConfig.ConfigurableField(
210 target=InterpImageTask,
211 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
213 doWrite = pexConfig.Field(
214 doc=
"Persist coadd?",
218 doNImage = pexConfig.Field(
219 doc=
"Create image of number of contributing exposures for each pixel",
223 doUsePsfMatchedPolygons = pexConfig.Field(
224 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
228 maskPropagationThresholds = pexConfig.DictField(
231 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
232 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
233 "would have contributed exceeds this value."),
234 default={
"SAT": 0.1},
236 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
237 doc=
"Mask planes to remove before coadding")
238 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
239 doc=
"Set mask and flag bits for bright objects?")
240 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
241 doc=
"Name of mask bit used for bright objects")
242 coaddPsf = pexConfig.ConfigField(
243 doc=
"Configuration for CoaddPsf",
244 dtype=measAlg.CoaddPsfConfig,
246 doAttachTransmissionCurve = pexConfig.Field(
247 dtype=bool, default=
False, optional=
False,
248 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
249 "(requires all input Exposures to have TransmissionCurves).")
251 hasFakes = pexConfig.Field(
254 doc=
"Should be set to True if fake sources have been inserted into the input data."
256 doSelectVisits = pexConfig.Field(
257 doc=
"Coadd only visits selected by a SelectVisitsTask",
261 doInputMap = pexConfig.Field(
262 doc=
"Create a bitwise map of coadd inputs",
266 inputMapper = pexConfig.ConfigurableField(
267 doc=
"Input map creation subtask.",
268 target=HealSparseInputMapTask,
271 def setDefaults(self):
272 super().setDefaults()
273 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
280 log.warning(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
281 self.warpType =
'psfMatched'
282 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
283 log.warning(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
284 self.statistic =
"MEANCLIP"
285 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
286 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
287 "compute and set a non-zero coadd variance estimate." % (self.statistic))
289 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
290 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
291 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
292 if str(k)
not in unstackableStats]
293 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
294 % (self.statistic, stackableStats))
297class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
298 """Assemble a coadded image from a set of warps (coadded temporary exposures).
300 We want to assemble a coadded image from a set of Warps (also called
301 coadded temporary exposures
or ``coaddTempExps``).
302 Each input Warp covers a patch on the sky
and corresponds to a single
303 run/visit/exposure of the covered patch. We provide the task
with a list
304 of Warps (``selectDataList``)
from which it selects Warps that cover the
305 specified patch (pointed at by ``dataRef``).
306 Each Warp that goes into a coadd will typically have an independent
307 photometric zero-point. Therefore, we must scale each Warp to set it to
308 a common photometric zeropoint. WarpType may be one of
'direct' or
309 'psfMatched',
and the boolean configs `config.makeDirect`
and
310 `config.makePsfMatched` set which of the warp types will be coadded.
311 The coadd
is computed
as a mean
with optional outlier rejection.
312 Criteria
for outlier rejection are set
in `AssembleCoaddConfig`.
313 Finally, Warps can have bad
'NaN' pixels which received no input
from the
314 source calExps. We interpolate over these bad (NaN) pixels.
316 `AssembleCoaddTask` uses several sub-tasks. These are
318 - `ScaleZeroPointTask`
319 - create
and use an ``imageScaler`` object to scale the photometric zeropoint
for each Warp
321 - interpolate across bad pixels (NaN)
in the final coadd
323 You can retarget these subtasks
if you wish.
327 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
328 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``; see
329 `baseDebug`
for more about ``debug.py`` files. `AssembleCoaddTask` has
330 no debug variables of its own. Some of the subtasks may support debug
331 variables. See the documentation
for the subtasks
for further information.
335 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
336 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
337 with the flag
'--legacyCoadd'. Usage of assembleCoadd.py expects two
338 inputs: a data reference to the tract patch
and filter to be coadded,
and
339 a list of Warps to attempt to coadd. These are specified using ``--id``
and
340 ``--selectId``, respectively:
344 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
345 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
347 Only the Warps that cover the specified tract
and patch will be coadded.
348 A list of the available optional arguments can be obtained by calling
349 ``assembleCoadd.py``
with the ``--help`` command line argument:
353 assembleCoadd.py --help
355 To demonstrate usage of the `AssembleCoaddTask`
in the larger context of
356 multi-band processing, we will generate the HSC-I & -R band coadds
from
357 HSC engineering test data provided
in the ``ci_hsc`` package. To begin,
358 assuming that the lsst stack has been already set up, we must set up the
359 obs_subaru
and ``ci_hsc`` packages. This defines the environment variable
360 ``$CI_HSC_DIR``
and points at the location of the package. The raw HSC
361 data live
in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
362 coadds, we must first
365 - process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
367 - create a skymap that covers the area of the sky present
in the raw exposures
369 - warp the individual calibrated exposures to the tangent plane of the coadd
371 We can perform all of these steps by running
375 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
377 This will produce warped exposures
for each visit. To coadd the warped
378 data, we call assembleCoadd.py
as follows:
382 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
383 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
384 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
385 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
386 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
387 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
388 --selectId visit=903988 ccd=24
390 that will process the HSC-I band data. The results are written
in
391 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
393 You may also choose to run:
397 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
398 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
399 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
400 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
401 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
402 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
403 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
404 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
406 to generate the coadd
for the HSC-R band
if you are interested
in
407 following multiBand Coadd processing
as discussed
in `pipeTasks_multiBand`
408 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
409 rather than `AssembleCoaddTask` to make the coadd.
411 ConfigClass = AssembleCoaddConfig
412 _DefaultName = "assembleCoadd"
414 def __init__(self, *args, **kwargs):
417 argNames = [
"config",
"name",
"parentTask",
"log"]
418 kwargs.update({k: v
for k, v
in zip(argNames, args)})
419 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
420 "PipelineTask will not take positional args" % argNames, FutureWarning)
422 super().__init__(**kwargs)
423 self.makeSubtask(
"interpImage")
424 self.makeSubtask(
"scaleZeroPoint")
426 if self.config.doMaskBrightObjects:
427 mask = afwImage.Mask()
429 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
430 except pexExceptions.LsstCppException:
431 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
432 mask.getMaskPlaneDict().keys())
435 if self.config.doInputMap:
436 self.makeSubtask(
"inputMapper")
438 self.warpType = self.config.warpType
440 @utils.inheritDoc(pipeBase.PipelineTask)
441 def runQuantum(self, butlerQC, inputRefs, outputRefs):
446 Assemble a coadd from a set of Warps.
448 PipelineTask (Gen3) entry point to Coadd a set of Warps.
449 Analogous to `runDataRef`, it prepares all the data products to be
450 passed to `run`,
and processes the results before returning a struct
451 of results to be written out. AssembleCoadd cannot fit all Warps
in memory.
452 Therefore, its inputs are accessed subregion by subregion
453 by the Gen3 `DeferredDatasetHandle` that
is analagous to the Gen2
454 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
455 correspond to an update
in `runDataRef`
while both entry points
458 inputData = butlerQC.get(inputRefs)
462 skyMap = inputData[
"skyMap"]
463 outputDataId = butlerQC.quantum.dataId
466 tractId=outputDataId[
'tract'],
467 patchId=outputDataId[
'patch'])
469 if self.config.doSelectVisits:
470 warpRefList = self.filterWarps(inputData[
'inputWarps'], inputData[
'selectedVisits'])
472 warpRefList = inputData[
'inputWarps']
475 inputs = self.prepareInputs(warpRefList)
476 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
477 self.getTempExpDatasetName(self.warpType))
478 if len(inputs.tempExpRefList) == 0:
479 raise pipeBase.NoWorkFound(
"No coadd temporary exposures found")
481 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
482 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
483 inputs.weightList, supplementaryData=supplementaryData)
485 inputData.setdefault(
'brightObjectMask',
None)
486 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
488 if self.config.doWrite:
489 butlerQC.put(retStruct, outputRefs)
493 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
494 """Assemble a coadd from a set of Warps.
496 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
497 Compute weights to be applied to each Warp and
498 find scalings to match the photometric zeropoint to a reference Warp.
499 Assemble the Warps using `run`. Interpolate over NaNs
and
500 optionally write the coadd to disk. Return the coadded exposure.
505 Data reference defining the patch
for coaddition
and the
506 reference Warp (
if ``config.autoReference=
False``).
507 Used to access the following data products:
508 - ``self.config.coaddName +
"Coadd_skyMap"``
509 - ``self.config.coaddName +
"Coadd_ + <warpType> + "Warp
"`` (optionally)
510 - ``self.config.coaddName + "Coadd"``
511 selectDataList : `list`
512 List of data references to Calexps. Data to be coadded will be
513 selected
from this list based on overlap
with the patch defined
514 by dataRef, grouped by visit,
and converted to a list of data
517 List of data references to Warps to be coadded.
518 Note: `warpRefList`
is just the new name
for `tempExpRefList`.
522 retStruct : `lsst.pipe.base.Struct`
523 Result struct
with components:
525 - ``coaddExposure``: coadded exposure (``Exposure``).
526 - ``nImage``: exposure count image (``Image``).
528 if selectDataList
and warpRefList:
529 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, "
530 "and which to use is ambiguous. Please pass only one.")
532 skyInfo = self.getSkyInfo(dataRef)
533 if warpRefList
is None:
534 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
535 if len(calExpRefList) == 0:
536 self.log.warning(
"No exposures to coadd")
538 self.log.info(
"Coadding %d exposures", len(calExpRefList))
540 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
542 inputData = self.prepareInputs(warpRefList)
543 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
544 self.getTempExpDatasetName(self.warpType))
545 if len(inputData.tempExpRefList) == 0:
546 self.log.warning(
"No coadd temporary exposures found")
549 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
551 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
552 inputData.weightList, supplementaryData=supplementaryData)
554 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None
555 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
557 if self.config.doWrite:
558 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
559 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
561 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
562 self.log.info(
"Persisting %s", coaddDatasetName)
563 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
564 if self.config.doNImage
and retStruct.nImage
is not None:
565 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
570 """Interpolate over missing data and mask bright stars.
575 The coadded exposure to process.
576 dataRef : `lsst.daf.persistence.ButlerDataRef`
577 Butler data reference for supplementary data.
579 if self.config.doInterp:
580 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
582 varArray = coaddExposure.variance.array
583 with numpy.errstate(invalid=
"ignore"):
584 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
586 if self.config.doMaskBrightObjects:
587 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
590 """Make additional inputs to run() specific to subclasses (Gen2)
592 Duplicates interface of `runDataRef` method
593 Available to be implemented by subclasses only if they need the
594 coadd dataRef
for performing preliminary processing before
595 assembling the coadd.
599 dataRef : `lsst.daf.persistence.ButlerDataRef`
600 Butler data reference
for supplementary data.
601 selectDataList : `list` (optional)
602 Optional List of data references to Calexps.
603 warpRefList : `list` (optional)
604 Optional List of data references to Warps.
606 return pipeBase.Struct()
609 """Make additional inputs to run() specific to subclasses (Gen3)
611 Duplicates interface of `runQuantum` method.
612 Available to be implemented by subclasses only if they need the
613 coadd dataRef
for performing preliminary processing before
614 assembling the coadd.
618 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
619 Gen3 Butler object
for fetching additional data products before
620 running the Task specialized
for quantum being processed
621 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
622 Attributes are the names of the connections describing input dataset types.
623 Values are DatasetRefs that task consumes
for corresponding dataset type.
624 DataIds are guaranteed to match data objects
in ``inputData``.
625 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
626 Attributes are the names of the connections describing output dataset types.
627 Values are DatasetRefs that task
is to produce
628 for corresponding dataset type.
630 return pipeBase.Struct()
633 """Generate list data references corresponding to warped exposures
634 that lie within the patch to be coadded.
639 Data reference for patch.
640 calExpRefList : `list`
641 List of data references
for input calexps.
645 tempExpRefList : `list`
646 List of Warp/CoaddTempExp data references.
648 butler = patchRef.getButler()
649 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
650 self.getTempExpDatasetName(self.warpType))
651 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
652 g, groupData.keys) for
653 g
in groupData.groups.keys()]
654 return tempExpRefList
657 """Prepare the input warps for coaddition by measuring the weight for
658 each warp and the scaling
for the photometric zero point.
660 Each Warp has its own photometric zeropoint
and background variance.
661 Before coadding these Warps together, compute a scale factor to
662 normalize the photometric zeropoint
and compute the weight
for each Warp.
667 List of data references to tempExp
671 result : `lsst.pipe.base.Struct`
672 Result struct
with components:
674 - ``tempExprefList``: `list` of data references to tempExp.
675 - ``weightList``: `list` of weightings.
676 - ``imageScalerList``: `list` of image scalers.
678 statsCtrl = afwMath.StatisticsControl()
679 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
680 statsCtrl.setNumIter(self.config.clipIter)
681 statsCtrl.setAndMask(self.getBadPixelMask())
682 statsCtrl.setNanSafe(True)
689 tempExpName = self.getTempExpDatasetName(self.warpType)
690 for tempExpRef
in refList:
693 if not isinstance(tempExpRef, DeferredDatasetHandle):
694 if not tempExpRef.datasetExists(tempExpName):
695 self.log.warning(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
698 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
700 if numpy.isnan(tempExp.image.array).all():
702 maskedImage = tempExp.getMaskedImage()
703 imageScaler = self.scaleZeroPoint.computeImageScaler(
708 imageScaler.scaleMaskedImage(maskedImage)
709 except Exception
as e:
710 self.log.warning(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
712 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
713 afwMath.MEANCLIP, statsCtrl)
714 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
715 weight = 1.0 / float(meanVar)
716 if not numpy.isfinite(weight):
717 self.log.warning(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
719 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
724 tempExpRefList.append(tempExpRef)
725 weightList.append(weight)
726 imageScalerList.append(imageScaler)
728 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
729 imageScalerList=imageScalerList)
732 """Prepare the statistics for coadding images.
736 mask : `int`, optional
737 Bit mask value to exclude from coaddition.
741 stats : `lsst.pipe.base.Struct`
742 Statistics structure
with the following fields:
744 - ``statsCtrl``: Statistics control object
for coadd
746 - ``statsFlags``: Statistic
for coadd (`lsst.afw.math.Property`)
749 mask = self.getBadPixelMask()
750 statsCtrl = afwMath.StatisticsControl()
751 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
752 statsCtrl.setNumIter(self.config.clipIter)
753 statsCtrl.setAndMask(mask)
754 statsCtrl.setNanSafe(
True)
755 statsCtrl.setWeighted(
True)
756 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
757 for plane, threshold
in self.config.maskPropagationThresholds.items():
758 bit = afwImage.Mask.getMaskPlane(plane)
759 statsCtrl.setMaskPropagationThreshold(bit, threshold)
760 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
761 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
764 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
765 altMaskList=None, mask=None, supplementaryData=None):
766 """Assemble a coadd from input warps
768 Assemble the coadd using the provided list of coaddTempExps. Since
769 the full coadd covers a patch (a large area), the assembly is
770 performed over small areas on the image at a time
in order to
771 conserve memory usage. Iterate over subregions within the outer
772 bbox of the patch using `assembleSubregion` to stack the corresponding
773 subregions
from the coaddTempExps
with the statistic specified.
774 Set the edge bits the coadd mask based on the weight map.
778 skyInfo : `lsst.pipe.base.Struct`
779 Struct
with geometric information about the patch.
780 tempExpRefList : `list`
781 List of data references to Warps (previously called CoaddTempExps).
782 imageScalerList : `list`
783 List of image scalers.
786 altMaskList : `list`, optional
787 List of alternate masks to use rather than those stored
with
789 mask : `int`, optional
790 Bit mask value to exclude
from coaddition.
791 supplementaryData : lsst.pipe.base.Struct, optional
792 Struct
with additional data products needed to assemble coadd.
793 Only used by subclasses that implement `makeSupplementaryData`
798 result : `lsst.pipe.base.Struct`
799 Result struct
with components:
803 - ``inputMap``: bit-wise map of inputs,
if requested.
804 - ``warpRefList``: input list of refs to the warps (
805 ``lsst.daf.butler.DeferredDatasetHandle``
or
806 ``lsst.daf.persistence.ButlerDataRef``)
808 - ``imageScalerList``: input list of image scalers (unmodified)
809 - ``weightList``: input list of weights (unmodified)
811 tempExpName = self.getTempExpDatasetName(self.warpType)
812 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
813 stats = self.prepareStats(mask=mask)
815 if altMaskList
is None:
816 altMaskList = [
None]*len(tempExpRefList)
818 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
819 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
820 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
821 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
822 coaddMaskedImage = coaddExposure.getMaskedImage()
823 subregionSizeArr = self.config.subregionSize
824 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
826 if self.config.doNImage:
827 nImage = afwImage.ImageU(skyInfo.bbox)
832 if self.config.doInputMap:
833 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
835 coaddExposure.getInfo().getCoaddInputs().ccds)
837 if self.config.doOnlineForMean
and self.config.statistic ==
"MEAN":
839 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
840 weightList, altMaskList, stats.ctrl,
842 except Exception
as e:
843 self.log.fatal(
"Cannot compute online coadd %s", e)
845 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
847 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
848 weightList, altMaskList, stats.flags, stats.ctrl,
850 except Exception
as e:
851 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
854 if self.config.doInputMap:
855 self.inputMapper.finalize_ccd_input_map_mask()
856 inputMap = self.inputMapper.ccd_input_map
860 self.setInexactPsf(coaddMaskedImage.getMask())
863 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
864 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
865 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
866 weightList=weightList, inputMap=inputMap)
869 """Set the metadata for the coadd.
871 This basic implementation sets the filter from the first input.
876 The target exposure
for the coadd.
877 tempExpRefList : `list`
878 List of data references to tempExp.
882 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
883 tempExpName = self.getTempExpDatasetName(self.warpType)
889 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
891 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
894 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
895 for tempExpRef
in tempExpRefList]
896 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
900 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
901 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
902 coaddInputs.ccds.reserve(numCcds)
903 coaddInputs.visits.reserve(len(tempExpList))
905 for tempExp, weight
in zip(tempExpList, weightList):
906 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
908 if self.config.doUsePsfMatchedPolygons:
909 self.shrinkValidPolygons(coaddInputs)
911 coaddInputs.visits.sort()
912 coaddInputs.ccds.sort()
913 if self.warpType ==
"psfMatched":
918 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
919 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
920 for modelPsf
in modelPsfList]
921 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
923 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
924 self.config.coaddPsf.makeControl())
925 coaddExposure.setPsf(psf)
926 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
927 coaddExposure.getWcs())
928 coaddExposure.getInfo().setApCorrMap(apCorrMap)
929 if self.config.doAttachTransmissionCurve:
930 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
931 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
934 altMaskList, statsFlags, statsCtrl, nImage=None):
935 """Assemble the coadd for a sub-region.
937 For each coaddTempExp, check for (
and swap
in) an alternative mask
938 if one
is passed. Remove mask planes listed
in
939 `config.removeMaskPlanes`. Finally, stack the actual exposures using
940 `lsst.afw.math.statisticsStack`
with the statistic specified by
941 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
for
942 a mean-stack
or `lsst.afw.math.MEANCLIP`
for outlier rejection using
943 an N-sigma clipped mean where N
and iterations are specified by
944 statsCtrl. Assign the stacked subregion back to the coadd.
949 The target exposure
for the coadd.
950 bbox : `lsst.geom.Box`
952 tempExpRefList : `list`
953 List of data reference to tempExp.
954 imageScalerList : `list`
955 List of image scalers.
959 List of alternate masks to use rather than those stored
with
960 tempExp,
or None. Each element
is dict
with keys = mask plane
961 name to which to add the spans.
962 statsFlags : `lsst.afw.math.Property`
963 Property object
for statistic
for coadd.
965 Statistics control object
for coadd.
966 nImage : `lsst.afw.image.ImageU`, optional
967 Keeps track of exposure count
for each pixel.
969 self.log.debug("Computing coadd over %s", bbox)
970 tempExpName = self.getTempExpDatasetName(self.warpType)
971 coaddExposure.mask.addMaskPlane(
"REJECTED")
972 coaddExposure.mask.addMaskPlane(
"CLIPPED")
973 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
974 maskMap = self.setRejectedMaskMapping(statsCtrl)
975 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
977 if nImage
is not None:
978 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
979 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
981 if isinstance(tempExpRef, DeferredDatasetHandle):
983 exposure = tempExpRef.get(parameters={
'bbox': bbox})
986 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
988 maskedImage = exposure.getMaskedImage()
989 mask = maskedImage.getMask()
990 if altMask
is not None:
991 self.applyAltMaskPlanes(mask, altMask)
992 imageScaler.scaleMaskedImage(maskedImage)
996 if nImage
is not None:
997 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
998 if self.config.removeMaskPlanes:
999 self.removeMaskPlanes(maskedImage)
1000 maskedImageList.append(maskedImage)
1002 if self.config.doInputMap:
1003 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1004 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1006 with self.timer(
"stack"):
1007 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
1010 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
1011 if nImage
is not None:
1012 nImage.assign(subNImage, bbox)
1015 altMaskList, statsCtrl, nImage=None):
1016 """Assemble the coadd using the "online" method.
1018 This method takes a running sum of images and weights to save memory.
1019 It only works
for MEAN statistics.
1024 The target exposure
for the coadd.
1025 tempExpRefList : `list`
1026 List of data reference to tempExp.
1027 imageScalerList : `list`
1028 List of image scalers.
1031 altMaskList : `list`
1032 List of alternate masks to use rather than those stored
with
1033 tempExp,
or None. Each element
is dict
with keys = mask plane
1034 name to which to add the spans.
1036 Statistics control object
for coadd
1037 nImage : `lsst.afw.image.ImageU`, optional
1038 Keeps track of exposure count
for each pixel.
1040 self.log.debug("Computing online coadd.")
1041 tempExpName = self.getTempExpDatasetName(self.warpType)
1042 coaddExposure.mask.addMaskPlane(
"REJECTED")
1043 coaddExposure.mask.addMaskPlane(
"CLIPPED")
1044 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
1045 maskMap = self.setRejectedMaskMapping(statsCtrl)
1046 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
1048 bbox = coaddExposure.maskedImage.getBBox()
1050 stacker = AccumulatorMeanStack(
1051 coaddExposure.image.array.shape,
1052 statsCtrl.getAndMask(),
1053 mask_threshold_dict=thresholdDict,
1055 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
1056 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
1057 compute_n_image=(nImage
is not None)
1060 for tempExpRef, imageScaler, altMask, weight
in zip(tempExpRefList,
1064 if isinstance(tempExpRef, DeferredDatasetHandle):
1066 exposure = tempExpRef.get()
1069 exposure = tempExpRef.get(tempExpName)
1071 maskedImage = exposure.getMaskedImage()
1072 mask = maskedImage.getMask()
1073 if altMask
is not None:
1074 self.applyAltMaskPlanes(mask, altMask)
1075 imageScaler.scaleMaskedImage(maskedImage)
1076 if self.config.removeMaskPlanes:
1077 self.removeMaskPlanes(maskedImage)
1079 stacker.add_masked_image(maskedImage, weight=weight)
1081 if self.config.doInputMap:
1082 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1083 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1085 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
1087 if nImage
is not None:
1088 nImage.array[:, :] = stacker.n_image
1091 """Unset the mask of an image for mask planes specified in the config.
1096 The masked image to be modified.
1098 mask = maskedImage.getMask()
1099 for maskPlane
in self.config.removeMaskPlanes:
1101 mask &= ~mask.getPlaneBitMask(maskPlane)
1102 except pexExceptions.InvalidParameterError:
1103 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
1107 def setRejectedMaskMapping(statsCtrl):
1108 """Map certain mask planes of the warps to new planes for the coadd.
1110 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1111 or CLIPPED, set it to REJECTED on the coadd.
1112 If a pixel
is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1113 If a pixel
is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1118 Statistics control object
for coadd
1122 maskMap : `list` of `tuple` of `int`
1123 A list of mappings of mask planes of the warped exposures to
1124 mask planes of the coadd.
1126 edge = afwImage.Mask.getPlaneBitMask("EDGE")
1127 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
1128 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
1129 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1130 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
1131 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
1136 """Apply in place alt mask formatted as SpanSets to a mask.
1142 altMaskSpans : `dict`
1143 SpanSet lists to apply. Each element contains the new mask
1144 plane name (e.g. "CLIPPED and/or "NO_DATA
") as the key,
1145 and list of SpanSets to apply to the mask.
1152 if self.config.doUsePsfMatchedPolygons:
1153 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1158 for spanSet
in altMaskSpans[
'NO_DATA']:
1159 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1161 for plane, spanSetList
in altMaskSpans.items():
1162 maskClipValue = mask.addMaskPlane(plane)
1163 for spanSet
in spanSetList:
1164 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1168 """Shrink coaddInputs' ccds' ValidPolygons in place.
1170 Either modify each ccd's validPolygon in place, or if CoaddInputs
1171 does not have a validPolygon, create one
from its bbox.
1175 coaddInputs : `lsst.afw.image.coaddInputs`
1179 for ccd
in coaddInputs.ccds:
1180 polyOrig = ccd.getValidPolygon()
1181 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1182 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1184 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1186 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1187 ccd.setValidPolygon(validPolygon)
1190 """Retrieve the bright object masks.
1192 Returns None on failure.
1202 Bright object mask
from the Butler object,
or None if it cannot
1206 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1207 except Exception
as e:
1208 self.log.warning(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1212 """Set the bright object masks.
1217 Exposure under consideration.
1219 Data identifier dict for patch.
1221 Table of bright objects to mask.
1224 if brightObjectMasks
is None:
1225 self.log.warning(
"Unable to apply bright object mask: none supplied")
1227 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1228 mask = exposure.getMaskedImage().getMask()
1229 wcs = exposure.getWcs()
1230 plateScale = wcs.getPixelScale().asArcseconds()
1232 for rec
in brightObjectMasks:
1233 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1234 if rec[
"type"] ==
"box":
1235 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1236 width = rec[
"width"].asArcseconds()/plateScale
1237 height = rec[
"height"].asArcseconds()/plateScale
1240 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1243 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1244 spans = afwGeom.SpanSet(bbox)
1245 elif rec[
"type"] ==
"circle":
1246 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1247 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1249 self.log.warning(
"Unexpected region type %s at %s", rec[
"type"], center)
1251 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1254 """Set INEXACT_PSF mask plane.
1256 If any of the input images isn't represented in the coadd (due to
1257 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1263 Coadded exposure
's mask, modified in-place.
1265 mask.addMaskPlane("INEXACT_PSF")
1266 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1267 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1268 clipped = mask.getPlaneBitMask(
"CLIPPED")
1269 rejected = mask.getPlaneBitMask(
"REJECTED")
1270 array = mask.getArray()
1271 selected = array & (sensorEdge | clipped | rejected) > 0
1272 array[selected] |= inexactPsf
1275 def _makeArgumentParser(cls):
1276 """Create an argument parser.
1278 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1279 parser.add_id_argument("--id", cls.ConfigClass().coaddName +
"Coadd_"
1280 + cls.ConfigClass().warpType +
"Warp",
1281 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1282 ContainerClass=AssembleCoaddDataIdContainer)
1283 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1284 ContainerClass=SelectDataIdContainer)
1288 def _subBBoxIter(bbox, subregionSize):
1289 """Iterate over subregions of a bbox.
1294 Bounding box over which to iterate.
1301 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1302 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1303 the edges of ``bbox``, but it will never be empty.
1306 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1307 if subregionSize[0] < 1
or subregionSize[1] < 1:
1308 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1310 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1311 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1314 if subBBox.isEmpty():
1315 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1316 "colShift=%s, rowShift=%s" %
1317 (bbox, subregionSize, colShift, rowShift))
1321 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1326 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1328 Dictionary
with good visitIds
as the keys. Value ignored.
1332 filteredInputs : `list`
1333 Filtered
and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1335 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef
for inputRef
in inputs}
1337 for visit
in goodVisits.keys():
1338 if visit
in inputWarpDict:
1339 filteredInputs.append(inputWarpDict[visit])
1340 return filteredInputs
1344 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1348 """Make self.refList from self.idList.
1353 Results of parsing command-line (with ``butler``
and ``log`` elements).
1355 datasetType = namespace.config.coaddName + "Coadd"
1356 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1358 for dataId
in self.idList:
1360 for key
in keysCoadd:
1361 if key
not in dataId:
1362 raise RuntimeError(
"--id must include " + key)
1364 dataRef = namespace.butler.dataRef(
1365 datasetType=datasetType,
1368 self.refList.append(dataRef)
1372 """Function to count the number of pixels with a specific mask in a
1375 Find the intersection of mask & footprint. Count all pixels in the mask
1376 that are
in the intersection that have bitmask set but do
not have
1377 ignoreMask set. Return the count.
1382 Mask to define intersection region by.
1384 Footprint to define the intersection region by.
1386 Specific mask that we wish to count the number of occurances of.
1388 Pixels to
not consider.
1393 Count of number of pixels
in footprint
with specified mask.
1395 bbox = footprint.getBBox()
1396 bbox.clip(mask.getBBox(afwImage.PARENT))
1397 fp = afwImage.Mask(bbox)
1398 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1399 footprint.spans.setMask(fp, bitmask)
1400 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1401 (subMask.getArray() & ignoreMask) == 0).sum()
1405 """Configuration parameters for the SafeClipAssembleCoaddTask.
1407 clipDetection = pexConfig.ConfigurableField(
1408 target=SourceDetectionTask,
1409 doc="Detect sources on difference between unclipped and clipped coadd")
1410 minClipFootOverlap = pexConfig.Field(
1411 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1415 minClipFootOverlapSingle = pexConfig.Field(
1416 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1417 "clipped when only one visit overlaps",
1421 minClipFootOverlapDouble = pexConfig.Field(
1422 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1423 "clipped when two visits overlap",
1427 maxClipFootOverlapDouble = pexConfig.Field(
1428 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
1429 "considering two visits",
1433 minBigOverlap = pexConfig.Field(
1434 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1435 "when labeling clipped footprints",
1441 """Set default values for clipDetection.
1445 The numeric values for these configuration parameters were
1446 empirically determined, future work may further refine them.
1448 AssembleCoaddConfig.setDefaults(self)
1449 self.clipDetectionclipDetection.doTempLocalBackground = False
1450 self.
clipDetectionclipDetection.reEstimateBackground =
False
1451 self.
clipDetectionclipDetection.returnOriginalFootprints =
False
1457 self.
clipDetectionclipDetection.thresholdType =
"pixel_stdev"
1464 log.warning(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1465 "Ignoring doSigmaClip.")
1468 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1469 "(%s chosen). Please set statistic to MEAN."
1471 AssembleCoaddTask.ConfigClass.validate(self)
1475 """Assemble a coadded image from a set of coadded temporary exposures,
1476 being careful to clip & flag areas with potential artifacts.
1478 In ``AssembleCoaddTask``, we compute the coadd
as an clipped mean (i.e.,
1479 we clip outliers). The problem
with doing this
is that when computing the
1480 coadd PSF at a given location, individual visit PSFs
from visits
with
1481 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
1482 In this task, we correct
for this behavior by creating a new
1483 ``badMaskPlane``
'CLIPPED'. We populate this plane on the input
1484 coaddTempExps
and the final coadd where
1486 i. difference imaging suggests that there
is an outlier
and
1487 ii. this outlier appears on only one
or two images.
1489 Such regions will
not contribute to the final coadd. Furthermore, any
1490 routine to determine the coadd PSF can now be cognizant of clipped regions.
1491 Note that the algorithm implemented by this task
is preliminary
and works
1492 correctly
for HSC data. Parameter modifications
and or considerable
1493 redesigning of the algorithm
is likley required
for other surveys.
1495 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1496 "clipDetection" subtask
and also sub-classes ``AssembleCoaddTask``.
1497 You can retarget the ``SourceDetectionTask``
"clipDetection" subtask
1502 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1503 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``;
1504 see `baseDebug`
for more about ``debug.py`` files.
1505 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1506 The ``SourceDetectionTask``
"clipDetection" subtasks may support debug
1507 variables. See the documetation
for `SourceDetectionTask`
"clipDetection"
1508 for further information.
1512 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1513 images into a coadded image. The `SafeClipAssembleCoaddTask`
is invoked by
1514 running assembleCoadd.py *without* the flag
'--legacyCoadd'.
1516 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1517 and filter to be coadded (specified using
1518 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1519 along
with a list of coaddTempExps to attempt to coadd (specified using
1520 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1521 Only the coaddTempExps that cover the specified tract
and patch will be
1522 coadded. A list of the available optional arguments can be obtained by
1523 calling assembleCoadd.py
with the --help command line argument:
1525 .. code-block:: none
1527 assembleCoadd.py --help
1529 To demonstrate usage of the `SafeClipAssembleCoaddTask`
in the larger
1530 context of multi-band processing, we will generate the HSC-I & -R band
1531 coadds
from HSC engineering test data provided
in the ci_hsc package.
1532 To begin, assuming that the lsst stack has been already set up, we must
1533 set up the obs_subaru
and ci_hsc packages. This defines the environment
1534 variable $CI_HSC_DIR
and points at the location of the package. The raw
1535 HSC data live
in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1536 the coadds, we must first
1539 process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
1541 create a skymap that covers the area of the sky present
in the raw exposures
1542 - ``makeCoaddTempExp``
1543 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1545 We can perform all of these steps by running
1547 .. code-block:: none
1549 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1551 This will produce warped coaddTempExps
for each visit. To coadd the
1552 warped data, we call ``assembleCoadd.py``
as follows:
1554 .. code-block:: none
1556 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1557 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1558 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1559 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1560 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1561 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1562 --selectId visit=903988 ccd=24
1564 This will process the HSC-I band data. The results are written
in
1565 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1567 You may also choose to run:
1569 .. code-block:: none
1571 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1572 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1573 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1574 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1575 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1576 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1577 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1578 --selectId visit=903346 ccd=12
1580 to generate the coadd
for the HSC-R band
if you are interested
in following
1581 multiBand Coadd processing
as discussed
in ``pipeTasks_multiBand``.
1583 ConfigClass = SafeClipAssembleCoaddConfig
1584 _DefaultName = "safeClipAssembleCoadd"
1587 AssembleCoaddTask.__init__(self, *args, **kwargs)
1588 schema = afwTable.SourceTable.makeMinimalSchema()
1589 self.makeSubtask(
"clipDetection", schema=schema)
1591 @utils.inheritDoc(AssembleCoaddTask)
1593 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1594 """Assemble the coadd for a region.
1596 Compute the difference of coadds created with and without outlier
1597 rejection to identify coadd pixels that have outlier values
in some
1599 Detect clipped regions on the difference image
and mark these regions
1600 on the one
or two individual coaddTempExps where they occur
if there
1601 is significant overlap between the clipped region
and a source. This
1602 leaves us
with a set of footprints
from the difference image that have
1603 been identified
as having occured on just one
or two individual visits.
1604 However, these footprints were generated
from a difference image. It
1605 is conceivable
for a large diffuse source to have become broken up
1606 into multiple footprints acrosss the coadd difference
in this process.
1607 Determine the clipped region
from all overlapping footprints
from the
1608 detected sources
in each visit - these are big footprints.
1609 Combine the small
and big clipped footprints
and mark them on a new
1611 Generate the coadd using `AssembleCoaddTask.run` without outlier
1612 removal. Clipped footprints will no longer make it into the coadd
1613 because they are marked
in the new bad mask plane.
1617 args
and kwargs are passed but ignored
in order to match the call
1618 signature expected by the parent task.
1620 exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1621 mask = exp.getMaskedImage().getMask()
1622 mask.addMaskPlane("CLIPPED")
1624 result = self.
detectClipdetectClip(exp, tempExpRefList)
1626 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1628 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1629 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1631 bigFootprints = self.
detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1632 result.detectionFootprints, maskClipValue, maskDetValue,
1635 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1636 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1638 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1639 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1640 maskClip |= maskClipBig
1643 badMaskPlanes = self.config.badMaskPlanes[:]
1644 badMaskPlanes.append(
"CLIPPED")
1645 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1646 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1647 result.clipSpans, mask=badPixelMask)
1650 """Return an exposure that contains the difference between unclipped
1653 Generate a difference image between clipped
and unclipped coadds.
1654 Compute the difference image by subtracting an outlier-clipped coadd
1655 from an outlier-unclipped coadd. Return the difference image.
1659 skyInfo : `lsst.pipe.base.Struct`
1660 Patch geometry information,
from getSkyInfo
1661 tempExpRefList : `list`
1662 List of data reference to tempExp
1663 imageScalerList : `list`
1664 List of image scalers
1671 Difference image of unclipped
and clipped coadd wrapped
in an Exposure
1673 config = AssembleCoaddConfig()
1678 configIntersection = {k: getattr(self.config, k)
1679 for k, v
in self.config.toDict().items()
1680 if (k
in config.keys()
and k !=
"connections")}
1681 configIntersection[
'doInputMap'] =
False
1682 configIntersection[
'doNImage'] =
False
1683 config.update(**configIntersection)
1686 config.statistic =
'MEAN'
1687 task = AssembleCoaddTask(config=config)
1688 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1690 config.statistic =
'MEANCLIP'
1691 task = AssembleCoaddTask(config=config)
1692 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1694 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1695 coaddDiff -= coaddClip.getMaskedImage()
1696 exp = afwImage.ExposureF(coaddDiff)
1697 exp.setPsf(coaddMean.getPsf())
1701 """Detect clipped regions on an exposure and set the mask on the
1702 individual tempExp masks.
1704 Detect footprints in the difference image after smoothing the
1705 difference image
with a Gaussian kernal. Identify footprints that
1706 overlap
with one
or two input ``coaddTempExps`` by comparing the
1707 computed overlap fraction to thresholds set
in the config. A different
1708 threshold
is applied depending on the number of overlapping visits
1709 (restricted to one
or two). If the overlap exceeds the thresholds,
1710 the footprint
is considered
"CLIPPED" and is marked
as such on the
1711 coaddTempExp. Return a struct
with the clipped footprints, the indices
1712 of the ``coaddTempExps`` that end up overlapping
with the clipped
1713 footprints,
and a list of new masks
for the ``coaddTempExps``.
1718 Exposure to run detection on.
1719 tempExpRefList : `list`
1720 List of data reference to tempExp.
1724 result : `lsst.pipe.base.Struct`
1725 Result struct
with components:
1727 - ``clipFootprints``: list of clipped footprints.
1728 - ``clipIndices``: indices
for each ``clippedFootprint``
in
1730 - ``clipSpans``: List of dictionaries containing spanSet lists
1731 to clip. Each element contains the new maskplane name
1732 (
"CLIPPED")
as the key
and list of ``SpanSets``
as the value.
1733 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1734 compressed into footprints.
1736 mask = exp.getMaskedImage().getMask()
1737 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1738 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1740 fpSet.positive.merge(fpSet.negative)
1741 footprints = fpSet.positive
1742 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1743 ignoreMask = self.getBadPixelMask()
1747 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1750 visitDetectionFootprints = []
1752 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1753 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1754 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1757 for i, warpRef
in enumerate(tempExpRefList):
1758 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1759 immediate=
True).getMaskedImage().getMask()
1760 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1761 afwImage.PARENT,
True)
1762 maskVisitDet &= maskDetValue
1763 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1764 visitDetectionFootprints.append(visitFootprints)
1766 for j, footprint
in enumerate(footprints.getFootprints()):
1771 for j, footprint
in enumerate(footprints.getFootprints()):
1772 nPixel = footprint.getArea()
1775 for i
in range(len(tempExpRefList)):
1776 ignore = ignoreArr[i, j]
1777 overlapDet = overlapDetArr[i, j]
1778 totPixel = nPixel - ignore
1781 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1783 overlap.append(overlapDet/float(totPixel))
1786 overlap = numpy.array(overlap)
1787 if not len(overlap):
1794 if len(overlap) == 1:
1795 if overlap[0] > self.config.minClipFootOverlapSingle:
1800 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1801 if len(clipIndex) == 1:
1803 keepIndex = [clipIndex[0]]
1806 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1807 if len(clipIndex) == 2
and len(overlap) > 3:
1808 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1809 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1811 keepIndex = clipIndex
1816 for index
in keepIndex:
1817 globalIndex = indexList[index]
1818 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1820 clipIndices.append(numpy.array(indexList)[keepIndex])
1821 clipFootprints.append(footprint)
1823 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1824 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1826 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1827 maskClipValue, maskDetValue, coaddBBox):
1828 """Return individual warp footprints for large artifacts and append
1829 them to ``clipList`` in place.
1831 Identify big footprints composed of many sources
in the coadd
1832 difference that may have originated
in a large diffuse source
in the
1833 coadd. We do this by indentifying all clipped footprints that overlap
1834 significantly
with each source
in all the coaddTempExps.
1839 List of alt mask SpanSets
with clipping information. Modified.
1840 clipFootprints : `list`
1841 List of clipped footprints.
1842 clipIndices : `list`
1843 List of which entries
in tempExpClipList each footprint belongs to.
1845 Mask value of clipped pixels.
1847 Mask value of detected pixels.
1848 coaddBBox : `lsst.geom.Box`
1849 BBox of the coadd
and warps.
1853 bigFootprintsCoadd : `list`
1854 List of big footprints
1856 bigFootprintsCoadd = []
1857 ignoreMask = self.getBadPixelMask()
1858 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1859 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1860 for footprint
in visitFootprints.getFootprints():
1861 footprint.spans.setMask(maskVisitDet, maskDetValue)
1864 clippedFootprintsVisit = []
1865 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1866 if index
not in clipIndex:
1868 clippedFootprintsVisit.append(foot)
1869 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1870 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1872 bigFootprintsVisit = []
1873 for foot
in visitFootprints.getFootprints():
1874 if foot.getArea() < self.config.minBigOverlap:
1877 if nCount > self.config.minBigOverlap:
1878 bigFootprintsVisit.append(foot)
1879 bigFootprintsCoadd.append(foot)
1881 for footprint
in bigFootprintsVisit:
1882 clippedSpans[
"CLIPPED"].append(footprint.spans)
1884 return bigFootprintsCoadd
1888 psfMatchedWarps = pipeBase.connectionTypes.Input(
1889 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1890 "Only PSF-Matched Warps make sense for image subtraction. "
1891 "Therefore, they must be an additional declared input."),
1892 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1893 storageClass=
"ExposureF",
1894 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1898 templateCoadd = pipeBase.connectionTypes.Output(
1899 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1900 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1901 name=
"{outputCoaddName}CoaddPsfMatched",
1902 storageClass=
"ExposureF",
1903 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1908 if not config.assembleStaticSkyModel.doWrite:
1909 self.outputs.remove(
"templateCoadd")
1914 pipelineConnections=CompareWarpAssembleCoaddConnections):
1915 assembleStaticSkyModel = pexConfig.ConfigurableField(
1916 target=AssembleCoaddTask,
1917 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1918 " naive/first-iteration model of the static sky.",
1920 detect = pexConfig.ConfigurableField(
1921 target=SourceDetectionTask,
1922 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1924 detectTemplate = pexConfig.ConfigurableField(
1925 target=SourceDetectionTask,
1926 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1928 maskStreaks = pexConfig.ConfigurableField(
1929 target=MaskStreaksTask,
1930 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1931 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1934 streakMaskName = pexConfig.Field(
1937 doc=
"Name of mask bit used for streaks"
1939 maxNumEpochs = pexConfig.Field(
1940 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1941 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1942 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1943 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1944 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1945 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1946 "than transient and not masked.",
1950 maxFractionEpochsLow = pexConfig.RangeField(
1951 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1952 "Effective maxNumEpochs = "
1953 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1958 maxFractionEpochsHigh = pexConfig.RangeField(
1959 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1960 "Effective maxNumEpochs = "
1961 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1966 spatialThreshold = pexConfig.RangeField(
1967 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1968 "temporal criteria. If 0, clip all. If 1, clip none.",
1972 inclusiveMin=
True, inclusiveMax=
True
1974 doScaleWarpVariance = pexConfig.Field(
1975 doc=
"Rescale Warp variance plane using empirical noise?",
1979 scaleWarpVariance = pexConfig.ConfigurableField(
1980 target=ScaleVarianceTask,
1981 doc=
"Rescale variance on warps",
1983 doPreserveContainedBySource = pexConfig.Field(
1984 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1985 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1989 doPrefilterArtifacts = pexConfig.Field(
1990 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1991 "because they will be excluded anyway. This prevents them from contributing "
1992 "to the outlier epoch count image and potentially being labeled as persistant."
1993 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1997 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1998 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
2000 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
2002 prefilterArtifactsRatio = pexConfig.Field(
2003 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
2007 doFilterMorphological = pexConfig.Field(
2008 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
2015 AssembleCoaddConfig.setDefaults(self)
2021 if "EDGE" in self.badMaskPlanes:
2022 self.badMaskPlanes.remove(
'EDGE')
2023 self.removeMaskPlanes.append(
'EDGE')
2032 self.
detectdetect.doTempLocalBackground =
False
2033 self.
detectdetect.reEstimateBackground =
False
2034 self.
detectdetect.returnOriginalFootprints =
False
2035 self.
detectdetect.thresholdPolarity =
"both"
2036 self.
detectdetect.thresholdValue = 5
2037 self.
detectdetect.minPixels = 4
2038 self.
detectdetect.isotropicGrow =
True
2039 self.
detectdetect.thresholdType =
"pixel_stdev"
2040 self.
detectdetect.nSigmaToGrow = 0.4
2046 self.
detectTemplatedetectTemplate.returnOriginalFootprints =
False
2051 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
2052 "Please set assembleStaticSkyModel.doNImage=False")
2055 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
2056 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
2057 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
2062 """Assemble a compareWarp coadded image from a set of warps
2063 by masking artifacts detected by comparing PSF-matched warps.
2065 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
2066 we clip outliers). The problem
with doing this
is that when computing the
2067 coadd PSF at a given location, individual visit PSFs
from visits
with
2068 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
2069 In this task, we correct
for this behavior by creating a new badMaskPlane
2070 'CLIPPED' which marks pixels
in the individual warps suspected to contain
2071 an artifact. We populate this plane on the input warps by comparing
2072 PSF-matched warps
with a PSF-matched median coadd which serves
as a
2073 model of the static sky. Any group of pixels that deviates
from the
2074 PSF-matched template coadd by more than config.detect.threshold sigma,
2075 is an artifact candidate. The candidates are then filtered to remove
2076 variable sources
and sources that are difficult to subtract such
as
2077 bright stars. This filter
is configured using the config parameters
2078 ``temporalThreshold``
and ``spatialThreshold``. The temporalThreshold
is
2079 the maximum fraction of epochs that the deviation can appear
in and still
2080 be considered an artifact. The spatialThreshold
is the maximum fraction of
2081 pixels
in the footprint of the deviation that appear
in other epochs
2082 (where other epochs
is defined by the temporalThreshold). If the deviant
2083 region meets this criteria of having a significant percentage of pixels
2084 that deviate
in only a few epochs, these pixels have the
'CLIPPED' bit
2085 set
in the mask. These regions will
not contribute to the final coadd.
2086 Furthermore, any routine to determine the coadd PSF can now be cognizant
2087 of clipped regions. Note that the algorithm implemented by this task
is
2088 preliminary
and works correctly
for HSC data. Parameter modifications
and
2089 or considerable redesigning of the algorithm
is likley required
for other
2092 ``CompareWarpAssembleCoaddTask`` sub-classes
2093 ``AssembleCoaddTask``
and instantiates ``AssembleCoaddTask``
2094 as a subtask to generate the TemplateCoadd (the model of the static sky).
2098 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2099 flag ``-d`` to
import ``debug.py``
from your ``PYTHONPATH``; see
2100 ``baseDebug``
for more about ``debug.py`` files.
2102 This task supports the following debug variables:
2105 If
True then save the Epoch Count Image
as a fits file
in the `figPath`
2107 Path to save the debug fits images
and figures
2109 For example, put something like:
2111 .. code-block:: python
2114 def DebugInfo(name):
2116 if name ==
"lsst.pipe.tasks.assembleCoadd":
2117 di.saveCountIm =
True
2118 di.figPath =
"/desired/path/to/debugging/output/images"
2122 into your ``debug.py`` file
and run ``assemebleCoadd.py``
with the
2123 ``--debug`` flag. Some subtasks may have their own debug variables;
2124 see individual Task documentation.
2128 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2129 coadded image. The ``CompareWarpAssembleCoaddTask``
is invoked by running
2130 ``assembleCoadd.py``
with the flag ``--compareWarpCoadd``.
2131 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2132 and filter to be coadded (specified using
2133 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2134 along
with a list of coaddTempExps to attempt to coadd (specified using
2135 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2136 Only the warps that cover the specified tract
and patch will be coadded.
2137 A list of the available optional arguments can be obtained by calling
2138 ``assembleCoadd.py``
with the ``--help`` command line argument:
2140 .. code-block:: none
2142 assembleCoadd.py --help
2144 To demonstrate usage of the ``CompareWarpAssembleCoaddTask``
in the larger
2145 context of multi-band processing, we will generate the HSC-I & -R band
2146 oadds
from HSC engineering test data provided
in the ``ci_hsc`` package.
2147 To begin, assuming that the lsst stack has been already set up, we must
2148 set up the ``obs_subaru``
and ``ci_hsc`` packages.
2149 This defines the environment variable ``$CI_HSC_DIR``
and points at the
2150 location of the package. The raw HSC data live
in the ``$CI_HSC_DIR/raw``
2151 directory. To begin assembling the coadds, we must first
2154 process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
2156 create a skymap that covers the area of the sky present
in the raw exposures
2158 warp the individual calibrated exposures to the tangent plane of the coadd
2160 We can perform all of these steps by running
2162 .. code-block:: none
2164 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2166 This will produce warped ``coaddTempExps``
for each visit. To coadd the
2167 warped data, we call ``assembleCoadd.py``
as follows:
2169 .. code-block:: none
2171 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2172 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2173 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2174 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2175 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2176 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2177 --selectId visit=903988 ccd=24
2179 This will process the HSC-I band data. The results are written
in
2180 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2182 ConfigClass = CompareWarpAssembleCoaddConfig
2183 _DefaultName = "compareWarpAssembleCoadd"
2186 AssembleCoaddTask.__init__(self, *args, **kwargs)
2187 self.makeSubtask(
"assembleStaticSkyModel")
2188 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2189 self.makeSubtask(
"detect", schema=detectionSchema)
2190 if self.config.doPreserveContainedBySource:
2191 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2192 if self.config.doScaleWarpVariance:
2193 self.makeSubtask(
"scaleWarpVariance")
2194 if self.config.doFilterMorphological:
2195 self.makeSubtask(
"maskStreaks")
2197 @utils.inheritDoc(AssembleCoaddTask)
2200 Generate a templateCoadd to use as a naive model of static sky to
2201 subtract
from PSF-Matched warps.
2205 result : `lsst.pipe.base.Struct`
2206 Result struct
with components:
2212 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2213 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2217 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2218 if self.config.assembleStaticSkyModel.doWrite:
2219 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2222 del outputRefs.templateCoadd
2223 del staticSkyModelOutputRefs.templateCoadd
2226 if 'nImage' in staticSkyModelOutputRefs.keys():
2227 del staticSkyModelOutputRefs.nImage
2229 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2230 staticSkyModelOutputRefs)
2231 if templateCoadd
is None:
2232 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2234 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2235 nImage=templateCoadd.nImage,
2236 warpRefList=templateCoadd.warpRefList,
2237 imageScalerList=templateCoadd.imageScalerList,
2238 weightList=templateCoadd.weightList)
2240 @utils.inheritDoc(AssembleCoaddTask)
2243 Generate a templateCoadd to use as a naive model of static sky to
2244 subtract
from PSF-Matched warps.
2248 result : `lsst.pipe.base.Struct`
2249 Result struct
with components:
2254 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2255 if templateCoadd
is None:
2256 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2258 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2259 nImage=templateCoadd.nImage,
2260 warpRefList=templateCoadd.warpRefList,
2261 imageScalerList=templateCoadd.imageScalerList,
2262 weightList=templateCoadd.weightList)
2264 def _noTemplateMessage(self, warpType):
2265 warpName = (warpType[0].upper() + warpType[1:])
2266 message =
"""No %(warpName)s warps were found to build the template coadd which is
2267 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2268 first either rerun makeCoaddTempExp
with config.make%(warpName)s=
True or
2269 coaddDriver
with config.makeCoadTempExp.make%(warpName)s=
True, before assembleCoadd.
2271 Alternatively, to use another algorithm
with existing warps, retarget the CoaddDriverConfig to
2272 another algorithm like:
2275 config.assemble.retarget(SafeClipAssembleCoaddTask)
2276 """ % {"warpName": warpName}
2279 @utils.inheritDoc(AssembleCoaddTask)
2281 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2282 supplementaryData, *args, **kwargs):
2283 """Assemble the coadd.
2285 Find artifacts and apply them to the warps
' masks creating a list of
2286 alternative masks with a new
"CLIPPED" plane
and updated
"NO_DATA"
2287 plane. Then
pass these alternative masks to the base
class's `run`
2290 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2291 that must contain a ``templateCoadd`` that serves
as the
2292 model of the static sky.
2298 dataIds = [ref.dataId
for ref
in tempExpRefList]
2299 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
2301 if dataIds != psfMatchedDataIds:
2302 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2303 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2304 psfMatchedDataIds, dataIds)
2305 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2306 psfMatchedDataIds, dataIds)
2309 spanSetMaskList = self.
findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2310 supplementaryData.warpRefList,
2311 supplementaryData.imageScalerList)
2313 badMaskPlanes = self.config.badMaskPlanes[:]
2314 badMaskPlanes.append(
"CLIPPED")
2315 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2317 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2318 spanSetMaskList, mask=badPixelMask)
2322 self.
applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2326 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2332 altMaskList : `list`
2333 List of Dicts containing ``spanSet`` lists.
2334 Each element contains the new mask plane name (e.g. "CLIPPED
2335 and/
or "NO_DATA")
as the key,
and list of ``SpanSets`` to apply to
2338 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE",
"INEXACT_PSF"])
2339 for visitMask
in altMaskList:
2340 if "EDGE" in visitMask:
2341 for spanSet
in visitMask[
'EDGE']:
2342 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2347 Loop through warps twice. The first loop builds a map with the count
2348 of how many epochs each pixel deviates
from the templateCoadd by more
2349 than ``config.chiThreshold`` sigma. The second loop takes each
2350 difference image
and filters the artifacts detected
in each using
2351 count map to filter out variable sources
and sources that are
2352 difficult to subtract cleanly.
2357 Exposure to serve
as model of static sky.
2358 tempExpRefList : `list`
2359 List of data references to warps.
2360 imageScalerList : `list`
2361 List of image scalers.
2366 List of dicts containing information about CLIPPED
2367 (i.e., artifacts), NO_DATA,
and EDGE pixels.
2370 self.log.debug("Generating Count Image, and mask lists.")
2371 coaddBBox = templateCoadd.getBBox()
2372 slateIm = afwImage.ImageU(coaddBBox)
2373 epochCountImage = afwImage.ImageU(coaddBBox)
2374 nImage = afwImage.ImageU(coaddBBox)
2375 spanSetArtifactList = []
2376 spanSetNoDataMaskList = []
2377 spanSetEdgeList = []
2378 spanSetBadMorphoList = []
2379 badPixelMask = self.getBadPixelMask()
2382 templateCoadd.mask.clearAllMaskPlanes()
2384 if self.config.doPreserveContainedBySource:
2385 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2387 templateFootprints =
None
2389 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2391 if warpDiffExp
is not None:
2393 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2394 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2395 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2396 fpSet.positive.merge(fpSet.negative)
2397 footprints = fpSet.positive
2399 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2402 if self.config.doPrefilterArtifacts:
2406 self.detect.clearMask(warpDiffExp.mask)
2407 for spans
in spanSetList:
2408 spans.setImage(slateIm, 1, doClip=
True)
2409 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
2410 epochCountImage += slateIm
2412 if self.config.doFilterMorphological:
2413 maskName = self.config.streakMaskName
2414 _ = self.maskStreaks.
run(warpDiffExp)
2415 streakMask = warpDiffExp.mask
2416 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2417 streakMask.getPlaneBitMask(maskName)).split()
2423 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2424 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2425 nansMask.setXY0(warpDiffExp.getXY0())
2426 edgeMask = warpDiffExp.mask
2427 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2428 edgeMask.getPlaneBitMask(
"EDGE")).split()
2432 nansMask = afwImage.MaskX(coaddBBox, 1)
2434 spanSetEdgeMask = []
2437 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2439 spanSetNoDataMaskList.append(spanSetNoDataMask)
2440 spanSetArtifactList.append(spanSetList)
2441 spanSetEdgeList.append(spanSetEdgeMask)
2442 if self.config.doFilterMorphological:
2443 spanSetBadMorphoList.append(spanSetStreak)
2446 path = self.
_dataRef2DebugPath_dataRef2DebugPath(
"epochCountIm", tempExpRefList[0], coaddLevel=
True)
2447 epochCountImage.writeFits(path)
2449 for i, spanSetList
in enumerate(spanSetArtifactList):
2451 filteredSpanSetList = self.
filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2453 spanSetArtifactList[i] = filteredSpanSetList
2454 if self.config.doFilterMorphological:
2455 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2458 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2459 altMasks.append({
'CLIPPED': artifacts,
2465 """Remove artifact candidates covered by bad mask plane.
2467 Any future editing of the candidate list that does not depend on
2468 temporal information should go
in this method.
2472 spanSetList : `list`
2473 List of SpanSets representing artifact candidates.
2475 Exposure containing mask planes used to prefilter.
2479 returnSpanSetList : `list`
2480 List of SpanSets
with artifacts.
2482 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2483 goodArr = (exp.mask.array & badPixelMask) == 0
2484 returnSpanSetList = []
2485 bbox = exp.getBBox()
2486 x0, y0 = exp.getXY0()
2487 for i, span
in enumerate(spanSetList):
2488 y, x = span.clippedTo(bbox).indices()
2489 yIndexLocal = numpy.array(y) - y0
2490 xIndexLocal = numpy.array(x) - x0
2491 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2492 if goodRatio > self.config.prefilterArtifactsRatio:
2493 returnSpanSetList.append(span)
2494 return returnSpanSetList
2496 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2497 """Filter artifact candidates.
2501 spanSetList : `list`
2502 List of SpanSets representing artifact candidates.
2504 Image of accumulated number of warpDiff detections.
2506 Image of the accumulated number of total epochs contributing.
2510 maskSpanSetList : `list`
2511 List of SpanSets with artifacts.
2514 maskSpanSetList = []
2515 x0, y0 = epochCountImage.getXY0()
2516 for i, span
in enumerate(spanSetList):
2517 y, x = span.indices()
2518 yIdxLocal = [y1 - y0
for y1
in y]
2519 xIdxLocal = [x1 - x0
for x1
in x]
2520 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2521 totalN = nImage.array[yIdxLocal, xIdxLocal]
2524 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2525 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2526 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2527 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2528 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2529 & (outlierN <= effectiveMaxNumEpochs))
2530 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2531 if percentBelowThreshold > self.config.spatialThreshold:
2532 maskSpanSetList.append(span)
2534 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2536 filteredMaskSpanSetList = []
2537 for span
in maskSpanSetList:
2539 for footprint
in footprintsToExclude.positive.getFootprints():
2540 if footprint.spans.contains(span):
2544 filteredMaskSpanSetList.append(span)
2545 maskSpanSetList = filteredMaskSpanSetList
2547 return maskSpanSetList
2549 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2550 """Fetch a warp from the butler and return a warpDiff.
2555 Butler dataRef for the warp.
2557 An image scaler object.
2559 Exposure to be substracted
from the scaled warp.
2564 Exposure of the image difference between the warp
and template.
2572 warpName = self.getTempExpDatasetName(
'psfMatched')
2573 if not isinstance(warpRef, DeferredDatasetHandle):
2574 if not warpRef.datasetExists(warpName):
2575 self.log.warning(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2577 warp = warpRef.get(datasetType=warpName, immediate=
True)
2579 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2580 mi = warp.getMaskedImage()
2581 if self.config.doScaleWarpVariance:
2583 self.scaleWarpVariance.
run(mi)
2584 except Exception
as exc:
2585 self.log.warning(
"Unable to rescale variance of warp (%s); leaving it as-is", exc)
2586 mi -= templateCoadd.getMaskedImage()
2589 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2590 """Return a path to which to write debugging output.
2592 Creates a hyphen-delimited string of dataId values for simple filenames.
2597 Prefix
for filename.
2599 Butler dataRef to make the path
from.
2600 coaddLevel : `bool`, optional.
2601 If
True, include only coadd-level keys (e.g.,
'tract',
'patch',
2602 'filter', but no
'visit').
2607 Path
for debugging output.
2610 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2612 keys = warpRef.dataId.keys()
2613 keyList = sorted(keys, reverse=
True)
2615 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2616 return os.path.join(directory, filename)
def makeDataRefList(self, namespace)
def __init__(self, *config=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def applyAltEdgeMask(self, mask, altMaskList)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def __init__(self, *args, **kwargs)
def detectClip(self, exp, tempExpRefList)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def setInexactPsf(self, mask)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeCoaddSuffix(warpType="direct")
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")