41 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42 from .interpImage
import InterpImageTask
43 from .scaleZeroPoint
import ScaleZeroPointTask
44 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
45 from .scaleVariance
import ScaleVarianceTask
46 from .maskStreaks
import MaskStreaksTask
47 from .healSparseMapping
import HealSparseInputMapTask
49 from lsst.daf.butler
import DeferredDatasetHandle
51 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
52 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
53 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
55 log = logging.getLogger(__name__.partition(
".")[2])
59 dimensions=(
"tract",
"patch",
"band",
"skymap"),
60 defaultTemplates={
"inputCoaddName":
"deep",
61 "outputCoaddName":
"deep",
63 "warpTypeSuffix":
""}):
65 inputWarps = pipeBase.connectionTypes.Input(
66 doc=(
"Input list of warps to be assemebled i.e. stacked."
67 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
68 name=
"{inputCoaddName}Coadd_{warpType}Warp",
69 storageClass=
"ExposureF",
70 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
74 skyMap = pipeBase.connectionTypes.Input(
75 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
77 storageClass=
"SkyMap",
78 dimensions=(
"skymap", ),
80 selectedVisits = pipeBase.connectionTypes.Input(
81 doc=
"Selected visits to be coadded.",
82 name=
"{outputCoaddName}Visits",
83 storageClass=
"StructuredDataDict",
84 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band")
86 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
87 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
89 name=
"brightObjectMask",
90 storageClass=
"ObjectMaskCatalog",
91 dimensions=(
"tract",
"patch",
"skymap",
"band"),
93 coaddExposure = pipeBase.connectionTypes.Output(
94 doc=
"Output coadded exposure, produced by stacking input warps",
95 name=
"{outputCoaddName}Coadd{warpTypeSuffix}",
96 storageClass=
"ExposureF",
97 dimensions=(
"tract",
"patch",
"skymap",
"band"),
99 nImage = pipeBase.connectionTypes.Output(
100 doc=
"Output image of number of input images per pixel",
101 name=
"{outputCoaddName}Coadd_nImage",
102 storageClass=
"ImageU",
103 dimensions=(
"tract",
"patch",
"skymap",
"band"),
105 inputMap = pipeBase.connectionTypes.Output(
106 doc=
"Output healsparse map of input images",
107 name=
"{outputCoaddName}Coadd_inputMap",
108 storageClass=
"HealSparseMap",
109 dimensions=(
"tract",
"patch",
"skymap",
"band"),
112 def __init__(self, *, config=None):
113 super().__init__(config=config)
118 templateValues = {name: getattr(config.connections, name)
for name
in self.defaultTemplates}
119 templateValues[
'warpType'] = config.warpType
121 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
122 for name
in self.allConnections}
123 self._typeNameToVarName = {v: k
for k, v
in self._nameOverrides.items()}
126 if not config.doMaskBrightObjects:
127 self.prerequisiteInputs.remove(
"brightObjectMask")
129 if not config.doSelectVisits:
130 self.inputs.remove(
"selectedVisits")
132 if not config.doNImage:
133 self.outputs.remove(
"nImage")
135 if not self.config.doInputMap:
136 self.outputs.remove(
"inputMap")
139 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
140 pipelineConnections=AssembleCoaddConnections):
141 """Configuration parameters for the `AssembleCoaddTask`.
145 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
146 only set the bitplane config.brightObjectMaskName. To make this useful you
147 *must* also configure the flags.pixel algorithm, for example by adding
151 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
152 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
154 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
156 warpType = pexConfig.Field(
157 doc=
"Warp name: one of 'direct' or 'psfMatched'",
161 subregionSize = pexConfig.ListField(
163 doc=
"Width, height of stack subregion size; "
164 "make small enough that a full stack of images will fit into memory at once.",
166 default=(2000, 2000),
168 statistic = pexConfig.Field(
170 doc=
"Main stacking statistic for aggregating over the epochs.",
173 doOnlineForMean = pexConfig.Field(
175 doc=
"Perform online coaddition when statistic=\"MEAN\" to save memory?",
178 doSigmaClip = pexConfig.Field(
180 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
183 sigmaClip = pexConfig.Field(
185 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
188 clipIter = pexConfig.Field(
190 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
193 calcErrorFromInputVariance = pexConfig.Field(
195 doc=
"Calculate coadd variance from input variance by stacking statistic."
196 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
199 scaleZeroPoint = pexConfig.ConfigurableField(
200 target=ScaleZeroPointTask,
201 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
203 doInterp = pexConfig.Field(
204 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
208 interpImage = pexConfig.ConfigurableField(
209 target=InterpImageTask,
210 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
212 doWrite = pexConfig.Field(
213 doc=
"Persist coadd?",
217 doNImage = pexConfig.Field(
218 doc=
"Create image of number of contributing exposures for each pixel",
222 doUsePsfMatchedPolygons = pexConfig.Field(
223 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
227 maskPropagationThresholds = pexConfig.DictField(
230 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
231 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
232 "would have contributed exceeds this value."),
233 default={
"SAT": 0.1},
235 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
236 doc=
"Mask planes to remove before coadding")
237 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
238 doc=
"Set mask and flag bits for bright objects?")
239 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
240 doc=
"Name of mask bit used for bright objects")
241 coaddPsf = pexConfig.ConfigField(
242 doc=
"Configuration for CoaddPsf",
243 dtype=measAlg.CoaddPsfConfig,
245 doAttachTransmissionCurve = pexConfig.Field(
246 dtype=bool, default=
False, optional=
False,
247 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
248 "(requires all input Exposures to have TransmissionCurves).")
250 hasFakes = pexConfig.Field(
253 doc=
"Should be set to True if fake sources have been inserted into the input data."
255 doSelectVisits = pexConfig.Field(
256 doc=
"Coadd only visits selected by a SelectVisitsTask",
260 doInputMap = pexConfig.Field(
261 doc=
"Create a bitwise map of coadd inputs",
265 inputMapper = pexConfig.ConfigurableField(
266 doc=
"Input map creation subtask.",
267 target=HealSparseInputMapTask,
270 def setDefaults(self):
271 super().setDefaults()
272 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
279 log.warning(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
280 self.warpType =
'psfMatched'
281 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
282 log.warning(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
283 self.statistic =
"MEANCLIP"
284 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
285 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
286 "compute and set a non-zero coadd variance estimate." % (self.statistic))
288 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
289 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
290 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
291 if str(k)
not in unstackableStats]
292 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
293 % (self.statistic, stackableStats))
296 class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
297 """Assemble a coadded image from a set of warps (coadded temporary exposures).
299 We want to assemble a coadded image from a set of Warps (also called
300 coadded temporary exposures or ``coaddTempExps``).
301 Each input Warp covers a patch on the sky and corresponds to a single
302 run/visit/exposure of the covered patch. We provide the task with a list
303 of Warps (``selectDataList``) from which it selects Warps that cover the
304 specified patch (pointed at by ``dataRef``).
305 Each Warp that goes into a coadd will typically have an independent
306 photometric zero-point. Therefore, we must scale each Warp to set it to
307 a common photometric zeropoint. WarpType may be one of 'direct' or
308 'psfMatched', and the boolean configs `config.makeDirect` and
309 `config.makePsfMatched` set which of the warp types will be coadded.
310 The coadd is computed as a mean with optional outlier rejection.
311 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
312 Finally, Warps can have bad 'NaN' pixels which received no input from the
313 source calExps. We interpolate over these bad (NaN) pixels.
315 `AssembleCoaddTask` uses several sub-tasks. These are
317 - `ScaleZeroPointTask`
318 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
320 - interpolate across bad pixels (NaN) in the final coadd
322 You can retarget these subtasks if you wish.
326 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
327 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
328 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
329 no debug variables of its own. Some of the subtasks may support debug
330 variables. See the documentation for the subtasks for further information.
334 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
335 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
336 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
337 inputs: a data reference to the tract patch and filter to be coadded, and
338 a list of Warps to attempt to coadd. These are specified using ``--id`` and
339 ``--selectId``, respectively:
343 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
344 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
346 Only the Warps that cover the specified tract and patch will be coadded.
347 A list of the available optional arguments can be obtained by calling
348 ``assembleCoadd.py`` with the ``--help`` command line argument:
352 assembleCoadd.py --help
354 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
355 multi-band processing, we will generate the HSC-I & -R band coadds from
356 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
357 assuming that the lsst stack has been already set up, we must set up the
358 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
359 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
360 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
361 coadds, we must first
364 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
366 - create a skymap that covers the area of the sky present in the raw exposures
368 - warp the individual calibrated exposures to the tangent plane of the coadd
370 We can perform all of these steps by running
374 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
376 This will produce warped exposures for each visit. To coadd the warped
377 data, we call assembleCoadd.py as follows:
381 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
382 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
383 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
384 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
385 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
386 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
387 --selectId visit=903988 ccd=24
389 that will process the HSC-I band data. The results are written in
390 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
392 You may also choose to run:
396 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
397 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
398 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
399 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
400 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
401 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
402 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
403 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
405 to generate the coadd for the HSC-R band if you are interested in
406 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
407 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
408 rather than `AssembleCoaddTask` to make the coadd.
410 ConfigClass = AssembleCoaddConfig
411 _DefaultName =
"assembleCoadd"
413 def __init__(self, *args, **kwargs):
416 argNames = [
"config",
"name",
"parentTask",
"log"]
417 kwargs.update({k: v
for k, v
in zip(argNames, args)})
418 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
419 "PipelineTask will not take positional args" % argNames, FutureWarning)
421 super().__init__(**kwargs)
422 self.makeSubtask(
"interpImage")
423 self.makeSubtask(
"scaleZeroPoint")
425 if self.config.doMaskBrightObjects:
426 mask = afwImage.Mask()
428 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
429 except pexExceptions.LsstCppException:
430 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
431 mask.getMaskPlaneDict().keys())
434 if self.config.doInputMap:
435 self.makeSubtask(
"inputMapper")
437 self.warpType = self.config.warpType
439 @utils.inheritDoc(pipeBase.PipelineTask)
440 def runQuantum(self, butlerQC, inputRefs, outputRefs):
445 Assemble a coadd from a set of Warps.
447 PipelineTask (Gen3) entry point to Coadd a set of Warps.
448 Analogous to `runDataRef`, it prepares all the data products to be
449 passed to `run`, and processes the results before returning a struct
450 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
451 Therefore, its inputs are accessed subregion by subregion
452 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
453 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
454 correspond to an update in `runDataRef` while both entry points
457 inputData = butlerQC.get(inputRefs)
461 skyMap = inputData[
"skyMap"]
462 outputDataId = butlerQC.quantum.dataId
465 tractId=outputDataId[
'tract'],
466 patchId=outputDataId[
'patch'])
468 if self.config.doSelectVisits:
469 warpRefList = self.filterWarps(inputData[
'inputWarps'], inputData[
'selectedVisits'])
471 warpRefList = inputData[
'inputWarps']
474 inputs = self.prepareInputs(warpRefList)
475 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
476 self.getTempExpDatasetName(self.warpType))
477 if len(inputs.tempExpRefList) == 0:
478 raise pipeBase.NoWorkFound(
"No coadd temporary exposures found")
480 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
481 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
482 inputs.weightList, supplementaryData=supplementaryData)
484 inputData.setdefault(
'brightObjectMask',
None)
485 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
487 if self.config.doWrite:
488 butlerQC.put(retStruct, outputRefs)
492 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
493 """Assemble a coadd from a set of Warps.
495 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
496 Compute weights to be applied to each Warp and
497 find scalings to match the photometric zeropoint to a reference Warp.
498 Assemble the Warps using `run`. Interpolate over NaNs and
499 optionally write the coadd to disk. Return the coadded exposure.
503 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
504 Data reference defining the patch for coaddition and the
505 reference Warp (if ``config.autoReference=False``).
506 Used to access the following data products:
507 - ``self.config.coaddName + "Coadd_skyMap"``
508 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
509 - ``self.config.coaddName + "Coadd"``
510 selectDataList : `list`
511 List of data references to Calexps. Data to be coadded will be
512 selected from this list based on overlap with the patch defined
513 by dataRef, grouped by visit, and converted to a list of data
516 List of data references to Warps to be coadded.
517 Note: `warpRefList` is just the new name for `tempExpRefList`.
521 retStruct : `lsst.pipe.base.Struct`
522 Result struct with components:
524 - ``coaddExposure``: coadded exposure (``Exposure``).
525 - ``nImage``: exposure count image (``Image``).
527 if selectDataList
and warpRefList:
528 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, "
529 "and which to use is ambiguous. Please pass only one.")
531 skyInfo = self.getSkyInfo(dataRef)
532 if warpRefList
is None:
533 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
534 if len(calExpRefList) == 0:
535 self.log.warning(
"No exposures to coadd")
537 self.log.info(
"Coadding %d exposures", len(calExpRefList))
539 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
541 inputData = self.prepareInputs(warpRefList)
542 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
543 self.getTempExpDatasetName(self.warpType))
544 if len(inputData.tempExpRefList) == 0:
545 self.log.warning(
"No coadd temporary exposures found")
548 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
550 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
551 inputData.weightList, supplementaryData=supplementaryData)
553 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None
554 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
556 if self.config.doWrite:
557 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
558 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
560 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
561 self.log.info(
"Persisting %s", coaddDatasetName)
562 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
563 if self.config.doNImage
and retStruct.nImage
is not None:
564 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
569 """Interpolate over missing data and mask bright stars.
573 coaddExposure : `lsst.afw.image.Exposure`
574 The coadded exposure to process.
575 dataRef : `lsst.daf.persistence.ButlerDataRef`
576 Butler data reference for supplementary data.
578 if self.config.doInterp:
579 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
581 varArray = coaddExposure.variance.array
582 with numpy.errstate(invalid=
"ignore"):
583 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
585 if self.config.doMaskBrightObjects:
586 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
589 """Make additional inputs to run() specific to subclasses (Gen2)
591 Duplicates interface of `runDataRef` method
592 Available to be implemented by subclasses only if they need the
593 coadd dataRef for performing preliminary processing before
594 assembling the coadd.
598 dataRef : `lsst.daf.persistence.ButlerDataRef`
599 Butler data reference for supplementary data.
600 selectDataList : `list` (optional)
601 Optional List of data references to Calexps.
602 warpRefList : `list` (optional)
603 Optional List of data references to Warps.
605 return pipeBase.Struct()
608 """Make additional inputs to run() specific to subclasses (Gen3)
610 Duplicates interface of `runQuantum` method.
611 Available to be implemented by subclasses only if they need the
612 coadd dataRef for performing preliminary processing before
613 assembling the coadd.
617 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
618 Gen3 Butler object for fetching additional data products before
619 running the Task specialized for quantum being processed
620 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
621 Attributes are the names of the connections describing input dataset types.
622 Values are DatasetRefs that task consumes for corresponding dataset type.
623 DataIds are guaranteed to match data objects in ``inputData``.
624 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
625 Attributes are the names of the connections describing output dataset types.
626 Values are DatasetRefs that task is to produce
627 for corresponding dataset type.
629 return pipeBase.Struct()
632 """Generate list data references corresponding to warped exposures
633 that lie within the patch to be coadded.
638 Data reference for patch.
639 calExpRefList : `list`
640 List of data references for input calexps.
644 tempExpRefList : `list`
645 List of Warp/CoaddTempExp data references.
647 butler = patchRef.getButler()
648 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
649 self.getTempExpDatasetName(self.warpType))
650 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
651 g, groupData.keys)
for
652 g
in groupData.groups.keys()]
653 return tempExpRefList
656 """Prepare the input warps for coaddition by measuring the weight for
657 each warp and the scaling for the photometric zero point.
659 Each Warp has its own photometric zeropoint and background variance.
660 Before coadding these Warps together, compute a scale factor to
661 normalize the photometric zeropoint and compute the weight for each Warp.
666 List of data references to tempExp
670 result : `lsst.pipe.base.Struct`
671 Result struct with components:
673 - ``tempExprefList``: `list` of data references to tempExp.
674 - ``weightList``: `list` of weightings.
675 - ``imageScalerList``: `list` of image scalers.
677 statsCtrl = afwMath.StatisticsControl()
678 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
679 statsCtrl.setNumIter(self.config.clipIter)
680 statsCtrl.setAndMask(self.getBadPixelMask())
681 statsCtrl.setNanSafe(
True)
688 tempExpName = self.getTempExpDatasetName(self.warpType)
689 for tempExpRef
in refList:
692 if not isinstance(tempExpRef, DeferredDatasetHandle):
693 if not tempExpRef.datasetExists(tempExpName):
694 self.log.warning(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
697 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
699 if numpy.isnan(tempExp.image.array).all():
701 maskedImage = tempExp.getMaskedImage()
702 imageScaler = self.scaleZeroPoint.computeImageScaler(
707 imageScaler.scaleMaskedImage(maskedImage)
708 except Exception
as e:
709 self.log.warning(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
711 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
712 afwMath.MEANCLIP, statsCtrl)
713 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
714 weight = 1.0 / float(meanVar)
715 if not numpy.isfinite(weight):
716 self.log.warning(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
718 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
723 tempExpRefList.append(tempExpRef)
724 weightList.append(weight)
725 imageScalerList.append(imageScaler)
727 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
728 imageScalerList=imageScalerList)
731 """Prepare the statistics for coadding images.
735 mask : `int`, optional
736 Bit mask value to exclude from coaddition.
740 stats : `lsst.pipe.base.Struct`
741 Statistics structure with the following fields:
743 - ``statsCtrl``: Statistics control object for coadd
744 (`lsst.afw.math.StatisticsControl`)
745 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
748 mask = self.getBadPixelMask()
749 statsCtrl = afwMath.StatisticsControl()
750 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
751 statsCtrl.setNumIter(self.config.clipIter)
752 statsCtrl.setAndMask(mask)
753 statsCtrl.setNanSafe(
True)
754 statsCtrl.setWeighted(
True)
755 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
756 for plane, threshold
in self.config.maskPropagationThresholds.items():
757 bit = afwImage.Mask.getMaskPlane(plane)
758 statsCtrl.setMaskPropagationThreshold(bit, threshold)
759 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
760 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
763 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
764 altMaskList=None, mask=None, supplementaryData=None):
765 """Assemble a coadd from input warps
767 Assemble the coadd using the provided list of coaddTempExps. Since
768 the full coadd covers a patch (a large area), the assembly is
769 performed over small areas on the image at a time in order to
770 conserve memory usage. Iterate over subregions within the outer
771 bbox of the patch using `assembleSubregion` to stack the corresponding
772 subregions from the coaddTempExps with the statistic specified.
773 Set the edge bits the coadd mask based on the weight map.
777 skyInfo : `lsst.pipe.base.Struct`
778 Struct with geometric information about the patch.
779 tempExpRefList : `list`
780 List of data references to Warps (previously called CoaddTempExps).
781 imageScalerList : `list`
782 List of image scalers.
785 altMaskList : `list`, optional
786 List of alternate masks to use rather than those stored with
788 mask : `int`, optional
789 Bit mask value to exclude from coaddition.
790 supplementaryData : lsst.pipe.base.Struct, optional
791 Struct with additional data products needed to assemble coadd.
792 Only used by subclasses that implement `makeSupplementaryData`
797 result : `lsst.pipe.base.Struct`
798 Result struct with components:
800 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
801 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
802 - ``inputMap``: bit-wise map of inputs, if requested.
803 - ``warpRefList``: input list of refs to the warps (
804 ``lsst.daf.butler.DeferredDatasetHandle`` or
805 ``lsst.daf.persistence.ButlerDataRef``)
807 - ``imageScalerList``: input list of image scalers (unmodified)
808 - ``weightList``: input list of weights (unmodified)
810 tempExpName = self.getTempExpDatasetName(self.warpType)
811 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
812 stats = self.prepareStats(mask=mask)
814 if altMaskList
is None:
815 altMaskList = [
None]*len(tempExpRefList)
817 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
818 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
819 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
820 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
821 coaddMaskedImage = coaddExposure.getMaskedImage()
822 subregionSizeArr = self.config.subregionSize
823 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
825 if self.config.doNImage:
826 nImage = afwImage.ImageU(skyInfo.bbox)
831 if self.config.doInputMap:
832 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
834 coaddExposure.getInfo().getCoaddInputs().ccds)
836 if self.config.doOnlineForMean
and self.config.statistic ==
"MEAN":
838 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
839 weightList, altMaskList, stats.ctrl,
841 except Exception
as e:
842 self.log.fatal(
"Cannot compute online coadd %s", e)
844 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
846 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
847 weightList, altMaskList, stats.flags, stats.ctrl,
849 except Exception
as e:
850 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
853 if self.config.doInputMap:
854 self.inputMapper.finalize_ccd_input_map_mask()
855 inputMap = self.inputMapper.ccd_input_map
859 self.setInexactPsf(coaddMaskedImage.getMask())
862 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
863 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
864 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
865 weightList=weightList, inputMap=inputMap)
868 """Set the metadata for the coadd.
870 This basic implementation sets the filter from the first input.
874 coaddExposure : `lsst.afw.image.Exposure`
875 The target exposure for the coadd.
876 tempExpRefList : `list`
877 List of data references to tempExp.
881 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
882 tempExpName = self.getTempExpDatasetName(self.warpType)
888 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
890 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
893 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
894 for tempExpRef
in tempExpRefList]
895 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
899 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
900 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
901 coaddInputs.ccds.reserve(numCcds)
902 coaddInputs.visits.reserve(len(tempExpList))
904 for tempExp, weight
in zip(tempExpList, weightList):
905 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
907 if self.config.doUsePsfMatchedPolygons:
908 self.shrinkValidPolygons(coaddInputs)
910 coaddInputs.visits.sort()
911 if self.warpType ==
"psfMatched":
916 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
917 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
918 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
920 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
921 self.config.coaddPsf.makeControl())
922 coaddExposure.setPsf(psf)
923 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
924 coaddExposure.getWcs())
925 coaddExposure.getInfo().setApCorrMap(apCorrMap)
926 if self.config.doAttachTransmissionCurve:
927 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
928 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
931 altMaskList, statsFlags, statsCtrl, nImage=None):
932 """Assemble the coadd for a sub-region.
934 For each coaddTempExp, check for (and swap in) an alternative mask
935 if one is passed. Remove mask planes listed in
936 `config.removeMaskPlanes`. Finally, stack the actual exposures using
937 `lsst.afw.math.statisticsStack` with the statistic specified by
938 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
939 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
940 an N-sigma clipped mean where N and iterations are specified by
941 statsCtrl. Assign the stacked subregion back to the coadd.
945 coaddExposure : `lsst.afw.image.Exposure`
946 The target exposure for the coadd.
947 bbox : `lsst.geom.Box`
949 tempExpRefList : `list`
950 List of data reference to tempExp.
951 imageScalerList : `list`
952 List of image scalers.
956 List of alternate masks to use rather than those stored with
957 tempExp, or None. Each element is dict with keys = mask plane
958 name to which to add the spans.
959 statsFlags : `lsst.afw.math.Property`
960 Property object for statistic for coadd.
961 statsCtrl : `lsst.afw.math.StatisticsControl`
962 Statistics control object for coadd.
963 nImage : `lsst.afw.image.ImageU`, optional
964 Keeps track of exposure count for each pixel.
966 self.log.debug(
"Computing coadd over %s", bbox)
967 tempExpName = self.getTempExpDatasetName(self.warpType)
968 coaddExposure.mask.addMaskPlane(
"REJECTED")
969 coaddExposure.mask.addMaskPlane(
"CLIPPED")
970 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
971 maskMap = self.setRejectedMaskMapping(statsCtrl)
972 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
974 if nImage
is not None:
975 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
976 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
978 if isinstance(tempExpRef, DeferredDatasetHandle):
980 exposure = tempExpRef.get(parameters={
'bbox': bbox})
983 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
985 maskedImage = exposure.getMaskedImage()
986 mask = maskedImage.getMask()
987 if altMask
is not None:
988 self.applyAltMaskPlanes(mask, altMask)
989 imageScaler.scaleMaskedImage(maskedImage)
993 if nImage
is not None:
994 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
995 if self.config.removeMaskPlanes:
996 self.removeMaskPlanes(maskedImage)
997 maskedImageList.append(maskedImage)
999 if self.config.doInputMap:
1000 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1001 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1003 with self.timer(
"stack"):
1004 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
1007 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
1008 if nImage
is not None:
1009 nImage.assign(subNImage, bbox)
1012 altMaskList, statsCtrl, nImage=None):
1013 """Assemble the coadd using the "online" method.
1015 This method takes a running sum of images and weights to save memory.
1016 It only works for MEAN statistics.
1020 coaddExposure : `lsst.afw.image.Exposure`
1021 The target exposure for the coadd.
1022 tempExpRefList : `list`
1023 List of data reference to tempExp.
1024 imageScalerList : `list`
1025 List of image scalers.
1028 altMaskList : `list`
1029 List of alternate masks to use rather than those stored with
1030 tempExp, or None. Each element is dict with keys = mask plane
1031 name to which to add the spans.
1032 statsCtrl : `lsst.afw.math.StatisticsControl`
1033 Statistics control object for coadd
1034 nImage : `lsst.afw.image.ImageU`, optional
1035 Keeps track of exposure count for each pixel.
1037 self.log.debug(
"Computing online coadd.")
1038 tempExpName = self.getTempExpDatasetName(self.warpType)
1039 coaddExposure.mask.addMaskPlane(
"REJECTED")
1040 coaddExposure.mask.addMaskPlane(
"CLIPPED")
1041 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
1042 maskMap = self.setRejectedMaskMapping(statsCtrl)
1043 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
1045 bbox = coaddExposure.maskedImage.getBBox()
1047 stacker = AccumulatorMeanStack(
1048 coaddExposure.image.array.shape,
1049 statsCtrl.getAndMask(),
1050 mask_threshold_dict=thresholdDict,
1052 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
1053 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
1054 compute_n_image=(nImage
is not None)
1057 for tempExpRef, imageScaler, altMask, weight
in zip(tempExpRefList,
1061 if isinstance(tempExpRef, DeferredDatasetHandle):
1063 exposure = tempExpRef.get()
1066 exposure = tempExpRef.get(tempExpName)
1068 maskedImage = exposure.getMaskedImage()
1069 mask = maskedImage.getMask()
1070 if altMask
is not None:
1071 self.applyAltMaskPlanes(mask, altMask)
1072 imageScaler.scaleMaskedImage(maskedImage)
1073 if self.config.removeMaskPlanes:
1074 self.removeMaskPlanes(maskedImage)
1076 stacker.add_masked_image(maskedImage, weight=weight)
1078 if self.config.doInputMap:
1079 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1080 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1082 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
1084 if nImage
is not None:
1085 nImage.array[:, :] = stacker.n_image
1088 """Unset the mask of an image for mask planes specified in the config.
1092 maskedImage : `lsst.afw.image.MaskedImage`
1093 The masked image to be modified.
1095 mask = maskedImage.getMask()
1096 for maskPlane
in self.config.removeMaskPlanes:
1098 mask &= ~mask.getPlaneBitMask(maskPlane)
1099 except pexExceptions.InvalidParameterError:
1100 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
1104 def setRejectedMaskMapping(statsCtrl):
1105 """Map certain mask planes of the warps to new planes for the coadd.
1107 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1108 or CLIPPED, set it to REJECTED on the coadd.
1109 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1110 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1114 statsCtrl : `lsst.afw.math.StatisticsControl`
1115 Statistics control object for coadd
1119 maskMap : `list` of `tuple` of `int`
1120 A list of mappings of mask planes of the warped exposures to
1121 mask planes of the coadd.
1123 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
1124 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
1125 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
1126 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1127 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
1128 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
1133 """Apply in place alt mask formatted as SpanSets to a mask.
1137 mask : `lsst.afw.image.Mask`
1139 altMaskSpans : `dict`
1140 SpanSet lists to apply. Each element contains the new mask
1141 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1142 and list of SpanSets to apply to the mask.
1146 mask : `lsst.afw.image.Mask`
1149 if self.config.doUsePsfMatchedPolygons:
1150 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1155 for spanSet
in altMaskSpans[
'NO_DATA']:
1156 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1158 for plane, spanSetList
in altMaskSpans.items():
1159 maskClipValue = mask.addMaskPlane(plane)
1160 for spanSet
in spanSetList:
1161 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1165 """Shrink coaddInputs' ccds' ValidPolygons in place.
1167 Either modify each ccd's validPolygon in place, or if CoaddInputs
1168 does not have a validPolygon, create one from its bbox.
1172 coaddInputs : `lsst.afw.image.coaddInputs`
1176 for ccd
in coaddInputs.ccds:
1177 polyOrig = ccd.getValidPolygon()
1178 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1179 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1181 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1183 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1184 ccd.setValidPolygon(validPolygon)
1187 """Retrieve the bright object masks.
1189 Returns None on failure.
1193 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1198 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1199 Bright object mask from the Butler object, or None if it cannot
1203 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1204 except Exception
as e:
1205 self.log.warning(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1209 """Set the bright object masks.
1213 exposure : `lsst.afw.image.Exposure`
1214 Exposure under consideration.
1215 dataId : `lsst.daf.persistence.dataId`
1216 Data identifier dict for patch.
1217 brightObjectMasks : `lsst.afw.table`
1218 Table of bright objects to mask.
1221 if brightObjectMasks
is None:
1222 self.log.warning(
"Unable to apply bright object mask: none supplied")
1224 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1225 mask = exposure.getMaskedImage().getMask()
1226 wcs = exposure.getWcs()
1227 plateScale = wcs.getPixelScale().asArcseconds()
1229 for rec
in brightObjectMasks:
1230 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1231 if rec[
"type"] ==
"box":
1232 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1233 width = rec[
"width"].asArcseconds()/plateScale
1234 height = rec[
"height"].asArcseconds()/plateScale
1237 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1240 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1241 spans = afwGeom.SpanSet(bbox)
1242 elif rec[
"type"] ==
"circle":
1243 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1244 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1246 self.log.warning(
"Unexpected region type %s at %s", rec[
"type"], center)
1248 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1251 """Set INEXACT_PSF mask plane.
1253 If any of the input images isn't represented in the coadd (due to
1254 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1259 mask : `lsst.afw.image.Mask`
1260 Coadded exposure's mask, modified in-place.
1262 mask.addMaskPlane(
"INEXACT_PSF")
1263 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1264 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1265 clipped = mask.getPlaneBitMask(
"CLIPPED")
1266 rejected = mask.getPlaneBitMask(
"REJECTED")
1267 array = mask.getArray()
1268 selected = array & (sensorEdge | clipped | rejected) > 0
1269 array[selected] |= inexactPsf
1272 def _makeArgumentParser(cls):
1273 """Create an argument parser.
1275 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1276 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_"
1277 + cls.ConfigClass().warpType +
"Warp",
1278 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1279 ContainerClass=AssembleCoaddDataIdContainer)
1280 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1281 ContainerClass=SelectDataIdContainer)
1285 def _subBBoxIter(bbox, subregionSize):
1286 """Iterate over subregions of a bbox.
1290 bbox : `lsst.geom.Box2I`
1291 Bounding box over which to iterate.
1292 subregionSize: `lsst.geom.Extent2I`
1297 subBBox : `lsst.geom.Box2I`
1298 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1299 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1300 the edges of ``bbox``, but it will never be empty.
1303 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1304 if subregionSize[0] < 1
or subregionSize[1] < 1:
1305 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1307 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1308 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1311 if subBBox.isEmpty():
1312 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1313 "colShift=%s, rowShift=%s" %
1314 (bbox, subregionSize, colShift, rowShift))
1318 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1323 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1325 Dictionary with good visitIds as the keys. Value ignored.
1329 filteredInputs : `list`
1330 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1332 inputWarpDict = {inputRef.ref.dataId[
'visit']: inputRef
for inputRef
in inputs}
1334 for visit
in goodVisits.keys():
1335 filteredInputs.append(inputWarpDict[visit])
1336 return filteredInputs
1340 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1344 """Make self.refList from self.idList.
1349 Results of parsing command-line (with ``butler`` and ``log`` elements).
1351 datasetType = namespace.config.coaddName +
"Coadd"
1352 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1354 for dataId
in self.idList:
1356 for key
in keysCoadd:
1357 if key
not in dataId:
1358 raise RuntimeError(
"--id must include " + key)
1360 dataRef = namespace.butler.dataRef(
1361 datasetType=datasetType,
1364 self.refList.append(dataRef)
1368 """Function to count the number of pixels with a specific mask in a
1371 Find the intersection of mask & footprint. Count all pixels in the mask
1372 that are in the intersection that have bitmask set but do not have
1373 ignoreMask set. Return the count.
1377 mask : `lsst.afw.image.Mask`
1378 Mask to define intersection region by.
1379 footprint : `lsst.afw.detection.Footprint`
1380 Footprint to define the intersection region by.
1382 Specific mask that we wish to count the number of occurances of.
1384 Pixels to not consider.
1389 Count of number of pixels in footprint with specified mask.
1391 bbox = footprint.getBBox()
1392 bbox.clip(mask.getBBox(afwImage.PARENT))
1393 fp = afwImage.Mask(bbox)
1394 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1395 footprint.spans.setMask(fp, bitmask)
1396 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1397 (subMask.getArray() & ignoreMask) == 0).sum()
1401 """Configuration parameters for the SafeClipAssembleCoaddTask.
1403 clipDetection = pexConfig.ConfigurableField(
1404 target=SourceDetectionTask,
1405 doc=
"Detect sources on difference between unclipped and clipped coadd")
1406 minClipFootOverlap = pexConfig.Field(
1407 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1411 minClipFootOverlapSingle = pexConfig.Field(
1412 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1413 "clipped when only one visit overlaps",
1417 minClipFootOverlapDouble = pexConfig.Field(
1418 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1419 "clipped when two visits overlap",
1423 maxClipFootOverlapDouble = pexConfig.Field(
1424 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
1425 "considering two visits",
1429 minBigOverlap = pexConfig.Field(
1430 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1431 "when labeling clipped footprints",
1437 """Set default values for clipDetection.
1441 The numeric values for these configuration parameters were
1442 empirically determined, future work may further refine them.
1444 AssembleCoaddConfig.setDefaults(self)
1445 self.
clipDetectionclipDetection.doTempLocalBackground =
False
1446 self.
clipDetectionclipDetection.reEstimateBackground =
False
1447 self.
clipDetectionclipDetection.returnOriginalFootprints =
False
1453 self.
clipDetectionclipDetection.thresholdType =
"pixel_stdev"
1460 log.warning(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1461 "Ignoring doSigmaClip.")
1464 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1465 "(%s chosen). Please set statistic to MEAN."
1467 AssembleCoaddTask.ConfigClass.validate(self)
1471 """Assemble a coadded image from a set of coadded temporary exposures,
1472 being careful to clip & flag areas with potential artifacts.
1474 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1475 we clip outliers). The problem with doing this is that when computing the
1476 coadd PSF at a given location, individual visit PSFs from visits with
1477 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1478 In this task, we correct for this behavior by creating a new
1479 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1480 coaddTempExps and the final coadd where
1482 i. difference imaging suggests that there is an outlier and
1483 ii. this outlier appears on only one or two images.
1485 Such regions will not contribute to the final coadd. Furthermore, any
1486 routine to determine the coadd PSF can now be cognizant of clipped regions.
1487 Note that the algorithm implemented by this task is preliminary and works
1488 correctly for HSC data. Parameter modifications and or considerable
1489 redesigning of the algorithm is likley required for other surveys.
1491 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1492 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1493 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1498 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1499 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1500 see `baseDebug` for more about ``debug.py`` files.
1501 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1502 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1503 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1504 for further information.
1508 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1509 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1510 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1512 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1513 and filter to be coadded (specified using
1514 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1515 along with a list of coaddTempExps to attempt to coadd (specified using
1516 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1517 Only the coaddTempExps that cover the specified tract and patch will be
1518 coadded. A list of the available optional arguments can be obtained by
1519 calling assembleCoadd.py with the --help command line argument:
1521 .. code-block:: none
1523 assembleCoadd.py --help
1525 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1526 context of multi-band processing, we will generate the HSC-I & -R band
1527 coadds from HSC engineering test data provided in the ci_hsc package.
1528 To begin, assuming that the lsst stack has been already set up, we must
1529 set up the obs_subaru and ci_hsc packages. This defines the environment
1530 variable $CI_HSC_DIR and points at the location of the package. The raw
1531 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1532 the coadds, we must first
1535 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1537 create a skymap that covers the area of the sky present in the raw exposures
1538 - ``makeCoaddTempExp``
1539 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1541 We can perform all of these steps by running
1543 .. code-block:: none
1545 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1547 This will produce warped coaddTempExps for each visit. To coadd the
1548 warped data, we call ``assembleCoadd.py`` as follows:
1550 .. code-block:: none
1552 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1553 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1554 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1555 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1556 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1557 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1558 --selectId visit=903988 ccd=24
1560 This will process the HSC-I band data. The results are written in
1561 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1563 You may also choose to run:
1565 .. code-block:: none
1567 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1568 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1569 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1570 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1571 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1572 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1573 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1574 --selectId visit=903346 ccd=12
1576 to generate the coadd for the HSC-R band if you are interested in following
1577 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1579 ConfigClass = SafeClipAssembleCoaddConfig
1580 _DefaultName =
"safeClipAssembleCoadd"
1583 AssembleCoaddTask.__init__(self, *args, **kwargs)
1584 schema = afwTable.SourceTable.makeMinimalSchema()
1585 self.makeSubtask(
"clipDetection", schema=schema)
1587 @utils.inheritDoc(AssembleCoaddTask)
1588 @pipeBase.timeMethod
1589 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1590 """Assemble the coadd for a region.
1592 Compute the difference of coadds created with and without outlier
1593 rejection to identify coadd pixels that have outlier values in some
1595 Detect clipped regions on the difference image and mark these regions
1596 on the one or two individual coaddTempExps where they occur if there
1597 is significant overlap between the clipped region and a source. This
1598 leaves us with a set of footprints from the difference image that have
1599 been identified as having occured on just one or two individual visits.
1600 However, these footprints were generated from a difference image. It
1601 is conceivable for a large diffuse source to have become broken up
1602 into multiple footprints acrosss the coadd difference in this process.
1603 Determine the clipped region from all overlapping footprints from the
1604 detected sources in each visit - these are big footprints.
1605 Combine the small and big clipped footprints and mark them on a new
1607 Generate the coadd using `AssembleCoaddTask.run` without outlier
1608 removal. Clipped footprints will no longer make it into the coadd
1609 because they are marked in the new bad mask plane.
1613 args and kwargs are passed but ignored in order to match the call
1614 signature expected by the parent task.
1616 exp = self.
buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1617 mask = exp.getMaskedImage().getMask()
1618 mask.addMaskPlane(
"CLIPPED")
1620 result = self.
detectClipdetectClip(exp, tempExpRefList)
1622 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1624 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1625 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1627 bigFootprints = self.
detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1628 result.detectionFootprints, maskClipValue, maskDetValue,
1631 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1632 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1634 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1635 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1636 maskClip |= maskClipBig
1639 badMaskPlanes = self.config.badMaskPlanes[:]
1640 badMaskPlanes.append(
"CLIPPED")
1641 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1642 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1643 result.clipSpans, mask=badPixelMask)
1646 """Return an exposure that contains the difference between unclipped
1649 Generate a difference image between clipped and unclipped coadds.
1650 Compute the difference image by subtracting an outlier-clipped coadd
1651 from an outlier-unclipped coadd. Return the difference image.
1655 skyInfo : `lsst.pipe.base.Struct`
1656 Patch geometry information, from getSkyInfo
1657 tempExpRefList : `list`
1658 List of data reference to tempExp
1659 imageScalerList : `list`
1660 List of image scalers
1666 exp : `lsst.afw.image.Exposure`
1667 Difference image of unclipped and clipped coadd wrapped in an Exposure
1669 config = AssembleCoaddConfig()
1674 configIntersection = {k: getattr(self.config, k)
1675 for k, v
in self.config.toDict().items()
1676 if (k
in config.keys()
and k !=
"connections")}
1677 configIntersection[
'doInputMap'] =
False
1678 configIntersection[
'doNImage'] =
False
1679 config.update(**configIntersection)
1682 config.statistic =
'MEAN'
1683 task = AssembleCoaddTask(config=config)
1684 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1686 config.statistic =
'MEANCLIP'
1687 task = AssembleCoaddTask(config=config)
1688 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1690 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1691 coaddDiff -= coaddClip.getMaskedImage()
1692 exp = afwImage.ExposureF(coaddDiff)
1693 exp.setPsf(coaddMean.getPsf())
1697 """Detect clipped regions on an exposure and set the mask on the
1698 individual tempExp masks.
1700 Detect footprints in the difference image after smoothing the
1701 difference image with a Gaussian kernal. Identify footprints that
1702 overlap with one or two input ``coaddTempExps`` by comparing the
1703 computed overlap fraction to thresholds set in the config. A different
1704 threshold is applied depending on the number of overlapping visits
1705 (restricted to one or two). If the overlap exceeds the thresholds,
1706 the footprint is considered "CLIPPED" and is marked as such on the
1707 coaddTempExp. Return a struct with the clipped footprints, the indices
1708 of the ``coaddTempExps`` that end up overlapping with the clipped
1709 footprints, and a list of new masks for the ``coaddTempExps``.
1713 exp : `lsst.afw.image.Exposure`
1714 Exposure to run detection on.
1715 tempExpRefList : `list`
1716 List of data reference to tempExp.
1720 result : `lsst.pipe.base.Struct`
1721 Result struct with components:
1723 - ``clipFootprints``: list of clipped footprints.
1724 - ``clipIndices``: indices for each ``clippedFootprint`` in
1726 - ``clipSpans``: List of dictionaries containing spanSet lists
1727 to clip. Each element contains the new maskplane name
1728 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1729 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1730 compressed into footprints.
1732 mask = exp.getMaskedImage().getMask()
1733 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1734 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1736 fpSet.positive.merge(fpSet.negative)
1737 footprints = fpSet.positive
1738 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1739 ignoreMask = self.getBadPixelMask()
1743 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1746 visitDetectionFootprints = []
1748 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1749 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1750 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1753 for i, warpRef
in enumerate(tempExpRefList):
1754 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1755 immediate=
True).getMaskedImage().getMask()
1756 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1757 afwImage.PARENT,
True)
1758 maskVisitDet &= maskDetValue
1759 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1760 visitDetectionFootprints.append(visitFootprints)
1762 for j, footprint
in enumerate(footprints.getFootprints()):
1767 for j, footprint
in enumerate(footprints.getFootprints()):
1768 nPixel = footprint.getArea()
1771 for i
in range(len(tempExpRefList)):
1772 ignore = ignoreArr[i, j]
1773 overlapDet = overlapDetArr[i, j]
1774 totPixel = nPixel - ignore
1777 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1779 overlap.append(overlapDet/float(totPixel))
1782 overlap = numpy.array(overlap)
1783 if not len(overlap):
1790 if len(overlap) == 1:
1791 if overlap[0] > self.config.minClipFootOverlapSingle:
1796 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1797 if len(clipIndex) == 1:
1799 keepIndex = [clipIndex[0]]
1802 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1803 if len(clipIndex) == 2
and len(overlap) > 3:
1804 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1805 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1807 keepIndex = clipIndex
1812 for index
in keepIndex:
1813 globalIndex = indexList[index]
1814 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1816 clipIndices.append(numpy.array(indexList)[keepIndex])
1817 clipFootprints.append(footprint)
1819 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1820 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1822 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1823 maskClipValue, maskDetValue, coaddBBox):
1824 """Return individual warp footprints for large artifacts and append
1825 them to ``clipList`` in place.
1827 Identify big footprints composed of many sources in the coadd
1828 difference that may have originated in a large diffuse source in the
1829 coadd. We do this by indentifying all clipped footprints that overlap
1830 significantly with each source in all the coaddTempExps.
1835 List of alt mask SpanSets with clipping information. Modified.
1836 clipFootprints : `list`
1837 List of clipped footprints.
1838 clipIndices : `list`
1839 List of which entries in tempExpClipList each footprint belongs to.
1841 Mask value of clipped pixels.
1843 Mask value of detected pixels.
1844 coaddBBox : `lsst.geom.Box`
1845 BBox of the coadd and warps.
1849 bigFootprintsCoadd : `list`
1850 List of big footprints
1852 bigFootprintsCoadd = []
1853 ignoreMask = self.getBadPixelMask()
1854 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1855 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1856 for footprint
in visitFootprints.getFootprints():
1857 footprint.spans.setMask(maskVisitDet, maskDetValue)
1860 clippedFootprintsVisit = []
1861 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1862 if index
not in clipIndex:
1864 clippedFootprintsVisit.append(foot)
1865 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1866 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1868 bigFootprintsVisit = []
1869 for foot
in visitFootprints.getFootprints():
1870 if foot.getArea() < self.config.minBigOverlap:
1873 if nCount > self.config.minBigOverlap:
1874 bigFootprintsVisit.append(foot)
1875 bigFootprintsCoadd.append(foot)
1877 for footprint
in bigFootprintsVisit:
1878 clippedSpans[
"CLIPPED"].append(footprint.spans)
1880 return bigFootprintsCoadd
1884 psfMatchedWarps = pipeBase.connectionTypes.Input(
1885 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1886 "Only PSF-Matched Warps make sense for image subtraction. "
1887 "Therefore, they must be an additional declared input."),
1888 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1889 storageClass=
"ExposureF",
1890 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1894 templateCoadd = pipeBase.connectionTypes.Output(
1895 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1896 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1897 name=
"{outputCoaddName}CoaddPsfMatched",
1898 storageClass=
"ExposureF",
1899 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1904 if not config.assembleStaticSkyModel.doWrite:
1905 self.outputs.remove(
"templateCoadd")
1910 pipelineConnections=CompareWarpAssembleCoaddConnections):
1911 assembleStaticSkyModel = pexConfig.ConfigurableField(
1912 target=AssembleCoaddTask,
1913 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1914 " naive/first-iteration model of the static sky.",
1916 detect = pexConfig.ConfigurableField(
1917 target=SourceDetectionTask,
1918 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1920 detectTemplate = pexConfig.ConfigurableField(
1921 target=SourceDetectionTask,
1922 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1924 maskStreaks = pexConfig.ConfigurableField(
1925 target=MaskStreaksTask,
1926 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1927 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1930 streakMaskName = pexConfig.Field(
1933 doc=
"Name of mask bit used for streaks"
1935 maxNumEpochs = pexConfig.Field(
1936 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1937 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1938 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1939 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1940 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1941 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1942 "than transient and not masked.",
1946 maxFractionEpochsLow = pexConfig.RangeField(
1947 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1948 "Effective maxNumEpochs = "
1949 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1954 maxFractionEpochsHigh = pexConfig.RangeField(
1955 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1956 "Effective maxNumEpochs = "
1957 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1962 spatialThreshold = pexConfig.RangeField(
1963 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1964 "temporal criteria. If 0, clip all. If 1, clip none.",
1968 inclusiveMin=
True, inclusiveMax=
True
1970 doScaleWarpVariance = pexConfig.Field(
1971 doc=
"Rescale Warp variance plane using empirical noise?",
1975 scaleWarpVariance = pexConfig.ConfigurableField(
1976 target=ScaleVarianceTask,
1977 doc=
"Rescale variance on warps",
1979 doPreserveContainedBySource = pexConfig.Field(
1980 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1981 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1985 doPrefilterArtifacts = pexConfig.Field(
1986 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1987 "because they will be excluded anyway. This prevents them from contributing "
1988 "to the outlier epoch count image and potentially being labeled as persistant."
1989 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1993 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1994 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1996 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1998 prefilterArtifactsRatio = pexConfig.Field(
1999 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
2003 doFilterMorphological = pexConfig.Field(
2004 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
2011 AssembleCoaddConfig.setDefaults(self)
2017 if "EDGE" in self.badMaskPlanes:
2018 self.badMaskPlanes.remove(
'EDGE')
2019 self.removeMaskPlanes.append(
'EDGE')
2028 self.
detectdetect.doTempLocalBackground =
False
2029 self.
detectdetect.reEstimateBackground =
False
2030 self.
detectdetect.returnOriginalFootprints =
False
2031 self.
detectdetect.thresholdPolarity =
"both"
2032 self.
detectdetect.thresholdValue = 5
2033 self.
detectdetect.minPixels = 4
2034 self.
detectdetect.isotropicGrow =
True
2035 self.
detectdetect.thresholdType =
"pixel_stdev"
2036 self.
detectdetect.nSigmaToGrow = 0.4
2042 self.
detectTemplatedetectTemplate.returnOriginalFootprints =
False
2047 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
2048 "Please set assembleStaticSkyModel.doNImage=False")
2051 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
2052 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
2053 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
2058 """Assemble a compareWarp coadded image from a set of warps
2059 by masking artifacts detected by comparing PSF-matched warps.
2061 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
2062 we clip outliers). The problem with doing this is that when computing the
2063 coadd PSF at a given location, individual visit PSFs from visits with
2064 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
2065 In this task, we correct for this behavior by creating a new badMaskPlane
2066 'CLIPPED' which marks pixels in the individual warps suspected to contain
2067 an artifact. We populate this plane on the input warps by comparing
2068 PSF-matched warps with a PSF-matched median coadd which serves as a
2069 model of the static sky. Any group of pixels that deviates from the
2070 PSF-matched template coadd by more than config.detect.threshold sigma,
2071 is an artifact candidate. The candidates are then filtered to remove
2072 variable sources and sources that are difficult to subtract such as
2073 bright stars. This filter is configured using the config parameters
2074 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
2075 the maximum fraction of epochs that the deviation can appear in and still
2076 be considered an artifact. The spatialThreshold is the maximum fraction of
2077 pixels in the footprint of the deviation that appear in other epochs
2078 (where other epochs is defined by the temporalThreshold). If the deviant
2079 region meets this criteria of having a significant percentage of pixels
2080 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
2081 set in the mask. These regions will not contribute to the final coadd.
2082 Furthermore, any routine to determine the coadd PSF can now be cognizant
2083 of clipped regions. Note that the algorithm implemented by this task is
2084 preliminary and works correctly for HSC data. Parameter modifications and
2085 or considerable redesigning of the algorithm is likley required for other
2088 ``CompareWarpAssembleCoaddTask`` sub-classes
2089 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
2090 as a subtask to generate the TemplateCoadd (the model of the static sky).
2094 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2095 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2096 ``baseDebug`` for more about ``debug.py`` files.
2098 This task supports the following debug variables:
2101 If True then save the Epoch Count Image as a fits file in the `figPath`
2103 Path to save the debug fits images and figures
2105 For example, put something like:
2107 .. code-block:: python
2110 def DebugInfo(name):
2111 di = lsstDebug.getInfo(name)
2112 if name == "lsst.pipe.tasks.assembleCoadd":
2113 di.saveCountIm = True
2114 di.figPath = "/desired/path/to/debugging/output/images"
2116 lsstDebug.Info = DebugInfo
2118 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2119 ``--debug`` flag. Some subtasks may have their own debug variables;
2120 see individual Task documentation.
2124 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2125 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2126 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2127 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2128 and filter to be coadded (specified using
2129 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2130 along with a list of coaddTempExps to attempt to coadd (specified using
2131 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2132 Only the warps that cover the specified tract and patch will be coadded.
2133 A list of the available optional arguments can be obtained by calling
2134 ``assembleCoadd.py`` with the ``--help`` command line argument:
2136 .. code-block:: none
2138 assembleCoadd.py --help
2140 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2141 context of multi-band processing, we will generate the HSC-I & -R band
2142 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2143 To begin, assuming that the lsst stack has been already set up, we must
2144 set up the ``obs_subaru`` and ``ci_hsc`` packages.
2145 This defines the environment variable ``$CI_HSC_DIR`` and points at the
2146 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2147 directory. To begin assembling the coadds, we must first
2150 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2152 create a skymap that covers the area of the sky present in the raw exposures
2154 warp the individual calibrated exposures to the tangent plane of the coadd
2156 We can perform all of these steps by running
2158 .. code-block:: none
2160 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2162 This will produce warped ``coaddTempExps`` for each visit. To coadd the
2163 warped data, we call ``assembleCoadd.py`` as follows:
2165 .. code-block:: none
2167 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2168 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2169 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2170 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2171 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2172 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2173 --selectId visit=903988 ccd=24
2175 This will process the HSC-I band data. The results are written in
2176 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2178 ConfigClass = CompareWarpAssembleCoaddConfig
2179 _DefaultName =
"compareWarpAssembleCoadd"
2182 AssembleCoaddTask.__init__(self, *args, **kwargs)
2183 self.makeSubtask(
"assembleStaticSkyModel")
2184 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2185 self.makeSubtask(
"detect", schema=detectionSchema)
2186 if self.config.doPreserveContainedBySource:
2187 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2188 if self.config.doScaleWarpVariance:
2189 self.makeSubtask(
"scaleWarpVariance")
2190 if self.config.doFilterMorphological:
2191 self.makeSubtask(
"maskStreaks")
2193 @utils.inheritDoc(AssembleCoaddTask)
2196 Generate a templateCoadd to use as a naive model of static sky to
2197 subtract from PSF-Matched warps.
2201 result : `lsst.pipe.base.Struct`
2202 Result struct with components:
2204 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2205 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2208 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2209 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2213 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2214 if self.config.assembleStaticSkyModel.doWrite:
2215 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2218 del outputRefs.templateCoadd
2219 del staticSkyModelOutputRefs.templateCoadd
2222 if 'nImage' in staticSkyModelOutputRefs.keys():
2223 del staticSkyModelOutputRefs.nImage
2225 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2226 staticSkyModelOutputRefs)
2227 if templateCoadd
is None:
2228 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2230 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2231 nImage=templateCoadd.nImage,
2232 warpRefList=templateCoadd.warpRefList,
2233 imageScalerList=templateCoadd.imageScalerList,
2234 weightList=templateCoadd.weightList)
2236 @utils.inheritDoc(AssembleCoaddTask)
2239 Generate a templateCoadd to use as a naive model of static sky to
2240 subtract from PSF-Matched warps.
2244 result : `lsst.pipe.base.Struct`
2245 Result struct with components:
2247 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2248 - ``nImage``: N Image (``lsst.afw.image.Image``)
2250 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2251 if templateCoadd
is None:
2252 raise RuntimeError(self.
_noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2254 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2255 nImage=templateCoadd.nImage,
2256 warpRefList=templateCoadd.warpRefList,
2257 imageScalerList=templateCoadd.imageScalerList,
2258 weightList=templateCoadd.weightList)
2260 def _noTemplateMessage(self, warpType):
2261 warpName = (warpType[0].upper() + warpType[1:])
2262 message =
"""No %(warpName)s warps were found to build the template coadd which is
2263 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2264 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2265 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2267 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2268 another algorithm like:
2270 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2271 config.assemble.retarget(SafeClipAssembleCoaddTask)
2272 """ % {
"warpName": warpName}
2275 @utils.inheritDoc(AssembleCoaddTask)
2276 @pipeBase.timeMethod
2277 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2278 supplementaryData, *args, **kwargs):
2279 """Assemble the coadd.
2281 Find artifacts and apply them to the warps' masks creating a list of
2282 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2283 plane. Then pass these alternative masks to the base class's `run`
2286 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2287 that must contain a ``templateCoadd`` that serves as the
2288 model of the static sky.
2294 dataIds = [ref.dataId
for ref
in tempExpRefList]
2295 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
2297 if dataIds != psfMatchedDataIds:
2298 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2299 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2300 psfMatchedDataIds, dataIds)
2301 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2302 psfMatchedDataIds, dataIds)
2305 spanSetMaskList = self.
findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2306 supplementaryData.warpRefList,
2307 supplementaryData.imageScalerList)
2309 badMaskPlanes = self.config.badMaskPlanes[:]
2310 badMaskPlanes.append(
"CLIPPED")
2311 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2313 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2314 spanSetMaskList, mask=badPixelMask)
2318 self.
applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2322 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2326 mask : `lsst.afw.image.Mask`
2328 altMaskList : `list`
2329 List of Dicts containing ``spanSet`` lists.
2330 Each element contains the new mask plane name (e.g. "CLIPPED
2331 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2334 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2335 for visitMask
in altMaskList:
2336 if "EDGE" in visitMask:
2337 for spanSet
in visitMask[
'EDGE']:
2338 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2343 Loop through warps twice. The first loop builds a map with the count
2344 of how many epochs each pixel deviates from the templateCoadd by more
2345 than ``config.chiThreshold`` sigma. The second loop takes each
2346 difference image and filters the artifacts detected in each using
2347 count map to filter out variable sources and sources that are
2348 difficult to subtract cleanly.
2352 templateCoadd : `lsst.afw.image.Exposure`
2353 Exposure to serve as model of static sky.
2354 tempExpRefList : `list`
2355 List of data references to warps.
2356 imageScalerList : `list`
2357 List of image scalers.
2362 List of dicts containing information about CLIPPED
2363 (i.e., artifacts), NO_DATA, and EDGE pixels.
2366 self.log.debug(
"Generating Count Image, and mask lists.")
2367 coaddBBox = templateCoadd.getBBox()
2368 slateIm = afwImage.ImageU(coaddBBox)
2369 epochCountImage = afwImage.ImageU(coaddBBox)
2370 nImage = afwImage.ImageU(coaddBBox)
2371 spanSetArtifactList = []
2372 spanSetNoDataMaskList = []
2373 spanSetEdgeList = []
2374 spanSetBadMorphoList = []
2375 badPixelMask = self.getBadPixelMask()
2378 templateCoadd.mask.clearAllMaskPlanes()
2380 if self.config.doPreserveContainedBySource:
2381 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2383 templateFootprints =
None
2385 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2387 if warpDiffExp
is not None:
2389 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2390 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2391 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2392 fpSet.positive.merge(fpSet.negative)
2393 footprints = fpSet.positive
2395 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2398 if self.config.doPrefilterArtifacts:
2402 self.detect.clearMask(warpDiffExp.mask)
2403 for spans
in spanSetList:
2404 spans.setImage(slateIm, 1, doClip=
True)
2405 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
2406 epochCountImage += slateIm
2408 if self.config.doFilterMorphological:
2409 maskName = self.config.streakMaskName
2410 _ = self.maskStreaks.
run(warpDiffExp)
2411 streakMask = warpDiffExp.mask
2412 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2413 streakMask.getPlaneBitMask(maskName)).split()
2419 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2420 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2421 nansMask.setXY0(warpDiffExp.getXY0())
2422 edgeMask = warpDiffExp.mask
2423 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2424 edgeMask.getPlaneBitMask(
"EDGE")).split()
2428 nansMask = afwImage.MaskX(coaddBBox, 1)
2430 spanSetEdgeMask = []
2433 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2435 spanSetNoDataMaskList.append(spanSetNoDataMask)
2436 spanSetArtifactList.append(spanSetList)
2437 spanSetEdgeList.append(spanSetEdgeMask)
2438 if self.config.doFilterMorphological:
2439 spanSetBadMorphoList.append(spanSetStreak)
2442 path = self.
_dataRef2DebugPath_dataRef2DebugPath(
"epochCountIm", tempExpRefList[0], coaddLevel=
True)
2443 epochCountImage.writeFits(path)
2445 for i, spanSetList
in enumerate(spanSetArtifactList):
2447 filteredSpanSetList = self.
filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2449 spanSetArtifactList[i] = filteredSpanSetList
2450 if self.config.doFilterMorphological:
2451 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2454 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2455 altMasks.append({
'CLIPPED': artifacts,
2461 """Remove artifact candidates covered by bad mask plane.
2463 Any future editing of the candidate list that does not depend on
2464 temporal information should go in this method.
2468 spanSetList : `list`
2469 List of SpanSets representing artifact candidates.
2470 exp : `lsst.afw.image.Exposure`
2471 Exposure containing mask planes used to prefilter.
2475 returnSpanSetList : `list`
2476 List of SpanSets with artifacts.
2478 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2479 goodArr = (exp.mask.array & badPixelMask) == 0
2480 returnSpanSetList = []
2481 bbox = exp.getBBox()
2482 x0, y0 = exp.getXY0()
2483 for i, span
in enumerate(spanSetList):
2484 y, x = span.clippedTo(bbox).indices()
2485 yIndexLocal = numpy.array(y) - y0
2486 xIndexLocal = numpy.array(x) - x0
2487 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2488 if goodRatio > self.config.prefilterArtifactsRatio:
2489 returnSpanSetList.append(span)
2490 return returnSpanSetList
2492 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2493 """Filter artifact candidates.
2497 spanSetList : `list`
2498 List of SpanSets representing artifact candidates.
2499 epochCountImage : `lsst.afw.image.Image`
2500 Image of accumulated number of warpDiff detections.
2501 nImage : `lsst.afw.image.Image`
2502 Image of the accumulated number of total epochs contributing.
2506 maskSpanSetList : `list`
2507 List of SpanSets with artifacts.
2510 maskSpanSetList = []
2511 x0, y0 = epochCountImage.getXY0()
2512 for i, span
in enumerate(spanSetList):
2513 y, x = span.indices()
2514 yIdxLocal = [y1 - y0
for y1
in y]
2515 xIdxLocal = [x1 - x0
for x1
in x]
2516 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2517 totalN = nImage.array[yIdxLocal, xIdxLocal]
2520 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2521 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2522 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2523 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2524 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2525 & (outlierN <= effectiveMaxNumEpochs))
2526 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2527 if percentBelowThreshold > self.config.spatialThreshold:
2528 maskSpanSetList.append(span)
2530 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2532 filteredMaskSpanSetList = []
2533 for span
in maskSpanSetList:
2535 for footprint
in footprintsToExclude.positive.getFootprints():
2536 if footprint.spans.contains(span):
2540 filteredMaskSpanSetList.append(span)
2541 maskSpanSetList = filteredMaskSpanSetList
2543 return maskSpanSetList
2545 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2546 """Fetch a warp from the butler and return a warpDiff.
2550 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2551 Butler dataRef for the warp.
2552 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2553 An image scaler object.
2554 templateCoadd : `lsst.afw.image.Exposure`
2555 Exposure to be substracted from the scaled warp.
2559 warp : `lsst.afw.image.Exposure`
2560 Exposure of the image difference between the warp and template.
2568 warpName = self.getTempExpDatasetName(
'psfMatched')
2569 if not isinstance(warpRef, DeferredDatasetHandle):
2570 if not warpRef.datasetExists(warpName):
2571 self.log.warning(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2573 warp = warpRef.get(datasetType=warpName, immediate=
True)
2575 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2576 mi = warp.getMaskedImage()
2577 if self.config.doScaleWarpVariance:
2579 self.scaleWarpVariance.
run(mi)
2580 except Exception
as exc:
2581 self.log.warning(
"Unable to rescale variance of warp (%s); leaving it as-is", exc)
2582 mi -= templateCoadd.getMaskedImage()
2585 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2586 """Return a path to which to write debugging output.
2588 Creates a hyphen-delimited string of dataId values for simple filenames.
2593 Prefix for filename.
2594 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2595 Butler dataRef to make the path from.
2596 coaddLevel : `bool`, optional.
2597 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2598 'filter', but no 'visit').
2603 Path for debugging output.
2606 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2608 keys = warpRef.dataId.keys()
2609 keyList = sorted(keys, reverse=
True)
2611 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2612 return os.path.join(directory, filename)
def makeDataRefList(self, namespace)
def __init__(self, *config=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def applyAltEdgeMask(self, mask, altMaskList)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def __init__(self, *args, **kwargs)
def detectClip(self, exp, tempExpRefList)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def setInexactPsf(self, mask)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeCoaddSuffix(warpType="direct")
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")