lsst.pipe.tasks g38dd528b55+8e7d1b62c5
assembleCoadd.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import copy
24import numpy
25import warnings
26import logging
27import lsst.pex.config as pexConfig
28import lsst.pex.exceptions as pexExceptions
29import lsst.geom as geom
30import lsst.afw.geom as afwGeom
31import lsst.afw.image as afwImage
32import lsst.afw.math as afwMath
33import lsst.afw.table as afwTable
34import lsst.afw.detection as afwDet
35import lsst.coadd.utils as coaddUtils
36import lsst.pipe.base as pipeBase
37import lsst.meas.algorithms as measAlg
38import lsstDebug
39import lsst.utils as utils
40from lsst.skymap import BaseSkyMap
41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42from .interpImage import InterpImageTask
43from .scaleZeroPoint import ScaleZeroPointTask
44from .coaddHelpers import groupPatchExposures, getGroupDataRef
45from .scaleVariance import ScaleVarianceTask
46from .maskStreaks import MaskStreaksTask
47from .healSparseMapping import HealSparseInputMapTask
48from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack
49from lsst.daf.butler import DeferredDatasetHandle
50from lsst.utils.timer import timeMethod
51
52__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
53 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
54 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
55
56log = logging.getLogger(__name__)
57
58
59class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("tract", "patch", "band", "skymap"),
61 defaultTemplates={"inputCoaddName": "deep",
62 "outputCoaddName": "deep",
63 "warpType": "direct",
64 "warpTypeSuffix": ""}):
65
66 inputWarps = pipeBase.connectionTypes.Input(
67 doc=("Input list of warps to be assemebled i.e. stacked."
68 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
69 name="{inputCoaddName}Coadd_{warpType}Warp",
70 storageClass="ExposureF",
71 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
72 deferLoad=True,
73 multiple=True
74 )
75 skyMap = pipeBase.connectionTypes.Input(
76 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78 storageClass="SkyMap",
79 dimensions=("skymap", ),
80 )
81 selectedVisits = pipeBase.connectionTypes.Input(
82 doc="Selected visits to be coadded.",
83 name="{outputCoaddName}Visits",
84 storageClass="StructuredDataDict",
85 dimensions=("instrument", "tract", "patch", "skymap", "band")
86 )
87 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
88 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
89 " BRIGHT_OBJECT."),
90 name="brightObjectMask",
91 storageClass="ObjectMaskCatalog",
92 dimensions=("tract", "patch", "skymap", "band"),
93 )
94 coaddExposure = pipeBase.connectionTypes.Output(
95 doc="Output coadded exposure, produced by stacking input warps",
96 name="{outputCoaddName}Coadd{warpTypeSuffix}",
97 storageClass="ExposureF",
98 dimensions=("tract", "patch", "skymap", "band"),
99 )
100 nImage = pipeBase.connectionTypes.Output(
101 doc="Output image of number of input images per pixel",
102 name="{outputCoaddName}Coadd_nImage",
103 storageClass="ImageU",
104 dimensions=("tract", "patch", "skymap", "band"),
105 )
106 inputMap = pipeBase.connectionTypes.Output(
107 doc="Output healsparse map of input images",
108 name="{outputCoaddName}Coadd_inputMap",
109 storageClass="HealSparseMap",
110 dimensions=("tract", "patch", "skymap", "band"),
111 )
112
113 def __init__(self, *, config=None):
114 super().__init__(config=config)
115
116 # Override the connection's name template with config to replicate Gen2 behavior
117 # This duplicates some of the logic in the base class, due to wanting Gen2 and
118 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
119 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
120 templateValues['warpType'] = config.warpType
121 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
122 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
123 for name in self.allConnections}
124 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
125 # End code to remove after deprecation
126
127 if not config.doMaskBrightObjects:
128 self.prerequisiteInputs.remove("brightObjectMask")
129
130 if not config.doSelectVisits:
131 self.inputs.remove("selectedVisits")
132
133 if not config.doNImage:
134 self.outputs.remove("nImage")
135
136 if not self.config.doInputMap:
137 self.outputs.remove("inputMap")
138
139
140class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
141 pipelineConnections=AssembleCoaddConnections):
142 """Configuration parameters for the `AssembleCoaddTask`.
143
144 Notes
145 -----
146 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
147 only set the bitplane config.brightObjectMaskName. To make this useful you
148 *must* also configure the flags.pixel algorithm, for example by adding
149
150 .. code-block:: none
151
152 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
153 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
154
155 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
156 """
157 warpType = pexConfig.Field(
158 doc="Warp name: one of 'direct' or 'psfMatched'",
159 dtype=str,
160 default="direct",
161 )
162 subregionSize = pexConfig.ListField(
163 dtype=int,
164 doc="Width, height of stack subregion size; "
165 "make small enough that a full stack of images will fit into memory at once.",
166 length=2,
167 default=(2000, 2000),
168 )
169 statistic = pexConfig.Field(
170 dtype=str,
171 doc="Main stacking statistic for aggregating over the epochs.",
172 default="MEANCLIP",
173 )
174 doOnlineForMean = pexConfig.Field(
175 dtype=bool,
176 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?",
177 default=False,
178 )
179 doSigmaClip = pexConfig.Field(
180 dtype=bool,
181 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
182 default=False,
183 )
184 sigmaClip = pexConfig.Field(
185 dtype=float,
186 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
187 default=3.0,
188 )
189 clipIter = pexConfig.Field(
190 dtype=int,
191 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
192 default=2,
193 )
194 calcErrorFromInputVariance = pexConfig.Field(
195 dtype=bool,
196 doc="Calculate coadd variance from input variance by stacking statistic."
197 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
198 default=True,
199 )
200 scaleZeroPoint = pexConfig.ConfigurableField(
201 target=ScaleZeroPointTask,
202 doc="Task to adjust the photometric zero point of the coadd temp exposures",
203 )
204 doInterp = pexConfig.Field(
205 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
206 dtype=bool,
207 default=True,
208 )
209 interpImage = pexConfig.ConfigurableField(
210 target=InterpImageTask,
211 doc="Task to interpolate (and extrapolate) over NaN pixels",
212 )
213 doWrite = pexConfig.Field(
214 doc="Persist coadd?",
215 dtype=bool,
216 default=True,
217 )
218 doNImage = pexConfig.Field(
219 doc="Create image of number of contributing exposures for each pixel",
220 dtype=bool,
221 default=False,
222 )
223 doUsePsfMatchedPolygons = pexConfig.Field(
224 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
225 dtype=bool,
226 default=False,
227 )
228 maskPropagationThresholds = pexConfig.DictField(
229 keytype=str,
230 itemtype=float,
231 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
232 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
233 "would have contributed exceeds this value."),
234 default={"SAT": 0.1},
235 )
236 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
237 doc="Mask planes to remove before coadding")
238 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
239 doc="Set mask and flag bits for bright objects?")
240 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
241 doc="Name of mask bit used for bright objects")
242 coaddPsf = pexConfig.ConfigField(
243 doc="Configuration for CoaddPsf",
244 dtype=measAlg.CoaddPsfConfig,
245 )
246 doAttachTransmissionCurve = pexConfig.Field(
247 dtype=bool, default=False, optional=False,
248 doc=("Attach a piecewise TransmissionCurve for the coadd? "
249 "(requires all input Exposures to have TransmissionCurves).")
250 )
251 hasFakes = pexConfig.Field(
252 dtype=bool,
253 default=False,
254 doc="Should be set to True if fake sources have been inserted into the input data."
255 )
256 doSelectVisits = pexConfig.Field(
257 doc="Coadd only visits selected by a SelectVisitsTask",
258 dtype=bool,
259 default=False,
260 )
261 doInputMap = pexConfig.Field(
262 doc="Create a bitwise map of coadd inputs",
263 dtype=bool,
264 default=False,
265 )
266 inputMapper = pexConfig.ConfigurableField(
267 doc="Input map creation subtask.",
268 target=HealSparseInputMapTask,
269 )
270
271 def setDefaults(self):
272 super().setDefaults()
273 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
274
275 def validate(self):
276 super().validate()
277 if self.doPsfMatch:
278 # Backwards compatibility.
279 # Configs do not have loggers
280 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
281 self.warpType = 'psfMatched'
282 if self.doSigmaClip and self.statistic != "MEANCLIP":
283 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
284 self.statistic = "MEANCLIP"
285 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
286 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
287 "compute and set a non-zero coadd variance estimate." % (self.statistic))
288
289 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
290 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
291 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
292 if str(k) not in unstackableStats]
293 raise ValueError("statistic %s is not allowed. Please choose one of %s."
294 % (self.statistic, stackableStats))
295
296
297class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
298 """Assemble a coadded image from a set of warps (coadded temporary exposures).
299
300 We want to assemble a coadded image from a set of Warps (also called
301 coadded temporary exposures or ``coaddTempExps``).
302 Each input Warp covers a patch on the sky and corresponds to a single
303 run/visit/exposure of the covered patch. We provide the task with a list
304 of Warps (``selectDataList``) from which it selects Warps that cover the
305 specified patch (pointed at by ``dataRef``).
306 Each Warp that goes into a coadd will typically have an independent
307 photometric zero-point. Therefore, we must scale each Warp to set it to
308 a common photometric zeropoint. WarpType may be one of 'direct' or
309 'psfMatched', and the boolean configs `config.makeDirect` and
310 `config.makePsfMatched` set which of the warp types will be coadded.
311 The coadd is computed as a mean with optional outlier rejection.
312 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
313 Finally, Warps can have bad 'NaN' pixels which received no input from the
314 source calExps. We interpolate over these bad (NaN) pixels.
315
316 `AssembleCoaddTask` uses several sub-tasks. These are
317
318 - `ScaleZeroPointTask`
319 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
320 - `InterpImageTask`
321 - interpolate across bad pixels (NaN) in the final coadd
322
323 You can retarget these subtasks if you wish.
324
325 Notes
326 -----
327 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
328 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
329 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
330 no debug variables of its own. Some of the subtasks may support debug
331 variables. See the documentation for the subtasks for further information.
332
333 Examples
334 --------
335 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
336 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
337 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
338 inputs: a data reference to the tract patch and filter to be coadded, and
339 a list of Warps to attempt to coadd. These are specified using ``--id`` and
340 ``--selectId``, respectively:
341
342 .. code-block:: none
343
344 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
345 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
346
347 Only the Warps that cover the specified tract and patch will be coadded.
348 A list of the available optional arguments can be obtained by calling
349 ``assembleCoadd.py`` with the ``--help`` command line argument:
350
351 .. code-block:: none
352
353 assembleCoadd.py --help
354
355 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
356 multi-band processing, we will generate the HSC-I & -R band coadds from
357 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
358 assuming that the lsst stack has been already set up, we must set up the
359 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
360 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
361 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
362 coadds, we must first
363
364 - processCcd
365 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
366 - makeSkyMap
367 - create a skymap that covers the area of the sky present in the raw exposures
368 - makeCoaddTempExp
369 - warp the individual calibrated exposures to the tangent plane of the coadd
370
371 We can perform all of these steps by running
372
373 .. code-block:: none
374
375 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
376
377 This will produce warped exposures for each visit. To coadd the warped
378 data, we call assembleCoadd.py as follows:
379
380 .. code-block:: none
381
382 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
383 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
384 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
385 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
386 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
387 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
388 --selectId visit=903988 ccd=24
389
390 that will process the HSC-I band data. The results are written in
391 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
392
393 You may also choose to run:
394
395 .. code-block:: none
396
397 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
398 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
399 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
400 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
401 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
402 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
403 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
404 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
405
406 to generate the coadd for the HSC-R band if you are interested in
407 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
408 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
409 rather than `AssembleCoaddTask` to make the coadd.
410 """
411 ConfigClass = AssembleCoaddConfig
412 _DefaultName = "assembleCoadd"
413
414 def __init__(self, *args, **kwargs):
415 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
416 if args:
417 argNames = ["config", "name", "parentTask", "log"]
418 kwargs.update({k: v for k, v in zip(argNames, args)})
419 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
420 "PipelineTask will not take positional args" % argNames, FutureWarning)
421
422 super().__init__(**kwargs)
423 self.makeSubtask("interpImage")
424 self.makeSubtask("scaleZeroPoint")
425
426 if self.config.doMaskBrightObjects:
427 mask = afwImage.Mask()
428 try:
429 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
430 except pexExceptions.LsstCppException:
431 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
432 mask.getMaskPlaneDict().keys())
433 del mask
434
435 if self.config.doInputMap:
436 self.makeSubtask("inputMapper")
437
438 self.warpType = self.config.warpType
439
440 @utils.inheritDoc(pipeBase.PipelineTask)
441 def runQuantum(self, butlerQC, inputRefs, outputRefs):
442 # Docstring to be formatted with info from PipelineTask.runQuantum
443 """
444 Notes
445 -----
446 Assemble a coadd from a set of Warps.
447
448 PipelineTask (Gen3) entry point to Coadd a set of Warps.
449 Analogous to `runDataRef`, it prepares all the data products to be
450 passed to `run`, and processes the results before returning a struct
451 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
452 Therefore, its inputs are accessed subregion by subregion
453 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
454 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
455 correspond to an update in `runDataRef` while both entry points
456 are used.
457 """
458 inputData = butlerQC.get(inputRefs)
459
460 # Construct skyInfo expected by run
461 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
462 skyMap = inputData["skyMap"]
463 outputDataId = butlerQC.quantum.dataId
464
465 inputData['skyInfo'] = makeSkyInfo(skyMap,
466 tractId=outputDataId['tract'],
467 patchId=outputDataId['patch'])
468
469 if self.config.doSelectVisits:
470 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
471 else:
472 warpRefList = inputData['inputWarps']
473
474 # Perform same middle steps as `runDataRef` does
475 inputs = self.prepareInputs(warpRefList)
476 self.log.info("Found %d %s", len(inputs.tempExpRefList),
477 self.getTempExpDatasetName(self.warpType))
478 if len(inputs.tempExpRefList) == 0:
479 raise pipeBase.NoWorkFound("No coadd temporary exposures found")
480
481 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
482 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
483 inputs.weightList, supplementaryData=supplementaryData)
484
485 inputData.setdefault('brightObjectMask', None)
486 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
487
488 if self.config.doWrite:
489 butlerQC.put(retStruct, outputRefs)
490 return retStruct
491
492 @timeMethod
493 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
494 """Assemble a coadd from a set of Warps.
495
496 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
497 Compute weights to be applied to each Warp and
498 find scalings to match the photometric zeropoint to a reference Warp.
499 Assemble the Warps using `run`. Interpolate over NaNs and
500 optionally write the coadd to disk. Return the coadded exposure.
501
502 Parameters
503 ----------
505 Data reference defining the patch for coaddition and the
506 reference Warp (if ``config.autoReference=False``).
507 Used to access the following data products:
508 - ``self.config.coaddName + "Coadd_skyMap"``
509 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
510 - ``self.config.coaddName + "Coadd"``
511 selectDataList : `list`
512 List of data references to Calexps. Data to be coadded will be
513 selected from this list based on overlap with the patch defined
514 by dataRef, grouped by visit, and converted to a list of data
515 references to warps.
516 warpRefList : `list`
517 List of data references to Warps to be coadded.
518 Note: `warpRefList` is just the new name for `tempExpRefList`.
519
520 Returns
521 -------
522 retStruct : `lsst.pipe.base.Struct`
523 Result struct with components:
524
525 - ``coaddExposure``: coadded exposure (``Exposure``).
526 - ``nImage``: exposure count image (``Image``).
527 """
528 if selectDataList and warpRefList:
529 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
530 "and which to use is ambiguous. Please pass only one.")
531
532 skyInfo = self.getSkyInfo(dataRef)
533 if warpRefList is None:
534 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
535 if len(calExpRefList) == 0:
536 self.log.warning("No exposures to coadd")
537 return
538 self.log.info("Coadding %d exposures", len(calExpRefList))
539
540 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
541
542 inputData = self.prepareInputs(warpRefList)
543 self.log.info("Found %d %s", len(inputData.tempExpRefList),
544 self.getTempExpDatasetName(self.warpType))
545 if len(inputData.tempExpRefList) == 0:
546 self.log.warning("No coadd temporary exposures found")
547 return
548
549 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
550
551 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
552 inputData.weightList, supplementaryData=supplementaryData)
553
554 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
555 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
556
557 if self.config.doWrite:
558 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
559 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
560 else:
561 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
562 self.log.info("Persisting %s", coaddDatasetName)
563 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
564 if self.config.doNImage and retStruct.nImage is not None:
565 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
566
567 return retStruct
568
569 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
570 """Interpolate over missing data and mask bright stars.
571
572 Parameters
573 ----------
574 coaddExposure : `lsst.afw.image.Exposure`
575 The coadded exposure to process.
576 dataRef : `lsst.daf.persistence.ButlerDataRef`
577 Butler data reference for supplementary data.
578 """
579 if self.config.doInterp:
580 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
581 # The variance must be positive; work around for DM-3201.
582 varArray = coaddExposure.variance.array
583 with numpy.errstate(invalid="ignore"):
584 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
585
586 if self.config.doMaskBrightObjects:
587 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
588
589 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
590 """Make additional inputs to run() specific to subclasses (Gen2)
591
592 Duplicates interface of `runDataRef` method
593 Available to be implemented by subclasses only if they need the
594 coadd dataRef for performing preliminary processing before
595 assembling the coadd.
596
597 Parameters
598 ----------
599 dataRef : `lsst.daf.persistence.ButlerDataRef`
600 Butler data reference for supplementary data.
601 selectDataList : `list` (optional)
602 Optional List of data references to Calexps.
603 warpRefList : `list` (optional)
604 Optional List of data references to Warps.
605 """
606 return pipeBase.Struct()
607
608 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
609 """Make additional inputs to run() specific to subclasses (Gen3)
610
611 Duplicates interface of `runQuantum` method.
612 Available to be implemented by subclasses only if they need the
613 coadd dataRef for performing preliminary processing before
614 assembling the coadd.
615
616 Parameters
617 ----------
618 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
619 Gen3 Butler object for fetching additional data products before
620 running the Task specialized for quantum being processed
621 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
622 Attributes are the names of the connections describing input dataset types.
623 Values are DatasetRefs that task consumes for corresponding dataset type.
624 DataIds are guaranteed to match data objects in ``inputData``.
625 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
626 Attributes are the names of the connections describing output dataset types.
627 Values are DatasetRefs that task is to produce
628 for corresponding dataset type.
629 """
630 return pipeBase.Struct()
631
632 def getTempExpRefList(self, patchRef, calExpRefList):
633 """Generate list data references corresponding to warped exposures
634 that lie within the patch to be coadded.
635
636 Parameters
637 ----------
638 patchRef : `dataRef`
639 Data reference for patch.
640 calExpRefList : `list`
641 List of data references for input calexps.
642
643 Returns
644 -------
645 tempExpRefList : `list`
646 List of Warp/CoaddTempExp data references.
647 """
648 butler = patchRef.getButler()
649 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
650 self.getTempExpDatasetName(self.warpType))
651 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
652 g, groupData.keys) for
653 g in groupData.groups.keys()]
654 return tempExpRefList
655
656 def prepareInputs(self, refList):
657 """Prepare the input warps for coaddition by measuring the weight for
658 each warp and the scaling for the photometric zero point.
659
660 Each Warp has its own photometric zeropoint and background variance.
661 Before coadding these Warps together, compute a scale factor to
662 normalize the photometric zeropoint and compute the weight for each Warp.
663
664 Parameters
665 ----------
666 refList : `list`
667 List of data references to tempExp
668
669 Returns
670 -------
671 result : `lsst.pipe.base.Struct`
672 Result struct with components:
673
674 - ``tempExprefList``: `list` of data references to tempExp.
675 - ``weightList``: `list` of weightings.
676 - ``imageScalerList``: `list` of image scalers.
677 """
678 statsCtrl = afwMath.StatisticsControl()
679 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
680 statsCtrl.setNumIter(self.config.clipIter)
681 statsCtrl.setAndMask(self.getBadPixelMask())
682 statsCtrl.setNanSafe(True)
683 # compute tempExpRefList: a list of tempExpRef that actually exist
684 # and weightList: a list of the weight of the associated coadd tempExp
685 # and imageScalerList: a list of scale factors for the associated coadd tempExp
686 tempExpRefList = []
687 weightList = []
688 imageScalerList = []
689 tempExpName = self.getTempExpDatasetName(self.warpType)
690 for tempExpRef in refList:
691 # Gen3's DeferredDatasetHandles are guaranteed to exist and
692 # therefore have no datasetExists() method
693 if not isinstance(tempExpRef, DeferredDatasetHandle):
694 if not tempExpRef.datasetExists(tempExpName):
695 self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
696 continue
697
698 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
699 # Ignore any input warp that is empty of data
700 if numpy.isnan(tempExp.image.array).all():
701 continue
702 maskedImage = tempExp.getMaskedImage()
703 imageScaler = self.scaleZeroPoint.computeImageScaler(
704 exposure=tempExp,
705 dataRef=tempExpRef,
706 )
707 try:
708 imageScaler.scaleMaskedImage(maskedImage)
709 except Exception as e:
710 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
711 continue
712 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
713 afwMath.MEANCLIP, statsCtrl)
714 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
715 weight = 1.0 / float(meanVar)
716 if not numpy.isfinite(weight):
717 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId)
718 continue
719 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
720
721 del maskedImage
722 del tempExp
723
724 tempExpRefList.append(tempExpRef)
725 weightList.append(weight)
726 imageScalerList.append(imageScaler)
727
728 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
729 imageScalerList=imageScalerList)
730
731 def prepareStats(self, mask=None):
732 """Prepare the statistics for coadding images.
733
734 Parameters
735 ----------
736 mask : `int`, optional
737 Bit mask value to exclude from coaddition.
738
739 Returns
740 -------
741 stats : `lsst.pipe.base.Struct`
742 Statistics structure with the following fields:
743
744 - ``statsCtrl``: Statistics control object for coadd
746 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
747 """
748 if mask is None:
749 mask = self.getBadPixelMask()
750 statsCtrl = afwMath.StatisticsControl()
751 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
752 statsCtrl.setNumIter(self.config.clipIter)
753 statsCtrl.setAndMask(mask)
754 statsCtrl.setNanSafe(True)
755 statsCtrl.setWeighted(True)
756 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
757 for plane, threshold in self.config.maskPropagationThresholds.items():
758 bit = afwImage.Mask.getMaskPlane(plane)
759 statsCtrl.setMaskPropagationThreshold(bit, threshold)
760 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
761 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
762
763 @timeMethod
764 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
765 altMaskList=None, mask=None, supplementaryData=None):
766 """Assemble a coadd from input warps
767
768 Assemble the coadd using the provided list of coaddTempExps. Since
769 the full coadd covers a patch (a large area), the assembly is
770 performed over small areas on the image at a time in order to
771 conserve memory usage. Iterate over subregions within the outer
772 bbox of the patch using `assembleSubregion` to stack the corresponding
773 subregions from the coaddTempExps with the statistic specified.
774 Set the edge bits the coadd mask based on the weight map.
775
776 Parameters
777 ----------
778 skyInfo : `lsst.pipe.base.Struct`
779 Struct with geometric information about the patch.
780 tempExpRefList : `list`
781 List of data references to Warps (previously called CoaddTempExps).
782 imageScalerList : `list`
783 List of image scalers.
784 weightList : `list`
785 List of weights
786 altMaskList : `list`, optional
787 List of alternate masks to use rather than those stored with
788 tempExp.
789 mask : `int`, optional
790 Bit mask value to exclude from coaddition.
791 supplementaryData : lsst.pipe.base.Struct, optional
792 Struct with additional data products needed to assemble coadd.
793 Only used by subclasses that implement `makeSupplementaryData`
794 and override `run`.
795
796 Returns
797 -------
798 result : `lsst.pipe.base.Struct`
799 Result struct with components:
800
801 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
802 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
803 - ``inputMap``: bit-wise map of inputs, if requested.
804 - ``warpRefList``: input list of refs to the warps (
805 ``lsst.daf.butler.DeferredDatasetHandle`` or
806 ``lsst.daf.persistence.ButlerDataRef``)
807 (unmodified)
808 - ``imageScalerList``: input list of image scalers (unmodified)
809 - ``weightList``: input list of weights (unmodified)
810 """
811 tempExpName = self.getTempExpDatasetName(self.warpType)
812 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
813 stats = self.prepareStats(mask=mask)
814
815 if altMaskList is None:
816 altMaskList = [None]*len(tempExpRefList)
817
818 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
819 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
820 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
821 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
822 coaddMaskedImage = coaddExposure.getMaskedImage()
823 subregionSizeArr = self.config.subregionSize
824 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
825 # if nImage is requested, create a zero one which can be passed to assembleSubregion
826 if self.config.doNImage:
827 nImage = afwImage.ImageU(skyInfo.bbox)
828 else:
829 nImage = None
830 # If inputMap is requested, create the initial version that can be masked in
831 # assembleSubregion.
832 if self.config.doInputMap:
833 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
834 skyInfo.wcs,
835 coaddExposure.getInfo().getCoaddInputs().ccds)
836
837 if self.config.doOnlineForMean and self.config.statistic == "MEAN":
838 try:
839 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
840 weightList, altMaskList, stats.ctrl,
841 nImage=nImage)
842 except Exception as e:
843 self.log.fatal("Cannot compute online coadd %s", e)
844 else:
845 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
846 try:
847 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
848 weightList, altMaskList, stats.flags, stats.ctrl,
849 nImage=nImage)
850 except Exception as e:
851 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
852
853 # If inputMap is requested, we must finalize the map after the accumulation.
854 if self.config.doInputMap:
855 self.inputMapper.finalize_ccd_input_map_mask()
856 inputMap = self.inputMapper.ccd_input_map
857 else:
858 inputMap = None
859
860 self.setInexactPsf(coaddMaskedImage.getMask())
861 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
862 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
863 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
864 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
865 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
866 weightList=weightList, inputMap=inputMap)
867
868 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
869 """Set the metadata for the coadd.
870
871 This basic implementation sets the filter from the first input.
872
873 Parameters
874 ----------
875 coaddExposure : `lsst.afw.image.Exposure`
876 The target exposure for the coadd.
877 tempExpRefList : `list`
878 List of data references to tempExp.
879 weightList : `list`
880 List of weights.
881 """
882 assert len(tempExpRefList) == len(weightList), "Length mismatch"
883 tempExpName = self.getTempExpDatasetName(self.warpType)
884 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
885 # (and we need more than just the PropertySet that contains the header), which is not possible
886 # with the current butler (see #2777).
887 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
888
889 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
890 # Gen 3 API
891 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
892 else:
893 # Gen 2 API. Delete this when Gen 2 retired
894 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
895 for tempExpRef in tempExpRefList]
896 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
897
898 # Set the coadd FilterLabel to the band of the first input exposure:
899 # Coadds are calibrated, so the physical label is now meaningless.
900 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
901 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
902 coaddInputs.ccds.reserve(numCcds)
903 coaddInputs.visits.reserve(len(tempExpList))
904
905 for tempExp, weight in zip(tempExpList, weightList):
906 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
907
908 if self.config.doUsePsfMatchedPolygons:
909 self.shrinkValidPolygons(coaddInputs)
910
911 coaddInputs.visits.sort()
912 coaddInputs.ccds.sort()
913 if self.warpType == "psfMatched":
914 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
915 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
916 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
917 # having the maximum width (sufficient because square)
918 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
919 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
920 for modelPsf in modelPsfList]
921 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
922 else:
923 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
924 self.config.coaddPsf.makeControl())
925 coaddExposure.setPsf(psf)
926 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
927 coaddExposure.getWcs())
928 coaddExposure.getInfo().setApCorrMap(apCorrMap)
929 if self.config.doAttachTransmissionCurve:
930 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
931 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
932
933 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
934 altMaskList, statsFlags, statsCtrl, nImage=None):
935 """Assemble the coadd for a sub-region.
936
937 For each coaddTempExp, check for (and swap in) an alternative mask
938 if one is passed. Remove mask planes listed in
939 `config.removeMaskPlanes`. Finally, stack the actual exposures using
940 `lsst.afw.math.statisticsStack` with the statistic specified by
941 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
942 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
943 an N-sigma clipped mean where N and iterations are specified by
944 statsCtrl. Assign the stacked subregion back to the coadd.
945
946 Parameters
947 ----------
948 coaddExposure : `lsst.afw.image.Exposure`
949 The target exposure for the coadd.
950 bbox : `lsst.geom.Box`
951 Sub-region to coadd.
952 tempExpRefList : `list`
953 List of data reference to tempExp.
954 imageScalerList : `list`
955 List of image scalers.
956 weightList : `list`
957 List of weights.
958 altMaskList : `list`
959 List of alternate masks to use rather than those stored with
960 tempExp, or None. Each element is dict with keys = mask plane
961 name to which to add the spans.
962 statsFlags : `lsst.afw.math.Property`
963 Property object for statistic for coadd.
965 Statistics control object for coadd.
966 nImage : `lsst.afw.image.ImageU`, optional
967 Keeps track of exposure count for each pixel.
968 """
969 self.log.debug("Computing coadd over %s", bbox)
970 tempExpName = self.getTempExpDatasetName(self.warpType)
971 coaddExposure.mask.addMaskPlane("REJECTED")
972 coaddExposure.mask.addMaskPlane("CLIPPED")
973 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
974 maskMap = self.setRejectedMaskMapping(statsCtrl)
975 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
976 maskedImageList = []
977 if nImage is not None:
978 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
979 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
980
981 if isinstance(tempExpRef, DeferredDatasetHandle):
982 # Gen 3 API
983 exposure = tempExpRef.get(parameters={'bbox': bbox})
984 else:
985 # Gen 2 API. Delete this when Gen 2 retired
986 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
987
988 maskedImage = exposure.getMaskedImage()
989 mask = maskedImage.getMask()
990 if altMask is not None:
991 self.applyAltMaskPlanes(mask, altMask)
992 imageScaler.scaleMaskedImage(maskedImage)
993
994 # Add 1 for each pixel which is not excluded by the exclude mask.
995 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
996 if nImage is not None:
997 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
998 if self.config.removeMaskPlanes:
999 self.removeMaskPlanes(maskedImage)
1000 maskedImageList.append(maskedImage)
1001
1002 if self.config.doInputMap:
1003 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1004 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1005
1006 with self.timer("stack"):
1007 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
1008 clipped, # also set output to CLIPPED if sigma-clipped
1009 maskMap)
1010 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
1011 if nImage is not None:
1012 nImage.assign(subNImage, bbox)
1013
1014 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList,
1015 altMaskList, statsCtrl, nImage=None):
1016 """Assemble the coadd using the "online" method.
1017
1018 This method takes a running sum of images and weights to save memory.
1019 It only works for MEAN statistics.
1020
1021 Parameters
1022 ----------
1023 coaddExposure : `lsst.afw.image.Exposure`
1024 The target exposure for the coadd.
1025 tempExpRefList : `list`
1026 List of data reference to tempExp.
1027 imageScalerList : `list`
1028 List of image scalers.
1029 weightList : `list`
1030 List of weights.
1031 altMaskList : `list`
1032 List of alternate masks to use rather than those stored with
1033 tempExp, or None. Each element is dict with keys = mask plane
1034 name to which to add the spans.
1036 Statistics control object for coadd
1037 nImage : `lsst.afw.image.ImageU`, optional
1038 Keeps track of exposure count for each pixel.
1039 """
1040 self.log.debug("Computing online coadd.")
1041 tempExpName = self.getTempExpDatasetName(self.warpType)
1042 coaddExposure.mask.addMaskPlane("REJECTED")
1043 coaddExposure.mask.addMaskPlane("CLIPPED")
1044 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
1045 maskMap = self.setRejectedMaskMapping(statsCtrl)
1046 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
1047
1048 bbox = coaddExposure.maskedImage.getBBox()
1049
1050 stacker = AccumulatorMeanStack(
1051 coaddExposure.image.array.shape,
1052 statsCtrl.getAndMask(),
1053 mask_threshold_dict=thresholdDict,
1054 mask_map=maskMap,
1055 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
1056 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
1057 compute_n_image=(nImage is not None)
1058 )
1059
1060 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList,
1061 imageScalerList,
1062 altMaskList,
1063 weightList):
1064 if isinstance(tempExpRef, DeferredDatasetHandle):
1065 # Gen 3 API
1066 exposure = tempExpRef.get()
1067 else:
1068 # Gen 2 API. Delete this when Gen 2 retired
1069 exposure = tempExpRef.get(tempExpName)
1070
1071 maskedImage = exposure.getMaskedImage()
1072 mask = maskedImage.getMask()
1073 if altMask is not None:
1074 self.applyAltMaskPlanes(mask, altMask)
1075 imageScaler.scaleMaskedImage(maskedImage)
1076 if self.config.removeMaskPlanes:
1077 self.removeMaskPlanes(maskedImage)
1078
1079 stacker.add_masked_image(maskedImage, weight=weight)
1080
1081 if self.config.doInputMap:
1082 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
1083 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
1084
1085 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
1086
1087 if nImage is not None:
1088 nImage.array[:, :] = stacker.n_image
1089
1090 def removeMaskPlanes(self, maskedImage):
1091 """Unset the mask of an image for mask planes specified in the config.
1092
1093 Parameters
1094 ----------
1095 maskedImage : `lsst.afw.image.MaskedImage`
1096 The masked image to be modified.
1097 """
1098 mask = maskedImage.getMask()
1099 for maskPlane in self.config.removeMaskPlanes:
1100 try:
1101 mask &= ~mask.getPlaneBitMask(maskPlane)
1102 except pexExceptions.InvalidParameterError:
1103 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
1104 maskPlane)
1105
1106 @staticmethod
1107 def setRejectedMaskMapping(statsCtrl):
1108 """Map certain mask planes of the warps to new planes for the coadd.
1109
1110 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1111 or CLIPPED, set it to REJECTED on the coadd.
1112 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1113 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1114
1115 Parameters
1116 ----------
1118 Statistics control object for coadd
1119
1120 Returns
1121 -------
1122 maskMap : `list` of `tuple` of `int`
1123 A list of mappings of mask planes of the warped exposures to
1124 mask planes of the coadd.
1125 """
1126 edge = afwImage.Mask.getPlaneBitMask("EDGE")
1127 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1128 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1129 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1130 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1131 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1132 (clipped, clipped)]
1133 return maskMap
1134
1135 def applyAltMaskPlanes(self, mask, altMaskSpans):
1136 """Apply in place alt mask formatted as SpanSets to a mask.
1137
1138 Parameters
1139 ----------
1140 mask : `lsst.afw.image.Mask`
1141 Original mask.
1142 altMaskSpans : `dict`
1143 SpanSet lists to apply. Each element contains the new mask
1144 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1145 and list of SpanSets to apply to the mask.
1146
1147 Returns
1148 -------
1149 mask : `lsst.afw.image.Mask`
1150 Updated mask.
1151 """
1152 if self.config.doUsePsfMatchedPolygons:
1153 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1154 # Clear away any other masks outside the validPolygons. These pixels are no longer
1155 # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1156 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1157 # This mask-clearing step must only occur *before* applying the new masks below
1158 for spanSet in altMaskSpans['NO_DATA']:
1159 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1160
1161 for plane, spanSetList in altMaskSpans.items():
1162 maskClipValue = mask.addMaskPlane(plane)
1163 for spanSet in spanSetList:
1164 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1165 return mask
1166
1167 def shrinkValidPolygons(self, coaddInputs):
1168 """Shrink coaddInputs' ccds' ValidPolygons in place.
1169
1170 Either modify each ccd's validPolygon in place, or if CoaddInputs
1171 does not have a validPolygon, create one from its bbox.
1172
1173 Parameters
1174 ----------
1175 coaddInputs : `lsst.afw.image.coaddInputs`
1176 Original mask.
1177
1178 """
1179 for ccd in coaddInputs.ccds:
1180 polyOrig = ccd.getValidPolygon()
1181 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1182 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1183 if polyOrig:
1184 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1185 else:
1186 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1187 ccd.setValidPolygon(validPolygon)
1188
1189 def readBrightObjectMasks(self, dataRef):
1190 """Retrieve the bright object masks.
1191
1192 Returns None on failure.
1193
1194 Parameters
1195 ----------
1197 A Butler dataRef.
1198
1199 Returns
1200 -------
1202 Bright object mask from the Butler object, or None if it cannot
1203 be retrieved.
1204 """
1205 try:
1206 return dataRef.get(datasetType="brightObjectMask", immediate=True)
1207 except Exception as e:
1208 self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1209 return None
1210
1211 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1212 """Set the bright object masks.
1213
1214 Parameters
1215 ----------
1216 exposure : `lsst.afw.image.Exposure`
1217 Exposure under consideration.
1219 Data identifier dict for patch.
1220 brightObjectMasks : `lsst.afw.table`
1221 Table of bright objects to mask.
1222 """
1223
1224 if brightObjectMasks is None:
1225 self.log.warning("Unable to apply bright object mask: none supplied")
1226 return
1227 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1228 mask = exposure.getMaskedImage().getMask()
1229 wcs = exposure.getWcs()
1230 plateScale = wcs.getPixelScale().asArcseconds()
1231
1232 for rec in brightObjectMasks:
1233 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1234 if rec["type"] == "box":
1235 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1236 width = rec["width"].asArcseconds()/plateScale # convert to pixels
1237 height = rec["height"].asArcseconds()/plateScale # convert to pixels
1238
1239 halfSize = geom.ExtentI(0.5*width, 0.5*height)
1240 bbox = geom.Box2I(center - halfSize, center + halfSize)
1241
1242 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1243 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1244 spans = afwGeom.SpanSet(bbox)
1245 elif rec["type"] == "circle":
1246 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1247 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1248 else:
1249 self.log.warning("Unexpected region type %s at %s", rec["type"], center)
1250 continue
1251 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1252
1253 def setInexactPsf(self, mask):
1254 """Set INEXACT_PSF mask plane.
1255
1256 If any of the input images isn't represented in the coadd (due to
1257 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1258 these pixels.
1259
1260 Parameters
1261 ----------
1262 mask : `lsst.afw.image.Mask`
1263 Coadded exposure's mask, modified in-place.
1264 """
1265 mask.addMaskPlane("INEXACT_PSF")
1266 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1267 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1268 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1269 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1270 array = mask.getArray()
1271 selected = array & (sensorEdge | clipped | rejected) > 0
1272 array[selected] |= inexactPsf
1273
1274 @classmethod
1275 def _makeArgumentParser(cls):
1276 """Create an argument parser.
1277 """
1278 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1279 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1280 + cls.ConfigClass().warpType + "Warp",
1281 help="data ID, e.g. --id tract=12345 patch=1,2",
1282 ContainerClass=AssembleCoaddDataIdContainer)
1283 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1284 ContainerClass=SelectDataIdContainer)
1285 return parser
1286
1287 @staticmethod
1288 def _subBBoxIter(bbox, subregionSize):
1289 """Iterate over subregions of a bbox.
1290
1291 Parameters
1292 ----------
1293 bbox : `lsst.geom.Box2I`
1294 Bounding box over which to iterate.
1295 subregionSize: `lsst.geom.Extent2I`
1296 Size of sub-bboxes.
1297
1298 Yields
1299 ------
1300 subBBox : `lsst.geom.Box2I`
1301 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1302 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1303 the edges of ``bbox``, but it will never be empty.
1304 """
1305 if bbox.isEmpty():
1306 raise RuntimeError("bbox %s is empty" % (bbox,))
1307 if subregionSize[0] < 1 or subregionSize[1] < 1:
1308 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1309
1310 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1311 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1312 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1313 subBBox.clip(bbox)
1314 if subBBox.isEmpty():
1315 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1316 "colShift=%s, rowShift=%s" %
1317 (bbox, subregionSize, colShift, rowShift))
1318 yield subBBox
1319
1320 def filterWarps(self, inputs, goodVisits):
1321 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1322
1323 Parameters
1324 ----------
1325 inputs : list
1326 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1327 goodVisit : `dict`
1328 Dictionary with good visitIds as the keys. Value ignored.
1329
1330 Returns:
1331 --------
1332 filteredInputs : `list`
1333 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1334 """
1335 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1336 filteredInputs = []
1337 for visit in goodVisits.keys():
1338 if visit in inputWarpDict:
1339 filteredInputs.append(inputWarpDict[visit])
1340 return filteredInputs
1341
1342
1343class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1344 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1345 """
1346
1347 def makeDataRefList(self, namespace):
1348 """Make self.refList from self.idList.
1349
1350 Parameters
1351 ----------
1352 namespace
1353 Results of parsing command-line (with ``butler`` and ``log`` elements).
1354 """
1355 datasetType = namespace.config.coaddName + "Coadd"
1356 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1357
1358 for dataId in self.idList:
1359 # tract and patch are required
1360 for key in keysCoadd:
1361 if key not in dataId:
1362 raise RuntimeError("--id must include " + key)
1363
1364 dataRef = namespace.butler.dataRef(
1365 datasetType=datasetType,
1366 dataId=dataId,
1367 )
1368 self.refList.append(dataRef)
1369
1370
1371def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1372 """Function to count the number of pixels with a specific mask in a
1373 footprint.
1374
1375 Find the intersection of mask & footprint. Count all pixels in the mask
1376 that are in the intersection that have bitmask set but do not have
1377 ignoreMask set. Return the count.
1378
1379 Parameters
1380 ----------
1381 mask : `lsst.afw.image.Mask`
1382 Mask to define intersection region by.
1383 footprint : `lsst.afw.detection.Footprint`
1384 Footprint to define the intersection region by.
1385 bitmask
1386 Specific mask that we wish to count the number of occurances of.
1387 ignoreMask
1388 Pixels to not consider.
1389
1390 Returns
1391 -------
1392 result : `int`
1393 Count of number of pixels in footprint with specified mask.
1394 """
1395 bbox = footprint.getBBox()
1396 bbox.clip(mask.getBBox(afwImage.PARENT))
1397 fp = afwImage.Mask(bbox)
1398 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1399 footprint.spans.setMask(fp, bitmask)
1400 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1401 (subMask.getArray() & ignoreMask) == 0).sum()
1402
1403
1404class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1405 """Configuration parameters for the SafeClipAssembleCoaddTask.
1406 """
1407 clipDetection = pexConfig.ConfigurableField(
1408 target=SourceDetectionTask,
1409 doc="Detect sources on difference between unclipped and clipped coadd")
1410 minClipFootOverlap = pexConfig.Field(
1411 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1412 dtype=float,
1413 default=0.6
1414 )
1415 minClipFootOverlapSingle = pexConfig.Field(
1416 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1417 "clipped when only one visit overlaps",
1418 dtype=float,
1419 default=0.5
1420 )
1421 minClipFootOverlapDouble = pexConfig.Field(
1422 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1423 "clipped when two visits overlap",
1424 dtype=float,
1425 default=0.45
1426 )
1427 maxClipFootOverlapDouble = pexConfig.Field(
1428 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1429 "considering two visits",
1430 dtype=float,
1431 default=0.15
1432 )
1433 minBigOverlap = pexConfig.Field(
1434 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1435 "when labeling clipped footprints",
1436 dtype=int,
1437 default=100
1438 )
1439
1440 def setDefaults(self):
1441 """Set default values for clipDetection.
1442
1443 Notes
1444 -----
1445 The numeric values for these configuration parameters were
1446 empirically determined, future work may further refine them.
1447 """
1448 AssembleCoaddConfig.setDefaults(self)
1449 self.clipDetectionclipDetection.doTempLocalBackground = False
1450 self.clipDetectionclipDetection.reEstimateBackground = False
1451 self.clipDetectionclipDetection.returnOriginalFootprints = False
1452 self.clipDetectionclipDetection.thresholdPolarity = "both"
1453 self.clipDetectionclipDetection.thresholdValue = 2
1454 self.clipDetectionclipDetection.nSigmaToGrow = 2
1455 self.clipDetectionclipDetection.minPixels = 4
1456 self.clipDetectionclipDetection.isotropicGrow = True
1457 self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1458 self.sigmaClipsigmaClip = 1.5
1459 self.clipIterclipIter = 3
1460 self.statisticstatistic = "MEAN"
1461
1462 def validate(self):
1463 if self.doSigmaClipdoSigmaClip:
1464 log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1465 "Ignoring doSigmaClip.")
1466 self.doSigmaClipdoSigmaClip = False
1467 if self.statisticstatistic != "MEAN":
1468 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1469 "(%s chosen). Please set statistic to MEAN."
1470 % (self.statisticstatistic))
1471 AssembleCoaddTask.ConfigClass.validate(self)
1472
1473
1474class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1475 """Assemble a coadded image from a set of coadded temporary exposures,
1476 being careful to clip & flag areas with potential artifacts.
1477
1478 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1479 we clip outliers). The problem with doing this is that when computing the
1480 coadd PSF at a given location, individual visit PSFs from visits with
1481 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1482 In this task, we correct for this behavior by creating a new
1483 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1484 coaddTempExps and the final coadd where
1485
1486 i. difference imaging suggests that there is an outlier and
1487 ii. this outlier appears on only one or two images.
1488
1489 Such regions will not contribute to the final coadd. Furthermore, any
1490 routine to determine the coadd PSF can now be cognizant of clipped regions.
1491 Note that the algorithm implemented by this task is preliminary and works
1492 correctly for HSC data. Parameter modifications and or considerable
1493 redesigning of the algorithm is likley required for other surveys.
1494
1495 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1496 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1497 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1498 if you wish.
1499
1500 Notes
1501 -----
1502 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1503 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1504 see `baseDebug` for more about ``debug.py`` files.
1505 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1506 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1507 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1508 for further information.
1509
1510 Examples
1511 --------
1512 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1513 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1514 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1515
1516 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1517 and filter to be coadded (specified using
1518 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1519 along with a list of coaddTempExps to attempt to coadd (specified using
1520 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1521 Only the coaddTempExps that cover the specified tract and patch will be
1522 coadded. A list of the available optional arguments can be obtained by
1523 calling assembleCoadd.py with the --help command line argument:
1524
1525 .. code-block:: none
1526
1527 assembleCoadd.py --help
1528
1529 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1530 context of multi-band processing, we will generate the HSC-I & -R band
1531 coadds from HSC engineering test data provided in the ci_hsc package.
1532 To begin, assuming that the lsst stack has been already set up, we must
1533 set up the obs_subaru and ci_hsc packages. This defines the environment
1534 variable $CI_HSC_DIR and points at the location of the package. The raw
1535 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1536 the coadds, we must first
1537
1538 - ``processCcd``
1539 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1540 - ``makeSkyMap``
1541 create a skymap that covers the area of the sky present in the raw exposures
1542 - ``makeCoaddTempExp``
1543 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1544
1545 We can perform all of these steps by running
1546
1547 .. code-block:: none
1548
1549 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1550
1551 This will produce warped coaddTempExps for each visit. To coadd the
1552 warped data, we call ``assembleCoadd.py`` as follows:
1553
1554 .. code-block:: none
1555
1556 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1557 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1558 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1559 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1560 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1561 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1562 --selectId visit=903988 ccd=24
1563
1564 This will process the HSC-I band data. The results are written in
1565 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1566
1567 You may also choose to run:
1568
1569 .. code-block:: none
1570
1571 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1572 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1573 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1574 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1575 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1576 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1577 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1578 --selectId visit=903346 ccd=12
1579
1580 to generate the coadd for the HSC-R band if you are interested in following
1581 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1582 """
1583 ConfigClass = SafeClipAssembleCoaddConfig
1584 _DefaultName = "safeClipAssembleCoadd"
1585
1586 def __init__(self, *args, **kwargs):
1587 AssembleCoaddTask.__init__(self, *args, **kwargs)
1588 schema = afwTable.SourceTable.makeMinimalSchema()
1589 self.makeSubtask("clipDetection", schema=schema)
1590
1591 @utils.inheritDoc(AssembleCoaddTask)
1592 @timeMethod
1593 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1594 """Assemble the coadd for a region.
1595
1596 Compute the difference of coadds created with and without outlier
1597 rejection to identify coadd pixels that have outlier values in some
1598 individual visits.
1599 Detect clipped regions on the difference image and mark these regions
1600 on the one or two individual coaddTempExps where they occur if there
1601 is significant overlap between the clipped region and a source. This
1602 leaves us with a set of footprints from the difference image that have
1603 been identified as having occured on just one or two individual visits.
1604 However, these footprints were generated from a difference image. It
1605 is conceivable for a large diffuse source to have become broken up
1606 into multiple footprints acrosss the coadd difference in this process.
1607 Determine the clipped region from all overlapping footprints from the
1608 detected sources in each visit - these are big footprints.
1609 Combine the small and big clipped footprints and mark them on a new
1610 bad mask plane.
1611 Generate the coadd using `AssembleCoaddTask.run` without outlier
1612 removal. Clipped footprints will no longer make it into the coadd
1613 because they are marked in the new bad mask plane.
1614
1615 Notes
1616 -----
1617 args and kwargs are passed but ignored in order to match the call
1618 signature expected by the parent task.
1619 """
1620 exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1621 mask = exp.getMaskedImage().getMask()
1622 mask.addMaskPlane("CLIPPED")
1623
1624 result = self.detectClipdetectClip(exp, tempExpRefList)
1625
1626 self.log.info('Found %d clipped objects', len(result.clipFootprints))
1627
1628 maskClipValue = mask.getPlaneBitMask("CLIPPED")
1629 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1630 # Append big footprints from individual Warps to result.clipSpans
1631 bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1632 result.detectionFootprints, maskClipValue, maskDetValue,
1633 exp.getBBox())
1634 # Create mask of the current clipped footprints
1635 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1636 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1637
1638 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1639 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1640 maskClip |= maskClipBig
1641
1642 # Assemble coadd from base class, but ignoring CLIPPED pixels
1643 badMaskPlanes = self.config.badMaskPlanes[:]
1644 badMaskPlanes.append("CLIPPED")
1645 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1646 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1647 result.clipSpans, mask=badPixelMask)
1648
1649 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1650 """Return an exposure that contains the difference between unclipped
1651 and clipped coadds.
1652
1653 Generate a difference image between clipped and unclipped coadds.
1654 Compute the difference image by subtracting an outlier-clipped coadd
1655 from an outlier-unclipped coadd. Return the difference image.
1656
1657 Parameters
1658 ----------
1659 skyInfo : `lsst.pipe.base.Struct`
1660 Patch geometry information, from getSkyInfo
1661 tempExpRefList : `list`
1662 List of data reference to tempExp
1663 imageScalerList : `list`
1664 List of image scalers
1665 weightList : `list`
1666 List of weights
1667
1668 Returns
1669 -------
1671 Difference image of unclipped and clipped coadd wrapped in an Exposure
1672 """
1673 config = AssembleCoaddConfig()
1674 # getattr necessary because subtasks do not survive Config.toDict()
1675 # exclude connections because the class of self.config.connections is not
1676 # the same as AssembleCoaddConfig.connections, and the connections are not
1677 # needed to run this task anyway.
1678 configIntersection = {k: getattr(self.config, k)
1679 for k, v in self.config.toDict().items()
1680 if (k in config.keys() and k != "connections")}
1681 configIntersection['doInputMap'] = False
1682 configIntersection['doNImage'] = False
1683 config.update(**configIntersection)
1684
1685 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1686 config.statistic = 'MEAN'
1687 task = AssembleCoaddTask(config=config)
1688 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1689
1690 config.statistic = 'MEANCLIP'
1691 task = AssembleCoaddTask(config=config)
1692 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1693
1694 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1695 coaddDiff -= coaddClip.getMaskedImage()
1696 exp = afwImage.ExposureF(coaddDiff)
1697 exp.setPsf(coaddMean.getPsf())
1698 return exp
1699
1700 def detectClip(self, exp, tempExpRefList):
1701 """Detect clipped regions on an exposure and set the mask on the
1702 individual tempExp masks.
1703
1704 Detect footprints in the difference image after smoothing the
1705 difference image with a Gaussian kernal. Identify footprints that
1706 overlap with one or two input ``coaddTempExps`` by comparing the
1707 computed overlap fraction to thresholds set in the config. A different
1708 threshold is applied depending on the number of overlapping visits
1709 (restricted to one or two). If the overlap exceeds the thresholds,
1710 the footprint is considered "CLIPPED" and is marked as such on the
1711 coaddTempExp. Return a struct with the clipped footprints, the indices
1712 of the ``coaddTempExps`` that end up overlapping with the clipped
1713 footprints, and a list of new masks for the ``coaddTempExps``.
1714
1715 Parameters
1716 ----------
1718 Exposure to run detection on.
1719 tempExpRefList : `list`
1720 List of data reference to tempExp.
1721
1722 Returns
1723 -------
1724 result : `lsst.pipe.base.Struct`
1725 Result struct with components:
1726
1727 - ``clipFootprints``: list of clipped footprints.
1728 - ``clipIndices``: indices for each ``clippedFootprint`` in
1729 ``tempExpRefList``.
1730 - ``clipSpans``: List of dictionaries containing spanSet lists
1731 to clip. Each element contains the new maskplane name
1732 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1733 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1734 compressed into footprints.
1735 """
1736 mask = exp.getMaskedImage().getMask()
1737 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1738 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1739 # Merge positive and negative together footprints together
1740 fpSet.positive.merge(fpSet.negative)
1741 footprints = fpSet.positive
1742 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1743 ignoreMask = self.getBadPixelMask()
1744
1745 clipFootprints = []
1746 clipIndices = []
1747 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1748
1749 # for use by detectClipBig
1750 visitDetectionFootprints = []
1751
1752 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1753 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1754 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1755
1756 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1757 for i, warpRef in enumerate(tempExpRefList):
1758 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1759 immediate=True).getMaskedImage().getMask()
1760 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1761 afwImage.PARENT, True)
1762 maskVisitDet &= maskDetValue
1763 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1764 visitDetectionFootprints.append(visitFootprints)
1765
1766 for j, footprint in enumerate(footprints.getFootprints()):
1767 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1768 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1769
1770 # build a list of clipped spans for each visit
1771 for j, footprint in enumerate(footprints.getFootprints()):
1772 nPixel = footprint.getArea()
1773 overlap = [] # hold the overlap with each visit
1774 indexList = [] # index of visit in global list
1775 for i in range(len(tempExpRefList)):
1776 ignore = ignoreArr[i, j]
1777 overlapDet = overlapDetArr[i, j]
1778 totPixel = nPixel - ignore
1779
1780 # If we have more bad pixels than detection skip
1781 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1782 continue
1783 overlap.append(overlapDet/float(totPixel))
1784 indexList.append(i)
1785
1786 overlap = numpy.array(overlap)
1787 if not len(overlap):
1788 continue
1789
1790 keep = False # Should this footprint be marked as clipped?
1791 keepIndex = [] # Which tempExps does the clipped footprint belong to
1792
1793 # If footprint only has one overlap use a lower threshold
1794 if len(overlap) == 1:
1795 if overlap[0] > self.config.minClipFootOverlapSingle:
1796 keep = True
1797 keepIndex = [0]
1798 else:
1799 # This is the general case where only visit should be clipped
1800 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1801 if len(clipIndex) == 1:
1802 keep = True
1803 keepIndex = [clipIndex[0]]
1804
1805 # Test if there are clipped objects that overlap two different visits
1806 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1807 if len(clipIndex) == 2 and len(overlap) > 3:
1808 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1809 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1810 keep = True
1811 keepIndex = clipIndex
1812
1813 if not keep:
1814 continue
1815
1816 for index in keepIndex:
1817 globalIndex = indexList[index]
1818 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1819
1820 clipIndices.append(numpy.array(indexList)[keepIndex])
1821 clipFootprints.append(footprint)
1822
1823 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1824 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1825
1826 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1827 maskClipValue, maskDetValue, coaddBBox):
1828 """Return individual warp footprints for large artifacts and append
1829 them to ``clipList`` in place.
1830
1831 Identify big footprints composed of many sources in the coadd
1832 difference that may have originated in a large diffuse source in the
1833 coadd. We do this by indentifying all clipped footprints that overlap
1834 significantly with each source in all the coaddTempExps.
1835
1836 Parameters
1837 ----------
1838 clipList : `list`
1839 List of alt mask SpanSets with clipping information. Modified.
1840 clipFootprints : `list`
1841 List of clipped footprints.
1842 clipIndices : `list`
1843 List of which entries in tempExpClipList each footprint belongs to.
1844 maskClipValue
1845 Mask value of clipped pixels.
1846 maskDetValue
1847 Mask value of detected pixels.
1848 coaddBBox : `lsst.geom.Box`
1849 BBox of the coadd and warps.
1850
1851 Returns
1852 -------
1853 bigFootprintsCoadd : `list`
1854 List of big footprints
1855 """
1856 bigFootprintsCoadd = []
1857 ignoreMask = self.getBadPixelMask()
1858 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1859 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1860 for footprint in visitFootprints.getFootprints():
1861 footprint.spans.setMask(maskVisitDet, maskDetValue)
1862
1863 # build a mask of clipped footprints that are in this visit
1864 clippedFootprintsVisit = []
1865 for foot, clipIndex in zip(clipFootprints, clipIndices):
1866 if index not in clipIndex:
1867 continue
1868 clippedFootprintsVisit.append(foot)
1869 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1870 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1871
1872 bigFootprintsVisit = []
1873 for foot in visitFootprints.getFootprints():
1874 if foot.getArea() < self.config.minBigOverlap:
1875 continue
1876 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1877 if nCount > self.config.minBigOverlap:
1878 bigFootprintsVisit.append(foot)
1879 bigFootprintsCoadd.append(foot)
1880
1881 for footprint in bigFootprintsVisit:
1882 clippedSpans["CLIPPED"].append(footprint.spans)
1883
1884 return bigFootprintsCoadd
1885
1886
1888 psfMatchedWarps = pipeBase.connectionTypes.Input(
1889 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1890 "Only PSF-Matched Warps make sense for image subtraction. "
1891 "Therefore, they must be an additional declared input."),
1892 name="{inputCoaddName}Coadd_psfMatchedWarp",
1893 storageClass="ExposureF",
1894 dimensions=("tract", "patch", "skymap", "visit"),
1895 deferLoad=True,
1896 multiple=True
1897 )
1898 templateCoadd = pipeBase.connectionTypes.Output(
1899 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1900 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1901 name="{outputCoaddName}CoaddPsfMatched",
1902 storageClass="ExposureF",
1903 dimensions=("tract", "patch", "skymap", "band"),
1904 )
1905
1906 def __init__(self, *, config=None):
1907 super().__init__(config=config)
1908 if not config.assembleStaticSkyModel.doWrite:
1909 self.outputs.remove("templateCoadd")
1910 config.validate()
1911
1912
1913class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1914 pipelineConnections=CompareWarpAssembleCoaddConnections):
1915 assembleStaticSkyModel = pexConfig.ConfigurableField(
1916 target=AssembleCoaddTask,
1917 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1918 " naive/first-iteration model of the static sky.",
1919 )
1920 detect = pexConfig.ConfigurableField(
1921 target=SourceDetectionTask,
1922 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1923 )
1924 detectTemplate = pexConfig.ConfigurableField(
1925 target=SourceDetectionTask,
1926 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1927 )
1928 maskStreaks = pexConfig.ConfigurableField(
1929 target=MaskStreaksTask,
1930 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1931 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1932 "streakMaskName"
1933 )
1934 streakMaskName = pexConfig.Field(
1935 dtype=str,
1936 default="STREAK",
1937 doc="Name of mask bit used for streaks"
1938 )
1939 maxNumEpochs = pexConfig.Field(
1940 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1941 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1942 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1943 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1944 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1945 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1946 "than transient and not masked.",
1947 dtype=int,
1948 default=2
1949 )
1950 maxFractionEpochsLow = pexConfig.RangeField(
1951 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1952 "Effective maxNumEpochs = "
1953 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1954 dtype=float,
1955 default=0.4,
1956 min=0., max=1.,
1957 )
1958 maxFractionEpochsHigh = pexConfig.RangeField(
1959 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1960 "Effective maxNumEpochs = "
1961 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1962 dtype=float,
1963 default=0.03,
1964 min=0., max=1.,
1965 )
1966 spatialThreshold = pexConfig.RangeField(
1967 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1968 "temporal criteria. If 0, clip all. If 1, clip none.",
1969 dtype=float,
1970 default=0.5,
1971 min=0., max=1.,
1972 inclusiveMin=True, inclusiveMax=True
1973 )
1974 doScaleWarpVariance = pexConfig.Field(
1975 doc="Rescale Warp variance plane using empirical noise?",
1976 dtype=bool,
1977 default=True,
1978 )
1979 scaleWarpVariance = pexConfig.ConfigurableField(
1980 target=ScaleVarianceTask,
1981 doc="Rescale variance on warps",
1982 )
1983 doPreserveContainedBySource = pexConfig.Field(
1984 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1985 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1986 dtype=bool,
1987 default=True,
1988 )
1989 doPrefilterArtifacts = pexConfig.Field(
1990 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1991 "because they will be excluded anyway. This prevents them from contributing "
1992 "to the outlier epoch count image and potentially being labeled as persistant."
1993 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1994 dtype=bool,
1995 default=True
1996 )
1997 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1998 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1999 dtype=str,
2000 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
2001 )
2002 prefilterArtifactsRatio = pexConfig.Field(
2003 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
2004 dtype=float,
2005 default=0.05
2006 )
2007 doFilterMorphological = pexConfig.Field(
2008 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
2009 "be streaks.",
2010 dtype=bool,
2011 default=False
2012 )
2013
2014 def setDefaults(self):
2015 AssembleCoaddConfig.setDefaults(self)
2016 self.statisticstatistic = 'MEAN'
2017 self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
2018
2019 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
2020 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
2021 if "EDGE" in self.badMaskPlanes:
2022 self.badMaskPlanes.remove('EDGE')
2023 self.removeMaskPlanes.append('EDGE')
2024 self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
2025 self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
2026 self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
2027 self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
2028 self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
2029 self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
2030 self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
2031 self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
2032 self.detectdetect.doTempLocalBackground = False
2033 self.detectdetect.reEstimateBackground = False
2034 self.detectdetect.returnOriginalFootprints = False
2035 self.detectdetect.thresholdPolarity = "both"
2036 self.detectdetect.thresholdValue = 5
2037 self.detectdetect.minPixels = 4
2038 self.detectdetect.isotropicGrow = True
2039 self.detectdetect.thresholdType = "pixel_stdev"
2040 self.detectdetect.nSigmaToGrow = 0.4
2041 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
2042 # Explicitly restating because ratio with detect.nSigmaToGrow matters
2043 self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
2044 self.detectTemplatedetectTemplate.doTempLocalBackground = False
2045 self.detectTemplatedetectTemplate.reEstimateBackground = False
2046 self.detectTemplatedetectTemplate.returnOriginalFootprints = False
2047
2048 def validate(self):
2049 super().validate()
2050 if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
2051 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
2052 "Please set assembleStaticSkyModel.doNImage=False")
2053
2054 if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
2055 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
2056 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
2057 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
2058 "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
2059
2060
2061class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
2062 """Assemble a compareWarp coadded image from a set of warps
2063 by masking artifacts detected by comparing PSF-matched warps.
2064
2065 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
2066 we clip outliers). The problem with doing this is that when computing the
2067 coadd PSF at a given location, individual visit PSFs from visits with
2068 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
2069 In this task, we correct for this behavior by creating a new badMaskPlane
2070 'CLIPPED' which marks pixels in the individual warps suspected to contain
2071 an artifact. We populate this plane on the input warps by comparing
2072 PSF-matched warps with a PSF-matched median coadd which serves as a
2073 model of the static sky. Any group of pixels that deviates from the
2074 PSF-matched template coadd by more than config.detect.threshold sigma,
2075 is an artifact candidate. The candidates are then filtered to remove
2076 variable sources and sources that are difficult to subtract such as
2077 bright stars. This filter is configured using the config parameters
2078 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
2079 the maximum fraction of epochs that the deviation can appear in and still
2080 be considered an artifact. The spatialThreshold is the maximum fraction of
2081 pixels in the footprint of the deviation that appear in other epochs
2082 (where other epochs is defined by the temporalThreshold). If the deviant
2083 region meets this criteria of having a significant percentage of pixels
2084 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
2085 set in the mask. These regions will not contribute to the final coadd.
2086 Furthermore, any routine to determine the coadd PSF can now be cognizant
2087 of clipped regions. Note that the algorithm implemented by this task is
2088 preliminary and works correctly for HSC data. Parameter modifications and
2089 or considerable redesigning of the algorithm is likley required for other
2090 surveys.
2091
2092 ``CompareWarpAssembleCoaddTask`` sub-classes
2093 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
2094 as a subtask to generate the TemplateCoadd (the model of the static sky).
2095
2096 Notes
2097 -----
2098 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2099 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2100 ``baseDebug`` for more about ``debug.py`` files.
2101
2102 This task supports the following debug variables:
2103
2104 - ``saveCountIm``
2105 If True then save the Epoch Count Image as a fits file in the `figPath`
2106 - ``figPath``
2107 Path to save the debug fits images and figures
2108
2109 For example, put something like:
2110
2111 .. code-block:: python
2112
2113 import lsstDebug
2114 def DebugInfo(name):
2115 di = lsstDebug.getInfo(name)
2116 if name == "lsst.pipe.tasks.assembleCoadd":
2117 di.saveCountIm = True
2118 di.figPath = "/desired/path/to/debugging/output/images"
2119 return di
2120 lsstDebug.Info = DebugInfo
2121
2122 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2123 ``--debug`` flag. Some subtasks may have their own debug variables;
2124 see individual Task documentation.
2125
2126 Examples
2127 --------
2128 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2129 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2130 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2131 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2132 and filter to be coadded (specified using
2133 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2134 along with a list of coaddTempExps to attempt to coadd (specified using
2135 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2136 Only the warps that cover the specified tract and patch will be coadded.
2137 A list of the available optional arguments can be obtained by calling
2138 ``assembleCoadd.py`` with the ``--help`` command line argument:
2139
2140 .. code-block:: none
2141
2142 assembleCoadd.py --help
2143
2144 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2145 context of multi-band processing, we will generate the HSC-I & -R band
2146 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2147 To begin, assuming that the lsst stack has been already set up, we must
2148 set up the ``obs_subaru`` and ``ci_hsc`` packages.
2149 This defines the environment variable ``$CI_HSC_DIR`` and points at the
2150 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2151 directory. To begin assembling the coadds, we must first
2152
2153 - processCcd
2154 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2155 - makeSkyMap
2156 create a skymap that covers the area of the sky present in the raw exposures
2157 - makeCoaddTempExp
2158 warp the individual calibrated exposures to the tangent plane of the coadd
2159
2160 We can perform all of these steps by running
2161
2162 .. code-block:: none
2163
2164 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2165
2166 This will produce warped ``coaddTempExps`` for each visit. To coadd the
2167 warped data, we call ``assembleCoadd.py`` as follows:
2168
2169 .. code-block:: none
2170
2171 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2172 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2173 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2174 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2175 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2176 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2177 --selectId visit=903988 ccd=24
2178
2179 This will process the HSC-I band data. The results are written in
2180 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2181 """
2182 ConfigClass = CompareWarpAssembleCoaddConfig
2183 _DefaultName = "compareWarpAssembleCoadd"
2184
2185 def __init__(self, *args, **kwargs):
2186 AssembleCoaddTask.__init__(self, *args, **kwargs)
2187 self.makeSubtask("assembleStaticSkyModel")
2188 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2189 self.makeSubtask("detect", schema=detectionSchema)
2190 if self.config.doPreserveContainedBySource:
2191 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2192 if self.config.doScaleWarpVariance:
2193 self.makeSubtask("scaleWarpVariance")
2194 if self.config.doFilterMorphological:
2195 self.makeSubtask("maskStreaks")
2196
2197 @utils.inheritDoc(AssembleCoaddTask)
2198 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2199 """
2200 Generate a templateCoadd to use as a naive model of static sky to
2201 subtract from PSF-Matched warps.
2202
2203 Returns
2204 -------
2205 result : `lsst.pipe.base.Struct`
2206 Result struct with components:
2207
2208 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2209 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2210 """
2211 # Ensure that psfMatchedWarps are used as input warps for template generation
2212 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2213 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2214
2215 # Because subtasks don't have connections we have to make one.
2216 # The main task's `templateCoadd` is the subtask's `coaddExposure`
2217 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2218 if self.config.assembleStaticSkyModel.doWrite:
2219 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2220 # Remove template coadd from both subtask's and main tasks outputs,
2221 # because it is handled by the subtask as `coaddExposure`
2222 del outputRefs.templateCoadd
2223 del staticSkyModelOutputRefs.templateCoadd
2224
2225 # A PSF-Matched nImage does not exist as a dataset type
2226 if 'nImage' in staticSkyModelOutputRefs.keys():
2227 del staticSkyModelOutputRefs.nImage
2228
2229 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2230 staticSkyModelOutputRefs)
2231 if templateCoadd is None:
2232 raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2233
2234 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2235 nImage=templateCoadd.nImage,
2236 warpRefList=templateCoadd.warpRefList,
2237 imageScalerList=templateCoadd.imageScalerList,
2238 weightList=templateCoadd.weightList)
2239
2240 @utils.inheritDoc(AssembleCoaddTask)
2241 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2242 """
2243 Generate a templateCoadd to use as a naive model of static sky to
2244 subtract from PSF-Matched warps.
2245
2246 Returns
2247 -------
2248 result : `lsst.pipe.base.Struct`
2249 Result struct with components:
2250
2251 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2252 - ``nImage``: N Image (``lsst.afw.image.Image``)
2253 """
2254 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2255 if templateCoadd is None:
2256 raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2257
2258 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2259 nImage=templateCoadd.nImage,
2260 warpRefList=templateCoadd.warpRefList,
2261 imageScalerList=templateCoadd.imageScalerList,
2262 weightList=templateCoadd.weightList)
2263
2264 def _noTemplateMessage(self, warpType):
2265 warpName = (warpType[0].upper() + warpType[1:])
2266 message = """No %(warpName)s warps were found to build the template coadd which is
2267 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2268 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2269 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2270
2271 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2272 another algorithm like:
2273
2274 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2275 config.assemble.retarget(SafeClipAssembleCoaddTask)
2276 """ % {"warpName": warpName}
2277 return message
2278
2279 @utils.inheritDoc(AssembleCoaddTask)
2280 @timeMethod
2281 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2282 supplementaryData, *args, **kwargs):
2283 """Assemble the coadd.
2284
2285 Find artifacts and apply them to the warps' masks creating a list of
2286 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2287 plane. Then pass these alternative masks to the base class's `run`
2288 method.
2289
2290 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2291 that must contain a ``templateCoadd`` that serves as the
2292 model of the static sky.
2293 """
2294
2295 # Check and match the order of the supplementaryData
2296 # (PSF-matched) inputs to the order of the direct inputs,
2297 # so that the artifact mask is applied to the right warp
2298 dataIds = [ref.dataId for ref in tempExpRefList]
2299 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2300
2301 if dataIds != psfMatchedDataIds:
2302 self.log.info("Reordering and or/padding PSF-matched visit input list")
2303 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2304 psfMatchedDataIds, dataIds)
2305 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2306 psfMatchedDataIds, dataIds)
2307
2308 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2309 spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2310 supplementaryData.warpRefList,
2311 supplementaryData.imageScalerList)
2312
2313 badMaskPlanes = self.config.badMaskPlanes[:]
2314 badMaskPlanes.append("CLIPPED")
2315 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2316
2317 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2318 spanSetMaskList, mask=badPixelMask)
2319
2320 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2321 # Psf-Matching moves the real edge inwards
2322 self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2323 return result
2324
2325 def applyAltEdgeMask(self, mask, altMaskList):
2326 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2327
2328 Parameters
2329 ----------
2330 mask : `lsst.afw.image.Mask`
2331 Original mask.
2332 altMaskList : `list`
2333 List of Dicts containing ``spanSet`` lists.
2334 Each element contains the new mask plane name (e.g. "CLIPPED
2335 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2336 the mask.
2337 """
2338 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2339 for visitMask in altMaskList:
2340 if "EDGE" in visitMask:
2341 for spanSet in visitMask['EDGE']:
2342 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2343
2344 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2345 """Find artifacts.
2346
2347 Loop through warps twice. The first loop builds a map with the count
2348 of how many epochs each pixel deviates from the templateCoadd by more
2349 than ``config.chiThreshold`` sigma. The second loop takes each
2350 difference image and filters the artifacts detected in each using
2351 count map to filter out variable sources and sources that are
2352 difficult to subtract cleanly.
2353
2354 Parameters
2355 ----------
2356 templateCoadd : `lsst.afw.image.Exposure`
2357 Exposure to serve as model of static sky.
2358 tempExpRefList : `list`
2359 List of data references to warps.
2360 imageScalerList : `list`
2361 List of image scalers.
2362
2363 Returns
2364 -------
2365 altMasks : `list`
2366 List of dicts containing information about CLIPPED
2367 (i.e., artifacts), NO_DATA, and EDGE pixels.
2368 """
2369
2370 self.log.debug("Generating Count Image, and mask lists.")
2371 coaddBBox = templateCoadd.getBBox()
2372 slateIm = afwImage.ImageU(coaddBBox)
2373 epochCountImage = afwImage.ImageU(coaddBBox)
2374 nImage = afwImage.ImageU(coaddBBox)
2375 spanSetArtifactList = []
2376 spanSetNoDataMaskList = []
2377 spanSetEdgeList = []
2378 spanSetBadMorphoList = []
2379 badPixelMask = self.getBadPixelMask()
2380
2381 # mask of the warp diffs should = that of only the warp
2382 templateCoadd.mask.clearAllMaskPlanes()
2383
2384 if self.config.doPreserveContainedBySource:
2385 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2386 else:
2387 templateFootprints = None
2388
2389 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2390 warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2391 if warpDiffExp is not None:
2392 # This nImage only approximates the final nImage because it uses the PSF-matched mask
2393 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2394 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2395 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2396 fpSet.positive.merge(fpSet.negative)
2397 footprints = fpSet.positive
2398 slateIm.set(0)
2399 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2400
2401 # Remove artifacts due to defects before they contribute to the epochCountImage
2402 if self.config.doPrefilterArtifacts:
2403 spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2404
2405 # Clear mask before adding prefiltered spanSets
2406 self.detect.clearMask(warpDiffExp.mask)
2407 for spans in spanSetList:
2408 spans.setImage(slateIm, 1, doClip=True)
2409 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2410 epochCountImage += slateIm
2411
2412 if self.config.doFilterMorphological:
2413 maskName = self.config.streakMaskName
2414 _ = self.maskStreaks.run(warpDiffExp)
2415 streakMask = warpDiffExp.mask
2416 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2417 streakMask.getPlaneBitMask(maskName)).split()
2418
2419 # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2420 # undergo a second convolution. Pixels with data in the direct warp
2421 # but not in the PSF-matched warp will not have their artifacts detected.
2422 # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2423 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2424 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2425 nansMask.setXY0(warpDiffExp.getXY0())
2426 edgeMask = warpDiffExp.mask
2427 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2428 edgeMask.getPlaneBitMask("EDGE")).split()
2429 else:
2430 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2431 # In this case, mask the whole epoch
2432 nansMask = afwImage.MaskX(coaddBBox, 1)
2433 spanSetList = []
2434 spanSetEdgeMask = []
2435 spanSetStreak = []
2436
2437 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2438
2439 spanSetNoDataMaskList.append(spanSetNoDataMask)
2440 spanSetArtifactList.append(spanSetList)
2441 spanSetEdgeList.append(spanSetEdgeMask)
2442 if self.config.doFilterMorphological:
2443 spanSetBadMorphoList.append(spanSetStreak)
2444
2445 if lsstDebug.Info(__name__).saveCountIm:
2446 path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2447 epochCountImage.writeFits(path)
2448
2449 for i, spanSetList in enumerate(spanSetArtifactList):
2450 if spanSetList:
2451 filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2452 templateFootprints)
2453 spanSetArtifactList[i] = filteredSpanSetList
2454 if self.config.doFilterMorphological:
2455 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2456
2457 altMasks = []
2458 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2459 altMasks.append({'CLIPPED': artifacts,
2460 'NO_DATA': noData,
2461 'EDGE': edge})
2462 return altMasks
2463
2464 def prefilterArtifacts(self, spanSetList, exp):
2465 """Remove artifact candidates covered by bad mask plane.
2466
2467 Any future editing of the candidate list that does not depend on
2468 temporal information should go in this method.
2469
2470 Parameters
2471 ----------
2472 spanSetList : `list`
2473 List of SpanSets representing artifact candidates.
2475 Exposure containing mask planes used to prefilter.
2476
2477 Returns
2478 -------
2479 returnSpanSetList : `list`
2480 List of SpanSets with artifacts.
2481 """
2482 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2483 goodArr = (exp.mask.array & badPixelMask) == 0
2484 returnSpanSetList = []
2485 bbox = exp.getBBox()
2486 x0, y0 = exp.getXY0()
2487 for i, span in enumerate(spanSetList):
2488 y, x = span.clippedTo(bbox).indices()
2489 yIndexLocal = numpy.array(y) - y0
2490 xIndexLocal = numpy.array(x) - x0
2491 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2492 if goodRatio > self.config.prefilterArtifactsRatio:
2493 returnSpanSetList.append(span)
2494 return returnSpanSetList
2495
2496 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2497 """Filter artifact candidates.
2498
2499 Parameters
2500 ----------
2501 spanSetList : `list`
2502 List of SpanSets representing artifact candidates.
2503 epochCountImage : `lsst.afw.image.Image`
2504 Image of accumulated number of warpDiff detections.
2505 nImage : `lsst.afw.image.Image`
2506 Image of the accumulated number of total epochs contributing.
2507
2508 Returns
2509 -------
2510 maskSpanSetList : `list`
2511 List of SpanSets with artifacts.
2512 """
2513
2514 maskSpanSetList = []
2515 x0, y0 = epochCountImage.getXY0()
2516 for i, span in enumerate(spanSetList):
2517 y, x = span.indices()
2518 yIdxLocal = [y1 - y0 for y1 in y]
2519 xIdxLocal = [x1 - x0 for x1 in x]
2520 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2521 totalN = nImage.array[yIdxLocal, xIdxLocal]
2522
2523 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2524 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2525 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2526 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2527 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2528 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2529 & (outlierN <= effectiveMaxNumEpochs))
2530 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2531 if percentBelowThreshold > self.config.spatialThreshold:
2532 maskSpanSetList.append(span)
2533
2534 if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2535 # If a candidate is contained by a footprint on the template coadd, do not clip
2536 filteredMaskSpanSetList = []
2537 for span in maskSpanSetList:
2538 doKeep = True
2539 for footprint in footprintsToExclude.positive.getFootprints():
2540 if footprint.spans.contains(span):
2541 doKeep = False
2542 break
2543 if doKeep:
2544 filteredMaskSpanSetList.append(span)
2545 maskSpanSetList = filteredMaskSpanSetList
2546
2547 return maskSpanSetList
2548
2549 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2550 """Fetch a warp from the butler and return a warpDiff.
2551
2552 Parameters
2553 ----------
2555 Butler dataRef for the warp.
2557 An image scaler object.
2558 templateCoadd : `lsst.afw.image.Exposure`
2559 Exposure to be substracted from the scaled warp.
2560
2561 Returns
2562 -------
2564 Exposure of the image difference between the warp and template.
2565 """
2566
2567 # If the PSF-Matched warp did not exist for this direct warp
2568 # None is holding its place to maintain order in Gen 3
2569 if warpRef is None:
2570 return None
2571 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2572 warpName = self.getTempExpDatasetName('psfMatched')
2573 if not isinstance(warpRef, DeferredDatasetHandle):
2574 if not warpRef.datasetExists(warpName):
2575 self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2576 return None
2577 warp = warpRef.get(datasetType=warpName, immediate=True)
2578 # direct image scaler OK for PSF-matched Warp
2579 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2580 mi = warp.getMaskedImage()
2581 if self.config.doScaleWarpVariance:
2582 try:
2583 self.scaleWarpVariance.run(mi)
2584 except Exception as exc:
2585 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc)
2586 mi -= templateCoadd.getMaskedImage()
2587 return warp
2588
2589 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2590 """Return a path to which to write debugging output.
2591
2592 Creates a hyphen-delimited string of dataId values for simple filenames.
2593
2594 Parameters
2595 ----------
2596 prefix : `str`
2597 Prefix for filename.
2599 Butler dataRef to make the path from.
2600 coaddLevel : `bool`, optional.
2601 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2602 'filter', but no 'visit').
2603
2604 Returns
2605 -------
2606 result : `str`
2607 Path for debugging output.
2608 """
2609 if coaddLevel:
2610 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2611 else:
2612 keys = warpRef.dataId.keys()
2613 keyList = sorted(keys, reverse=True)
2614 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2615 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2616 return os.path.join(directory, filename)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:141
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:346
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60