lsst.pipe.tasks  21.0.0-129-gdb38469a+1bd5e57dbb
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import logging
27 import lsst.pex.config as pexConfig
28 import lsst.pex.exceptions as pexExceptions
29 import lsst.geom as geom
30 import lsst.afw.geom as afwGeom
31 import lsst.afw.image as afwImage
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 import lsst.afw.detection as afwDet
35 import lsst.coadd.utils as coaddUtils
36 import lsst.pipe.base as pipeBase
37 import lsst.meas.algorithms as measAlg
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from .healSparseMapping import HealSparseInputMapTask
48 from lsst.meas.algorithms import SourceDetectionTask
49 from lsst.daf.butler import DeferredDatasetHandle
50 
51 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
52  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
53  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
54 
55 log = logging.getLogger(__name__.partition(".")[2])
56 
57 
58 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
59  dimensions=("tract", "patch", "band", "skymap"),
60  defaultTemplates={"inputCoaddName": "deep",
61  "outputCoaddName": "deep",
62  "warpType": "direct",
63  "warpTypeSuffix": ""}):
64 
65  inputWarps = pipeBase.connectionTypes.Input(
66  doc=("Input list of warps to be assemebled i.e. stacked."
67  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
68  name="{inputCoaddName}Coadd_{warpType}Warp",
69  storageClass="ExposureF",
70  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
71  deferLoad=True,
72  multiple=True
73  )
74  skyMap = pipeBase.connectionTypes.Input(
75  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
76  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
77  storageClass="SkyMap",
78  dimensions=("skymap", ),
79  )
80  selectedVisits = pipeBase.connectionTypes.Input(
81  doc="Selected visits to be coadded.",
82  name="{outputCoaddName}Visits",
83  storageClass="StructuredDataDict",
84  dimensions=("instrument", "tract", "patch", "skymap", "band")
85  )
86  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
87  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
88  " BRIGHT_OBJECT."),
89  name="brightObjectMask",
90  storageClass="ObjectMaskCatalog",
91  dimensions=("tract", "patch", "skymap", "band"),
92  )
93  coaddExposure = pipeBase.connectionTypes.Output(
94  doc="Output coadded exposure, produced by stacking input warps",
95  name="{outputCoaddName}Coadd{warpTypeSuffix}",
96  storageClass="ExposureF",
97  dimensions=("tract", "patch", "skymap", "band"),
98  )
99  nImage = pipeBase.connectionTypes.Output(
100  doc="Output image of number of input images per pixel",
101  name="{outputCoaddName}Coadd_nImage",
102  storageClass="ImageU",
103  dimensions=("tract", "patch", "skymap", "band"),
104  )
105  inputMap = pipeBase.connectionTypes.Output(
106  doc="Output healsparse map of input images",
107  name="{outputCoaddName}Coadd_inputMap",
108  storageClass="HealSparseMap",
109  dimensions=("tract", "patch", "skymap", "band"),
110  )
111 
112  def __init__(self, *, config=None):
113  super().__init__(config=config)
114 
115  # Override the connection's name template with config to replicate Gen2 behavior
116  # This duplicates some of the logic in the base class, due to wanting Gen2 and
117  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
118  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
119  templateValues['warpType'] = config.warpType
120  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
121  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
122  for name in self.allConnections}
123  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
124  # End code to remove after deprecation
125 
126  if not config.doMaskBrightObjects:
127  self.prerequisiteInputs.remove("brightObjectMask")
128 
129  if not config.doSelectVisits:
130  self.inputs.remove("selectedVisits")
131 
132  if not config.doNImage:
133  self.outputs.remove("nImage")
134 
135  if not self.config.doInputMap:
136  self.outputs.remove("inputMap")
137 
138 
139 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
140  pipelineConnections=AssembleCoaddConnections):
141  """Configuration parameters for the `AssembleCoaddTask`.
142 
143  Notes
144  -----
145  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
146  only set the bitplane config.brightObjectMaskName. To make this useful you
147  *must* also configure the flags.pixel algorithm, for example by adding
148 
149  .. code-block:: none
150 
151  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
152  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
153 
154  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
155  """
156  warpType = pexConfig.Field(
157  doc="Warp name: one of 'direct' or 'psfMatched'",
158  dtype=str,
159  default="direct",
160  )
161  subregionSize = pexConfig.ListField(
162  dtype=int,
163  doc="Width, height of stack subregion size; "
164  "make small enough that a full stack of images will fit into memory at once.",
165  length=2,
166  default=(2000, 2000),
167  )
168  statistic = pexConfig.Field(
169  dtype=str,
170  doc="Main stacking statistic for aggregating over the epochs.",
171  default="MEANCLIP",
172  )
173  doSigmaClip = pexConfig.Field(
174  dtype=bool,
175  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
176  default=False,
177  )
178  sigmaClip = pexConfig.Field(
179  dtype=float,
180  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
181  default=3.0,
182  )
183  clipIter = pexConfig.Field(
184  dtype=int,
185  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
186  default=2,
187  )
188  calcErrorFromInputVariance = pexConfig.Field(
189  dtype=bool,
190  doc="Calculate coadd variance from input variance by stacking statistic."
191  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
192  default=True,
193  )
194  scaleZeroPoint = pexConfig.ConfigurableField(
195  target=ScaleZeroPointTask,
196  doc="Task to adjust the photometric zero point of the coadd temp exposures",
197  )
198  doInterp = pexConfig.Field(
199  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
200  dtype=bool,
201  default=True,
202  )
203  interpImage = pexConfig.ConfigurableField(
204  target=InterpImageTask,
205  doc="Task to interpolate (and extrapolate) over NaN pixels",
206  )
207  doWrite = pexConfig.Field(
208  doc="Persist coadd?",
209  dtype=bool,
210  default=True,
211  )
212  doNImage = pexConfig.Field(
213  doc="Create image of number of contributing exposures for each pixel",
214  dtype=bool,
215  default=False,
216  )
217  doUsePsfMatchedPolygons = pexConfig.Field(
218  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
219  dtype=bool,
220  default=False,
221  )
222  maskPropagationThresholds = pexConfig.DictField(
223  keytype=str,
224  itemtype=float,
225  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
226  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
227  "would have contributed exceeds this value."),
228  default={"SAT": 0.1},
229  )
230  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
231  doc="Mask planes to remove before coadding")
232  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
233  doc="Set mask and flag bits for bright objects?")
234  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
235  doc="Name of mask bit used for bright objects")
236  coaddPsf = pexConfig.ConfigField(
237  doc="Configuration for CoaddPsf",
238  dtype=measAlg.CoaddPsfConfig,
239  )
240  doAttachTransmissionCurve = pexConfig.Field(
241  dtype=bool, default=False, optional=False,
242  doc=("Attach a piecewise TransmissionCurve for the coadd? "
243  "(requires all input Exposures to have TransmissionCurves).")
244  )
245  hasFakes = pexConfig.Field(
246  dtype=bool,
247  default=False,
248  doc="Should be set to True if fake sources have been inserted into the input data."
249  )
250  doSelectVisits = pexConfig.Field(
251  doc="Coadd only visits selected by a SelectVisitsTask",
252  dtype=bool,
253  default=False,
254  )
255  doInputMap = pexConfig.Field(
256  doc="Create a bitwise map of coadd inputs",
257  dtype=bool,
258  default=False,
259  )
260  inputMapper = pexConfig.ConfigurableField(
261  doc="Input map creation subtask.",
262  target=HealSparseInputMapTask,
263  )
264 
265  def setDefaults(self):
266  super().setDefaults()
267  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
268 
269  def validate(self):
270  super().validate()
271  if self.doPsfMatch:
272  # Backwards compatibility.
273  # Configs do not have loggers
274  log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
275  self.warpType = 'psfMatched'
276  if self.doSigmaClip and self.statistic != "MEANCLIP":
277  log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
278  self.statistic = "MEANCLIP"
279  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
280  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
281  "compute and set a non-zero coadd variance estimate." % (self.statistic))
282 
283  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
284  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
285  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
286  if str(k) not in unstackableStats]
287  raise ValueError("statistic %s is not allowed. Please choose one of %s."
288  % (self.statistic, stackableStats))
289 
290 
291 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
292  """Assemble a coadded image from a set of warps (coadded temporary exposures).
293 
294  We want to assemble a coadded image from a set of Warps (also called
295  coadded temporary exposures or ``coaddTempExps``).
296  Each input Warp covers a patch on the sky and corresponds to a single
297  run/visit/exposure of the covered patch. We provide the task with a list
298  of Warps (``selectDataList``) from which it selects Warps that cover the
299  specified patch (pointed at by ``dataRef``).
300  Each Warp that goes into a coadd will typically have an independent
301  photometric zero-point. Therefore, we must scale each Warp to set it to
302  a common photometric zeropoint. WarpType may be one of 'direct' or
303  'psfMatched', and the boolean configs `config.makeDirect` and
304  `config.makePsfMatched` set which of the warp types will be coadded.
305  The coadd is computed as a mean with optional outlier rejection.
306  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
307  Finally, Warps can have bad 'NaN' pixels which received no input from the
308  source calExps. We interpolate over these bad (NaN) pixels.
309 
310  `AssembleCoaddTask` uses several sub-tasks. These are
311 
312  - `ScaleZeroPointTask`
313  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
314  - `InterpImageTask`
315  - interpolate across bad pixels (NaN) in the final coadd
316 
317  You can retarget these subtasks if you wish.
318 
319  Notes
320  -----
321  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
322  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
323  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
324  no debug variables of its own. Some of the subtasks may support debug
325  variables. See the documentation for the subtasks for further information.
326 
327  Examples
328  --------
329  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
330  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
331  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
332  inputs: a data reference to the tract patch and filter to be coadded, and
333  a list of Warps to attempt to coadd. These are specified using ``--id`` and
334  ``--selectId``, respectively:
335 
336  .. code-block:: none
337 
338  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
339  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
340 
341  Only the Warps that cover the specified tract and patch will be coadded.
342  A list of the available optional arguments can be obtained by calling
343  ``assembleCoadd.py`` with the ``--help`` command line argument:
344 
345  .. code-block:: none
346 
347  assembleCoadd.py --help
348 
349  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
350  multi-band processing, we will generate the HSC-I & -R band coadds from
351  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
352  assuming that the lsst stack has been already set up, we must set up the
353  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
354  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
355  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
356  coadds, we must first
357 
358  - processCcd
359  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
360  - makeSkyMap
361  - create a skymap that covers the area of the sky present in the raw exposures
362  - makeCoaddTempExp
363  - warp the individual calibrated exposures to the tangent plane of the coadd
364 
365  We can perform all of these steps by running
366 
367  .. code-block:: none
368 
369  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
370 
371  This will produce warped exposures for each visit. To coadd the warped
372  data, we call assembleCoadd.py as follows:
373 
374  .. code-block:: none
375 
376  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
377  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
378  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
379  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
380  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
381  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
382  --selectId visit=903988 ccd=24
383 
384  that will process the HSC-I band data. The results are written in
385  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
386 
387  You may also choose to run:
388 
389  .. code-block:: none
390 
391  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
392  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
393  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
394  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
395  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
396  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
397  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
398  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
399 
400  to generate the coadd for the HSC-R band if you are interested in
401  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
402  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
403  rather than `AssembleCoaddTask` to make the coadd.
404  """
405  ConfigClass = AssembleCoaddConfig
406  _DefaultName = "assembleCoadd"
407 
408  def __init__(self, *args, **kwargs):
409  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
410  if args:
411  argNames = ["config", "name", "parentTask", "log"]
412  kwargs.update({k: v for k, v in zip(argNames, args)})
413  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
414  "PipelineTask will not take positional args" % argNames, FutureWarning)
415 
416  super().__init__(**kwargs)
417  self.makeSubtask("interpImage")
418  self.makeSubtask("scaleZeroPoint")
419 
420  if self.config.doMaskBrightObjects:
421  mask = afwImage.Mask()
422  try:
423  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
424  except pexExceptions.LsstCppException:
425  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
426  mask.getMaskPlaneDict().keys())
427  del mask
428 
429  if self.config.doInputMap:
430  self.makeSubtask("inputMapper")
431 
432  self.warpType = self.config.warpType
433 
434  @utils.inheritDoc(pipeBase.PipelineTask)
435  def runQuantum(self, butlerQC, inputRefs, outputRefs):
436  # Docstring to be formatted with info from PipelineTask.runQuantum
437  """
438  Notes
439  -----
440  Assemble a coadd from a set of Warps.
441 
442  PipelineTask (Gen3) entry point to Coadd a set of Warps.
443  Analogous to `runDataRef`, it prepares all the data products to be
444  passed to `run`, and processes the results before returning a struct
445  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
446  Therefore, its inputs are accessed subregion by subregion
447  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
448  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
449  correspond to an update in `runDataRef` while both entry points
450  are used.
451  """
452  inputData = butlerQC.get(inputRefs)
453 
454  # Construct skyInfo expected by run
455  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
456  skyMap = inputData["skyMap"]
457  outputDataId = butlerQC.quantum.dataId
458 
459  inputData['skyInfo'] = makeSkyInfo(skyMap,
460  tractId=outputDataId['tract'],
461  patchId=outputDataId['patch'])
462 
463  if self.config.doSelectVisits:
464  warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
465  else:
466  warpRefList = inputData['inputWarps']
467 
468  # Perform same middle steps as `runDataRef` does
469  inputs = self.prepareInputs(warpRefList)
470  self.log.info("Found %d %s", len(inputs.tempExpRefList),
471  self.getTempExpDatasetName(self.warpType))
472  if len(inputs.tempExpRefList) == 0:
473  raise pipeBase.NoWorkFound("No coadd temporary exposures found")
474 
475  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
476  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
477  inputs.weightList, supplementaryData=supplementaryData)
478 
479  inputData.setdefault('brightObjectMask', None)
480  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
481 
482  if self.config.doWrite:
483  butlerQC.put(retStruct, outputRefs)
484  return retStruct
485 
486  @pipeBase.timeMethod
487  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
488  """Assemble a coadd from a set of Warps.
489 
490  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
491  Compute weights to be applied to each Warp and
492  find scalings to match the photometric zeropoint to a reference Warp.
493  Assemble the Warps using `run`. Interpolate over NaNs and
494  optionally write the coadd to disk. Return the coadded exposure.
495 
496  Parameters
497  ----------
498  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
499  Data reference defining the patch for coaddition and the
500  reference Warp (if ``config.autoReference=False``).
501  Used to access the following data products:
502  - ``self.config.coaddName + "Coadd_skyMap"``
503  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
504  - ``self.config.coaddName + "Coadd"``
505  selectDataList : `list`
506  List of data references to Calexps. Data to be coadded will be
507  selected from this list based on overlap with the patch defined
508  by dataRef, grouped by visit, and converted to a list of data
509  references to warps.
510  warpRefList : `list`
511  List of data references to Warps to be coadded.
512  Note: `warpRefList` is just the new name for `tempExpRefList`.
513 
514  Returns
515  -------
516  retStruct : `lsst.pipe.base.Struct`
517  Result struct with components:
518 
519  - ``coaddExposure``: coadded exposure (``Exposure``).
520  - ``nImage``: exposure count image (``Image``).
521  """
522  if selectDataList and warpRefList:
523  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
524  "and which to use is ambiguous. Please pass only one.")
525 
526  skyInfo = self.getSkyInfo(dataRef)
527  if warpRefList is None:
528  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
529  if len(calExpRefList) == 0:
530  self.log.warning("No exposures to coadd")
531  return
532  self.log.info("Coadding %d exposures", len(calExpRefList))
533 
534  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
535 
536  inputData = self.prepareInputs(warpRefList)
537  self.log.info("Found %d %s", len(inputData.tempExpRefList),
538  self.getTempExpDatasetName(self.warpType))
539  if len(inputData.tempExpRefList) == 0:
540  self.log.warning("No coadd temporary exposures found")
541  return
542 
543  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
544 
545  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
546  inputData.weightList, supplementaryData=supplementaryData)
547 
548  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
549  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
550 
551  if self.config.doWrite:
552  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
553  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
554  else:
555  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
556  self.log.info("Persisting %s", coaddDatasetName)
557  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
558  if self.config.doNImage and retStruct.nImage is not None:
559  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
560 
561  return retStruct
562 
563  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
564  """Interpolate over missing data and mask bright stars.
565 
566  Parameters
567  ----------
568  coaddExposure : `lsst.afw.image.Exposure`
569  The coadded exposure to process.
570  dataRef : `lsst.daf.persistence.ButlerDataRef`
571  Butler data reference for supplementary data.
572  """
573  if self.config.doInterp:
574  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
575  # The variance must be positive; work around for DM-3201.
576  varArray = coaddExposure.variance.array
577  with numpy.errstate(invalid="ignore"):
578  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
579 
580  if self.config.doMaskBrightObjects:
581  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
582 
583  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
584  """Make additional inputs to run() specific to subclasses (Gen2)
585 
586  Duplicates interface of `runDataRef` method
587  Available to be implemented by subclasses only if they need the
588  coadd dataRef for performing preliminary processing before
589  assembling the coadd.
590 
591  Parameters
592  ----------
593  dataRef : `lsst.daf.persistence.ButlerDataRef`
594  Butler data reference for supplementary data.
595  selectDataList : `list` (optional)
596  Optional List of data references to Calexps.
597  warpRefList : `list` (optional)
598  Optional List of data references to Warps.
599  """
600  return pipeBase.Struct()
601 
602  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
603  """Make additional inputs to run() specific to subclasses (Gen3)
604 
605  Duplicates interface of `runQuantum` method.
606  Available to be implemented by subclasses only if they need the
607  coadd dataRef for performing preliminary processing before
608  assembling the coadd.
609 
610  Parameters
611  ----------
612  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
613  Gen3 Butler object for fetching additional data products before
614  running the Task specialized for quantum being processed
615  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
616  Attributes are the names of the connections describing input dataset types.
617  Values are DatasetRefs that task consumes for corresponding dataset type.
618  DataIds are guaranteed to match data objects in ``inputData``.
619  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
620  Attributes are the names of the connections describing output dataset types.
621  Values are DatasetRefs that task is to produce
622  for corresponding dataset type.
623  """
624  return pipeBase.Struct()
625 
626  def getTempExpRefList(self, patchRef, calExpRefList):
627  """Generate list data references corresponding to warped exposures
628  that lie within the patch to be coadded.
629 
630  Parameters
631  ----------
632  patchRef : `dataRef`
633  Data reference for patch.
634  calExpRefList : `list`
635  List of data references for input calexps.
636 
637  Returns
638  -------
639  tempExpRefList : `list`
640  List of Warp/CoaddTempExp data references.
641  """
642  butler = patchRef.getButler()
643  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
644  self.getTempExpDatasetName(self.warpType))
645  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
646  g, groupData.keys) for
647  g in groupData.groups.keys()]
648  return tempExpRefList
649 
650  def prepareInputs(self, refList):
651  """Prepare the input warps for coaddition by measuring the weight for
652  each warp and the scaling for the photometric zero point.
653 
654  Each Warp has its own photometric zeropoint and background variance.
655  Before coadding these Warps together, compute a scale factor to
656  normalize the photometric zeropoint and compute the weight for each Warp.
657 
658  Parameters
659  ----------
660  refList : `list`
661  List of data references to tempExp
662 
663  Returns
664  -------
665  result : `lsst.pipe.base.Struct`
666  Result struct with components:
667 
668  - ``tempExprefList``: `list` of data references to tempExp.
669  - ``weightList``: `list` of weightings.
670  - ``imageScalerList``: `list` of image scalers.
671  """
672  statsCtrl = afwMath.StatisticsControl()
673  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
674  statsCtrl.setNumIter(self.config.clipIter)
675  statsCtrl.setAndMask(self.getBadPixelMask())
676  statsCtrl.setNanSafe(True)
677  # compute tempExpRefList: a list of tempExpRef that actually exist
678  # and weightList: a list of the weight of the associated coadd tempExp
679  # and imageScalerList: a list of scale factors for the associated coadd tempExp
680  tempExpRefList = []
681  weightList = []
682  imageScalerList = []
683  tempExpName = self.getTempExpDatasetName(self.warpType)
684  for tempExpRef in refList:
685  # Gen3's DeferredDatasetHandles are guaranteed to exist and
686  # therefore have no datasetExists() method
687  if not isinstance(tempExpRef, DeferredDatasetHandle):
688  if not tempExpRef.datasetExists(tempExpName):
689  self.log.warning("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
690  continue
691 
692  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
693  # Ignore any input warp that is empty of data
694  if numpy.isnan(tempExp.image.array).all():
695  continue
696  maskedImage = tempExp.getMaskedImage()
697  imageScaler = self.scaleZeroPoint.computeImageScaler(
698  exposure=tempExp,
699  dataRef=tempExpRef,
700  )
701  try:
702  imageScaler.scaleMaskedImage(maskedImage)
703  except Exception as e:
704  self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
705  continue
706  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
707  afwMath.MEANCLIP, statsCtrl)
708  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
709  weight = 1.0 / float(meanVar)
710  if not numpy.isfinite(weight):
711  self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId)
712  continue
713  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
714 
715  del maskedImage
716  del tempExp
717 
718  tempExpRefList.append(tempExpRef)
719  weightList.append(weight)
720  imageScalerList.append(imageScaler)
721 
722  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
723  imageScalerList=imageScalerList)
724 
725  def prepareStats(self, mask=None):
726  """Prepare the statistics for coadding images.
727 
728  Parameters
729  ----------
730  mask : `int`, optional
731  Bit mask value to exclude from coaddition.
732 
733  Returns
734  -------
735  stats : `lsst.pipe.base.Struct`
736  Statistics structure with the following fields:
737 
738  - ``statsCtrl``: Statistics control object for coadd
739  (`lsst.afw.math.StatisticsControl`)
740  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
741  """
742  if mask is None:
743  mask = self.getBadPixelMask()
744  statsCtrl = afwMath.StatisticsControl()
745  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
746  statsCtrl.setNumIter(self.config.clipIter)
747  statsCtrl.setAndMask(mask)
748  statsCtrl.setNanSafe(True)
749  statsCtrl.setWeighted(True)
750  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
751  for plane, threshold in self.config.maskPropagationThresholds.items():
752  bit = afwImage.Mask.getMaskPlane(plane)
753  statsCtrl.setMaskPropagationThreshold(bit, threshold)
754  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
755  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
756 
757  @pipeBase.timeMethod
758  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
759  altMaskList=None, mask=None, supplementaryData=None):
760  """Assemble a coadd from input warps
761 
762  Assemble the coadd using the provided list of coaddTempExps. Since
763  the full coadd covers a patch (a large area), the assembly is
764  performed over small areas on the image at a time in order to
765  conserve memory usage. Iterate over subregions within the outer
766  bbox of the patch using `assembleSubregion` to stack the corresponding
767  subregions from the coaddTempExps with the statistic specified.
768  Set the edge bits the coadd mask based on the weight map.
769 
770  Parameters
771  ----------
772  skyInfo : `lsst.pipe.base.Struct`
773  Struct with geometric information about the patch.
774  tempExpRefList : `list`
775  List of data references to Warps (previously called CoaddTempExps).
776  imageScalerList : `list`
777  List of image scalers.
778  weightList : `list`
779  List of weights
780  altMaskList : `list`, optional
781  List of alternate masks to use rather than those stored with
782  tempExp.
783  mask : `int`, optional
784  Bit mask value to exclude from coaddition.
785  supplementaryData : lsst.pipe.base.Struct, optional
786  Struct with additional data products needed to assemble coadd.
787  Only used by subclasses that implement `makeSupplementaryData`
788  and override `run`.
789 
790  Returns
791  -------
792  result : `lsst.pipe.base.Struct`
793  Result struct with components:
794 
795  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
796  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
797  - ``inputMap``: bit-wise map of inputs, if requested.
798  - ``warpRefList``: input list of refs to the warps (
799  ``lsst.daf.butler.DeferredDatasetHandle`` or
800  ``lsst.daf.persistence.ButlerDataRef``)
801  (unmodified)
802  - ``imageScalerList``: input list of image scalers (unmodified)
803  - ``weightList``: input list of weights (unmodified)
804  """
805  tempExpName = self.getTempExpDatasetName(self.warpType)
806  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
807  stats = self.prepareStats(mask=mask)
808 
809  if altMaskList is None:
810  altMaskList = [None]*len(tempExpRefList)
811 
812  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
813  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
814  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
815  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
816  coaddMaskedImage = coaddExposure.getMaskedImage()
817  subregionSizeArr = self.config.subregionSize
818  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
819  # if nImage is requested, create a zero one which can be passed to assembleSubregion
820  if self.config.doNImage:
821  nImage = afwImage.ImageU(skyInfo.bbox)
822  else:
823  nImage = None
824  # If inputMap is requested, create the initial version that can be masked in
825  # assembleSubregion.
826  if self.config.doInputMap:
827  self.inputMapper.build_ccd_input_map(skyInfo.bbox,
828  skyInfo.wcs,
829  coaddExposure.getInfo().getCoaddInputs().ccds)
830 
831  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
832  try:
833  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
834  weightList, altMaskList, stats.flags, stats.ctrl,
835  nImage=nImage)
836  except Exception as e:
837  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
838 
839  # If inputMap is requested, we must finalize the map after the accumulation.
840  if self.config.doInputMap:
841  self.inputMapper.finalize_ccd_input_map_mask()
842  inputMap = self.inputMapper.ccd_input_map
843  else:
844  inputMap = None
845 
846  self.setInexactPsf(coaddMaskedImage.getMask())
847  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
848  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
849  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
850  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
851  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
852  weightList=weightList, inputMap=inputMap)
853 
854  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
855  """Set the metadata for the coadd.
856 
857  This basic implementation sets the filter from the first input.
858 
859  Parameters
860  ----------
861  coaddExposure : `lsst.afw.image.Exposure`
862  The target exposure for the coadd.
863  tempExpRefList : `list`
864  List of data references to tempExp.
865  weightList : `list`
866  List of weights.
867  """
868  assert len(tempExpRefList) == len(weightList), "Length mismatch"
869  tempExpName = self.getTempExpDatasetName(self.warpType)
870  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
871  # (and we need more than just the PropertySet that contains the header), which is not possible
872  # with the current butler (see #2777).
873  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
874 
875  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
876  # Gen 3 API
877  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
878  else:
879  # Gen 2 API. Delete this when Gen 2 retired
880  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
881  for tempExpRef in tempExpRefList]
882  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
883 
884  # Set the coadd FilterLabel to the band of the first input exposure:
885  # Coadds are calibrated, so the physical label is now meaningless.
886  coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
887  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
888  coaddInputs.ccds.reserve(numCcds)
889  coaddInputs.visits.reserve(len(tempExpList))
890 
891  for tempExp, weight in zip(tempExpList, weightList):
892  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
893 
894  if self.config.doUsePsfMatchedPolygons:
895  self.shrinkValidPolygons(coaddInputs)
896 
897  coaddInputs.visits.sort()
898  if self.warpType == "psfMatched":
899  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
900  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
901  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
902  # having the maximum width (sufficient because square)
903  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
904  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
905  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
906  else:
907  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
908  self.config.coaddPsf.makeControl())
909  coaddExposure.setPsf(psf)
910  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
911  coaddExposure.getWcs())
912  coaddExposure.getInfo().setApCorrMap(apCorrMap)
913  if self.config.doAttachTransmissionCurve:
914  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
915  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
916 
917  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
918  altMaskList, statsFlags, statsCtrl, nImage=None):
919  """Assemble the coadd for a sub-region.
920 
921  For each coaddTempExp, check for (and swap in) an alternative mask
922  if one is passed. Remove mask planes listed in
923  `config.removeMaskPlanes`. Finally, stack the actual exposures using
924  `lsst.afw.math.statisticsStack` with the statistic specified by
925  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
926  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
927  an N-sigma clipped mean where N and iterations are specified by
928  statsCtrl. Assign the stacked subregion back to the coadd.
929 
930  Parameters
931  ----------
932  coaddExposure : `lsst.afw.image.Exposure`
933  The target exposure for the coadd.
934  bbox : `lsst.geom.Box`
935  Sub-region to coadd.
936  tempExpRefList : `list`
937  List of data reference to tempExp.
938  imageScalerList : `list`
939  List of image scalers.
940  weightList : `list`
941  List of weights.
942  altMaskList : `list`
943  List of alternate masks to use rather than those stored with
944  tempExp, or None. Each element is dict with keys = mask plane
945  name to which to add the spans.
946  statsFlags : `lsst.afw.math.Property`
947  Property object for statistic for coadd.
948  statsCtrl : `lsst.afw.math.StatisticsControl`
949  Statistics control object for coadd.
950  nImage : `lsst.afw.image.ImageU`, optional
951  Keeps track of exposure count for each pixel.
952  """
953  self.log.debug("Computing coadd over %s", bbox)
954  tempExpName = self.getTempExpDatasetName(self.warpType)
955  coaddExposure.mask.addMaskPlane("REJECTED")
956  coaddExposure.mask.addMaskPlane("CLIPPED")
957  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
958  maskMap = self.setRejectedMaskMapping(statsCtrl)
959  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
960  maskedImageList = []
961  if nImage is not None:
962  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
963  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
964 
965  if isinstance(tempExpRef, DeferredDatasetHandle):
966  # Gen 3 API
967  exposure = tempExpRef.get(parameters={'bbox': bbox})
968  else:
969  # Gen 2 API. Delete this when Gen 2 retired
970  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
971 
972  maskedImage = exposure.getMaskedImage()
973  mask = maskedImage.getMask()
974  if altMask is not None:
975  self.applyAltMaskPlanes(mask, altMask)
976  imageScaler.scaleMaskedImage(maskedImage)
977 
978  # Add 1 for each pixel which is not excluded by the exclude mask.
979  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
980  if nImage is not None:
981  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
982  if self.config.removeMaskPlanes:
983  self.removeMaskPlanes(maskedImage)
984  maskedImageList.append(maskedImage)
985 
986  if self.config.doInputMap:
987  visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
988  self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
989 
990  with self.timer("stack"):
991  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
992  clipped, # also set output to CLIPPED if sigma-clipped
993  maskMap)
994  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
995  if nImage is not None:
996  nImage.assign(subNImage, bbox)
997 
998  def removeMaskPlanes(self, maskedImage):
999  """Unset the mask of an image for mask planes specified in the config.
1000 
1001  Parameters
1002  ----------
1003  maskedImage : `lsst.afw.image.MaskedImage`
1004  The masked image to be modified.
1005  """
1006  mask = maskedImage.getMask()
1007  for maskPlane in self.config.removeMaskPlanes:
1008  try:
1009  mask &= ~mask.getPlaneBitMask(maskPlane)
1010  except pexExceptions.InvalidParameterError:
1011  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
1012  maskPlane)
1013 
1014  @staticmethod
1015  def setRejectedMaskMapping(statsCtrl):
1016  """Map certain mask planes of the warps to new planes for the coadd.
1017 
1018  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1019  or CLIPPED, set it to REJECTED on the coadd.
1020  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1021  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1022 
1023  Parameters
1024  ----------
1025  statsCtrl : `lsst.afw.math.StatisticsControl`
1026  Statistics control object for coadd
1027 
1028  Returns
1029  -------
1030  maskMap : `list` of `tuple` of `int`
1031  A list of mappings of mask planes of the warped exposures to
1032  mask planes of the coadd.
1033  """
1034  edge = afwImage.Mask.getPlaneBitMask("EDGE")
1035  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1036  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1037  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1038  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1039  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1040  (clipped, clipped)]
1041  return maskMap
1042 
1043  def applyAltMaskPlanes(self, mask, altMaskSpans):
1044  """Apply in place alt mask formatted as SpanSets to a mask.
1045 
1046  Parameters
1047  ----------
1048  mask : `lsst.afw.image.Mask`
1049  Original mask.
1050  altMaskSpans : `dict`
1051  SpanSet lists to apply. Each element contains the new mask
1052  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1053  and list of SpanSets to apply to the mask.
1054 
1055  Returns
1056  -------
1057  mask : `lsst.afw.image.Mask`
1058  Updated mask.
1059  """
1060  if self.config.doUsePsfMatchedPolygons:
1061  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1062  # Clear away any other masks outside the validPolygons. These pixels are no longer
1063  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1064  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1065  # This mask-clearing step must only occur *before* applying the new masks below
1066  for spanSet in altMaskSpans['NO_DATA']:
1067  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1068 
1069  for plane, spanSetList in altMaskSpans.items():
1070  maskClipValue = mask.addMaskPlane(plane)
1071  for spanSet in spanSetList:
1072  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1073  return mask
1074 
1075  def shrinkValidPolygons(self, coaddInputs):
1076  """Shrink coaddInputs' ccds' ValidPolygons in place.
1077 
1078  Either modify each ccd's validPolygon in place, or if CoaddInputs
1079  does not have a validPolygon, create one from its bbox.
1080 
1081  Parameters
1082  ----------
1083  coaddInputs : `lsst.afw.image.coaddInputs`
1084  Original mask.
1085 
1086  """
1087  for ccd in coaddInputs.ccds:
1088  polyOrig = ccd.getValidPolygon()
1089  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1090  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1091  if polyOrig:
1092  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1093  else:
1094  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1095  ccd.setValidPolygon(validPolygon)
1096 
1097  def readBrightObjectMasks(self, dataRef):
1098  """Retrieve the bright object masks.
1099 
1100  Returns None on failure.
1101 
1102  Parameters
1103  ----------
1104  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1105  A Butler dataRef.
1106 
1107  Returns
1108  -------
1109  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1110  Bright object mask from the Butler object, or None if it cannot
1111  be retrieved.
1112  """
1113  try:
1114  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1115  except Exception as e:
1116  self.log.warning("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1117  return None
1118 
1119  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1120  """Set the bright object masks.
1121 
1122  Parameters
1123  ----------
1124  exposure : `lsst.afw.image.Exposure`
1125  Exposure under consideration.
1126  dataId : `lsst.daf.persistence.dataId`
1127  Data identifier dict for patch.
1128  brightObjectMasks : `lsst.afw.table`
1129  Table of bright objects to mask.
1130  """
1131 
1132  if brightObjectMasks is None:
1133  self.log.warning("Unable to apply bright object mask: none supplied")
1134  return
1135  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1136  mask = exposure.getMaskedImage().getMask()
1137  wcs = exposure.getWcs()
1138  plateScale = wcs.getPixelScale().asArcseconds()
1139 
1140  for rec in brightObjectMasks:
1141  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1142  if rec["type"] == "box":
1143  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1144  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1145  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1146 
1147  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1148  bbox = geom.Box2I(center - halfSize, center + halfSize)
1149 
1150  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1151  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1152  spans = afwGeom.SpanSet(bbox)
1153  elif rec["type"] == "circle":
1154  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1155  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1156  else:
1157  self.log.warning("Unexpected region type %s at %s", rec["type"], center)
1158  continue
1159  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1160 
1161  def setInexactPsf(self, mask):
1162  """Set INEXACT_PSF mask plane.
1163 
1164  If any of the input images isn't represented in the coadd (due to
1165  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1166  these pixels.
1167 
1168  Parameters
1169  ----------
1170  mask : `lsst.afw.image.Mask`
1171  Coadded exposure's mask, modified in-place.
1172  """
1173  mask.addMaskPlane("INEXACT_PSF")
1174  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1175  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1176  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1177  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1178  array = mask.getArray()
1179  selected = array & (sensorEdge | clipped | rejected) > 0
1180  array[selected] |= inexactPsf
1181 
1182  @classmethod
1183  def _makeArgumentParser(cls):
1184  """Create an argument parser.
1185  """
1186  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1187  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1188  + cls.ConfigClass().warpType + "Warp",
1189  help="data ID, e.g. --id tract=12345 patch=1,2",
1190  ContainerClass=AssembleCoaddDataIdContainer)
1191  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1192  ContainerClass=SelectDataIdContainer)
1193  return parser
1194 
1195  @staticmethod
1196  def _subBBoxIter(bbox, subregionSize):
1197  """Iterate over subregions of a bbox.
1198 
1199  Parameters
1200  ----------
1201  bbox : `lsst.geom.Box2I`
1202  Bounding box over which to iterate.
1203  subregionSize: `lsst.geom.Extent2I`
1204  Size of sub-bboxes.
1205 
1206  Yields
1207  ------
1208  subBBox : `lsst.geom.Box2I`
1209  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1210  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1211  the edges of ``bbox``, but it will never be empty.
1212  """
1213  if bbox.isEmpty():
1214  raise RuntimeError("bbox %s is empty" % (bbox,))
1215  if subregionSize[0] < 1 or subregionSize[1] < 1:
1216  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1217 
1218  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1219  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1220  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1221  subBBox.clip(bbox)
1222  if subBBox.isEmpty():
1223  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1224  "colShift=%s, rowShift=%s" %
1225  (bbox, subregionSize, colShift, rowShift))
1226  yield subBBox
1227 
1228  def filterWarps(self, inputs, goodVisits):
1229  """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1230 
1231  Parameters
1232  ----------
1233  inputs : list
1234  List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1235  goodVisit : `dict`
1236  Dictionary with good visitIds as the keys. Value ignored.
1237 
1238  Returns:
1239  --------
1240  filteredInputs : `list`
1241  Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1242  """
1243  inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1244  filteredInputs = []
1245  for visit in goodVisits.keys():
1246  filteredInputs.append(inputWarpDict[visit])
1247  return filteredInputs
1248 
1249 
1250 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1251  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1252  """
1253 
1254  def makeDataRefList(self, namespace):
1255  """Make self.refList from self.idList.
1256 
1257  Parameters
1258  ----------
1259  namespace
1260  Results of parsing command-line (with ``butler`` and ``log`` elements).
1261  """
1262  datasetType = namespace.config.coaddName + "Coadd"
1263  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1264 
1265  for dataId in self.idList:
1266  # tract and patch are required
1267  for key in keysCoadd:
1268  if key not in dataId:
1269  raise RuntimeError("--id must include " + key)
1270 
1271  dataRef = namespace.butler.dataRef(
1272  datasetType=datasetType,
1273  dataId=dataId,
1274  )
1275  self.refList.append(dataRef)
1276 
1277 
1278 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1279  """Function to count the number of pixels with a specific mask in a
1280  footprint.
1281 
1282  Find the intersection of mask & footprint. Count all pixels in the mask
1283  that are in the intersection that have bitmask set but do not have
1284  ignoreMask set. Return the count.
1285 
1286  Parameters
1287  ----------
1288  mask : `lsst.afw.image.Mask`
1289  Mask to define intersection region by.
1290  footprint : `lsst.afw.detection.Footprint`
1291  Footprint to define the intersection region by.
1292  bitmask
1293  Specific mask that we wish to count the number of occurances of.
1294  ignoreMask
1295  Pixels to not consider.
1296 
1297  Returns
1298  -------
1299  result : `int`
1300  Count of number of pixels in footprint with specified mask.
1301  """
1302  bbox = footprint.getBBox()
1303  bbox.clip(mask.getBBox(afwImage.PARENT))
1304  fp = afwImage.Mask(bbox)
1305  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1306  footprint.spans.setMask(fp, bitmask)
1307  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1308  (subMask.getArray() & ignoreMask) == 0).sum()
1309 
1310 
1311 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1312  """Configuration parameters for the SafeClipAssembleCoaddTask.
1313  """
1314  clipDetection = pexConfig.ConfigurableField(
1315  target=SourceDetectionTask,
1316  doc="Detect sources on difference between unclipped and clipped coadd")
1317  minClipFootOverlap = pexConfig.Field(
1318  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1319  dtype=float,
1320  default=0.6
1321  )
1322  minClipFootOverlapSingle = pexConfig.Field(
1323  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1324  "clipped when only one visit overlaps",
1325  dtype=float,
1326  default=0.5
1327  )
1328  minClipFootOverlapDouble = pexConfig.Field(
1329  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1330  "clipped when two visits overlap",
1331  dtype=float,
1332  default=0.45
1333  )
1334  maxClipFootOverlapDouble = pexConfig.Field(
1335  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1336  "considering two visits",
1337  dtype=float,
1338  default=0.15
1339  )
1340  minBigOverlap = pexConfig.Field(
1341  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1342  "when labeling clipped footprints",
1343  dtype=int,
1344  default=100
1345  )
1346 
1347  def setDefaults(self):
1348  """Set default values for clipDetection.
1349 
1350  Notes
1351  -----
1352  The numeric values for these configuration parameters were
1353  empirically determined, future work may further refine them.
1354  """
1355  AssembleCoaddConfig.setDefaults(self)
1356  self.clipDetectionclipDetection.doTempLocalBackground = False
1357  self.clipDetectionclipDetection.reEstimateBackground = False
1358  self.clipDetectionclipDetection.returnOriginalFootprints = False
1359  self.clipDetectionclipDetection.thresholdPolarity = "both"
1360  self.clipDetectionclipDetection.thresholdValue = 2
1361  self.clipDetectionclipDetection.nSigmaToGrow = 2
1362  self.clipDetectionclipDetection.minPixels = 4
1363  self.clipDetectionclipDetection.isotropicGrow = True
1364  self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1365  self.sigmaClipsigmaClip = 1.5
1366  self.clipIterclipIter = 3
1367  self.statisticstatistic = "MEAN"
1368 
1369  def validate(self):
1370  if self.doSigmaClipdoSigmaClip:
1371  log.warning("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1372  "Ignoring doSigmaClip.")
1373  self.doSigmaClipdoSigmaClip = False
1374  if self.statisticstatistic != "MEAN":
1375  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1376  "(%s chosen). Please set statistic to MEAN."
1377  % (self.statisticstatistic))
1378  AssembleCoaddTask.ConfigClass.validate(self)
1379 
1380 
1381 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1382  """Assemble a coadded image from a set of coadded temporary exposures,
1383  being careful to clip & flag areas with potential artifacts.
1384 
1385  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1386  we clip outliers). The problem with doing this is that when computing the
1387  coadd PSF at a given location, individual visit PSFs from visits with
1388  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1389  In this task, we correct for this behavior by creating a new
1390  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1391  coaddTempExps and the final coadd where
1392 
1393  i. difference imaging suggests that there is an outlier and
1394  ii. this outlier appears on only one or two images.
1395 
1396  Such regions will not contribute to the final coadd. Furthermore, any
1397  routine to determine the coadd PSF can now be cognizant of clipped regions.
1398  Note that the algorithm implemented by this task is preliminary and works
1399  correctly for HSC data. Parameter modifications and or considerable
1400  redesigning of the algorithm is likley required for other surveys.
1401 
1402  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1403  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1404  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1405  if you wish.
1406 
1407  Notes
1408  -----
1409  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1410  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1411  see `baseDebug` for more about ``debug.py`` files.
1412  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1413  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1414  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1415  for further information.
1416 
1417  Examples
1418  --------
1419  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1420  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1421  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1422 
1423  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1424  and filter to be coadded (specified using
1425  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1426  along with a list of coaddTempExps to attempt to coadd (specified using
1427  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1428  Only the coaddTempExps that cover the specified tract and patch will be
1429  coadded. A list of the available optional arguments can be obtained by
1430  calling assembleCoadd.py with the --help command line argument:
1431 
1432  .. code-block:: none
1433 
1434  assembleCoadd.py --help
1435 
1436  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1437  context of multi-band processing, we will generate the HSC-I & -R band
1438  coadds from HSC engineering test data provided in the ci_hsc package.
1439  To begin, assuming that the lsst stack has been already set up, we must
1440  set up the obs_subaru and ci_hsc packages. This defines the environment
1441  variable $CI_HSC_DIR and points at the location of the package. The raw
1442  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1443  the coadds, we must first
1444 
1445  - ``processCcd``
1446  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1447  - ``makeSkyMap``
1448  create a skymap that covers the area of the sky present in the raw exposures
1449  - ``makeCoaddTempExp``
1450  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1451 
1452  We can perform all of these steps by running
1453 
1454  .. code-block:: none
1455 
1456  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1457 
1458  This will produce warped coaddTempExps for each visit. To coadd the
1459  warped data, we call ``assembleCoadd.py`` as follows:
1460 
1461  .. code-block:: none
1462 
1463  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1464  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1465  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1466  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1467  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1468  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1469  --selectId visit=903988 ccd=24
1470 
1471  This will process the HSC-I band data. The results are written in
1472  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1473 
1474  You may also choose to run:
1475 
1476  .. code-block:: none
1477 
1478  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1479  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1480  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1481  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1482  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1483  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1484  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1485  --selectId visit=903346 ccd=12
1486 
1487  to generate the coadd for the HSC-R band if you are interested in following
1488  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1489  """
1490  ConfigClass = SafeClipAssembleCoaddConfig
1491  _DefaultName = "safeClipAssembleCoadd"
1492 
1493  def __init__(self, *args, **kwargs):
1494  AssembleCoaddTask.__init__(self, *args, **kwargs)
1495  schema = afwTable.SourceTable.makeMinimalSchema()
1496  self.makeSubtask("clipDetection", schema=schema)
1497 
1498  @utils.inheritDoc(AssembleCoaddTask)
1499  @pipeBase.timeMethod
1500  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1501  """Assemble the coadd for a region.
1502 
1503  Compute the difference of coadds created with and without outlier
1504  rejection to identify coadd pixels that have outlier values in some
1505  individual visits.
1506  Detect clipped regions on the difference image and mark these regions
1507  on the one or two individual coaddTempExps where they occur if there
1508  is significant overlap between the clipped region and a source. This
1509  leaves us with a set of footprints from the difference image that have
1510  been identified as having occured on just one or two individual visits.
1511  However, these footprints were generated from a difference image. It
1512  is conceivable for a large diffuse source to have become broken up
1513  into multiple footprints acrosss the coadd difference in this process.
1514  Determine the clipped region from all overlapping footprints from the
1515  detected sources in each visit - these are big footprints.
1516  Combine the small and big clipped footprints and mark them on a new
1517  bad mask plane.
1518  Generate the coadd using `AssembleCoaddTask.run` without outlier
1519  removal. Clipped footprints will no longer make it into the coadd
1520  because they are marked in the new bad mask plane.
1521 
1522  Notes
1523  -----
1524  args and kwargs are passed but ignored in order to match the call
1525  signature expected by the parent task.
1526  """
1527  exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1528  mask = exp.getMaskedImage().getMask()
1529  mask.addMaskPlane("CLIPPED")
1530 
1531  result = self.detectClipdetectClip(exp, tempExpRefList)
1532 
1533  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1534 
1535  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1536  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1537  # Append big footprints from individual Warps to result.clipSpans
1538  bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1539  result.detectionFootprints, maskClipValue, maskDetValue,
1540  exp.getBBox())
1541  # Create mask of the current clipped footprints
1542  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1543  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1544 
1545  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1546  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1547  maskClip |= maskClipBig
1548 
1549  # Assemble coadd from base class, but ignoring CLIPPED pixels
1550  badMaskPlanes = self.config.badMaskPlanes[:]
1551  badMaskPlanes.append("CLIPPED")
1552  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1553  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1554  result.clipSpans, mask=badPixelMask)
1555 
1556  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1557  """Return an exposure that contains the difference between unclipped
1558  and clipped coadds.
1559 
1560  Generate a difference image between clipped and unclipped coadds.
1561  Compute the difference image by subtracting an outlier-clipped coadd
1562  from an outlier-unclipped coadd. Return the difference image.
1563 
1564  Parameters
1565  ----------
1566  skyInfo : `lsst.pipe.base.Struct`
1567  Patch geometry information, from getSkyInfo
1568  tempExpRefList : `list`
1569  List of data reference to tempExp
1570  imageScalerList : `list`
1571  List of image scalers
1572  weightList : `list`
1573  List of weights
1574 
1575  Returns
1576  -------
1577  exp : `lsst.afw.image.Exposure`
1578  Difference image of unclipped and clipped coadd wrapped in an Exposure
1579  """
1580  config = AssembleCoaddConfig()
1581  # getattr necessary because subtasks do not survive Config.toDict()
1582  # exclude connections because the class of self.config.connections is not
1583  # the same as AssembleCoaddConfig.connections, and the connections are not
1584  # needed to run this task anyway.
1585  configIntersection = {k: getattr(self.config, k)
1586  for k, v in self.config.toDict().items()
1587  if (k in config.keys() and k != "connections")}
1588  configIntersection['doInputMap'] = False
1589  configIntersection['doNImage'] = False
1590  config.update(**configIntersection)
1591 
1592  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1593  config.statistic = 'MEAN'
1594  task = AssembleCoaddTask(config=config)
1595  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1596 
1597  config.statistic = 'MEANCLIP'
1598  task = AssembleCoaddTask(config=config)
1599  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1600 
1601  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1602  coaddDiff -= coaddClip.getMaskedImage()
1603  exp = afwImage.ExposureF(coaddDiff)
1604  exp.setPsf(coaddMean.getPsf())
1605  return exp
1606 
1607  def detectClip(self, exp, tempExpRefList):
1608  """Detect clipped regions on an exposure and set the mask on the
1609  individual tempExp masks.
1610 
1611  Detect footprints in the difference image after smoothing the
1612  difference image with a Gaussian kernal. Identify footprints that
1613  overlap with one or two input ``coaddTempExps`` by comparing the
1614  computed overlap fraction to thresholds set in the config. A different
1615  threshold is applied depending on the number of overlapping visits
1616  (restricted to one or two). If the overlap exceeds the thresholds,
1617  the footprint is considered "CLIPPED" and is marked as such on the
1618  coaddTempExp. Return a struct with the clipped footprints, the indices
1619  of the ``coaddTempExps`` that end up overlapping with the clipped
1620  footprints, and a list of new masks for the ``coaddTempExps``.
1621 
1622  Parameters
1623  ----------
1624  exp : `lsst.afw.image.Exposure`
1625  Exposure to run detection on.
1626  tempExpRefList : `list`
1627  List of data reference to tempExp.
1628 
1629  Returns
1630  -------
1631  result : `lsst.pipe.base.Struct`
1632  Result struct with components:
1633 
1634  - ``clipFootprints``: list of clipped footprints.
1635  - ``clipIndices``: indices for each ``clippedFootprint`` in
1636  ``tempExpRefList``.
1637  - ``clipSpans``: List of dictionaries containing spanSet lists
1638  to clip. Each element contains the new maskplane name
1639  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1640  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1641  compressed into footprints.
1642  """
1643  mask = exp.getMaskedImage().getMask()
1644  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1645  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1646  # Merge positive and negative together footprints together
1647  fpSet.positive.merge(fpSet.negative)
1648  footprints = fpSet.positive
1649  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1650  ignoreMask = self.getBadPixelMask()
1651 
1652  clipFootprints = []
1653  clipIndices = []
1654  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1655 
1656  # for use by detectClipBig
1657  visitDetectionFootprints = []
1658 
1659  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1660  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1661  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1662 
1663  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1664  for i, warpRef in enumerate(tempExpRefList):
1665  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1666  immediate=True).getMaskedImage().getMask()
1667  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1668  afwImage.PARENT, True)
1669  maskVisitDet &= maskDetValue
1670  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1671  visitDetectionFootprints.append(visitFootprints)
1672 
1673  for j, footprint in enumerate(footprints.getFootprints()):
1674  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1675  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1676 
1677  # build a list of clipped spans for each visit
1678  for j, footprint in enumerate(footprints.getFootprints()):
1679  nPixel = footprint.getArea()
1680  overlap = [] # hold the overlap with each visit
1681  indexList = [] # index of visit in global list
1682  for i in range(len(tempExpRefList)):
1683  ignore = ignoreArr[i, j]
1684  overlapDet = overlapDetArr[i, j]
1685  totPixel = nPixel - ignore
1686 
1687  # If we have more bad pixels than detection skip
1688  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1689  continue
1690  overlap.append(overlapDet/float(totPixel))
1691  indexList.append(i)
1692 
1693  overlap = numpy.array(overlap)
1694  if not len(overlap):
1695  continue
1696 
1697  keep = False # Should this footprint be marked as clipped?
1698  keepIndex = [] # Which tempExps does the clipped footprint belong to
1699 
1700  # If footprint only has one overlap use a lower threshold
1701  if len(overlap) == 1:
1702  if overlap[0] > self.config.minClipFootOverlapSingle:
1703  keep = True
1704  keepIndex = [0]
1705  else:
1706  # This is the general case where only visit should be clipped
1707  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1708  if len(clipIndex) == 1:
1709  keep = True
1710  keepIndex = [clipIndex[0]]
1711 
1712  # Test if there are clipped objects that overlap two different visits
1713  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1714  if len(clipIndex) == 2 and len(overlap) > 3:
1715  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1716  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1717  keep = True
1718  keepIndex = clipIndex
1719 
1720  if not keep:
1721  continue
1722 
1723  for index in keepIndex:
1724  globalIndex = indexList[index]
1725  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1726 
1727  clipIndices.append(numpy.array(indexList)[keepIndex])
1728  clipFootprints.append(footprint)
1729 
1730  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1731  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1732 
1733  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1734  maskClipValue, maskDetValue, coaddBBox):
1735  """Return individual warp footprints for large artifacts and append
1736  them to ``clipList`` in place.
1737 
1738  Identify big footprints composed of many sources in the coadd
1739  difference that may have originated in a large diffuse source in the
1740  coadd. We do this by indentifying all clipped footprints that overlap
1741  significantly with each source in all the coaddTempExps.
1742 
1743  Parameters
1744  ----------
1745  clipList : `list`
1746  List of alt mask SpanSets with clipping information. Modified.
1747  clipFootprints : `list`
1748  List of clipped footprints.
1749  clipIndices : `list`
1750  List of which entries in tempExpClipList each footprint belongs to.
1751  maskClipValue
1752  Mask value of clipped pixels.
1753  maskDetValue
1754  Mask value of detected pixels.
1755  coaddBBox : `lsst.geom.Box`
1756  BBox of the coadd and warps.
1757 
1758  Returns
1759  -------
1760  bigFootprintsCoadd : `list`
1761  List of big footprints
1762  """
1763  bigFootprintsCoadd = []
1764  ignoreMask = self.getBadPixelMask()
1765  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1766  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1767  for footprint in visitFootprints.getFootprints():
1768  footprint.spans.setMask(maskVisitDet, maskDetValue)
1769 
1770  # build a mask of clipped footprints that are in this visit
1771  clippedFootprintsVisit = []
1772  for foot, clipIndex in zip(clipFootprints, clipIndices):
1773  if index not in clipIndex:
1774  continue
1775  clippedFootprintsVisit.append(foot)
1776  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1777  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1778 
1779  bigFootprintsVisit = []
1780  for foot in visitFootprints.getFootprints():
1781  if foot.getArea() < self.config.minBigOverlap:
1782  continue
1783  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1784  if nCount > self.config.minBigOverlap:
1785  bigFootprintsVisit.append(foot)
1786  bigFootprintsCoadd.append(foot)
1787 
1788  for footprint in bigFootprintsVisit:
1789  clippedSpans["CLIPPED"].append(footprint.spans)
1790 
1791  return bigFootprintsCoadd
1792 
1793 
1795  psfMatchedWarps = pipeBase.connectionTypes.Input(
1796  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1797  "Only PSF-Matched Warps make sense for image subtraction. "
1798  "Therefore, they must be an additional declared input."),
1799  name="{inputCoaddName}Coadd_psfMatchedWarp",
1800  storageClass="ExposureF",
1801  dimensions=("tract", "patch", "skymap", "visit"),
1802  deferLoad=True,
1803  multiple=True
1804  )
1805  templateCoadd = pipeBase.connectionTypes.Output(
1806  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1807  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1808  name="{outputCoaddName}CoaddPsfMatched",
1809  storageClass="ExposureF",
1810  dimensions=("tract", "patch", "skymap", "band"),
1811  )
1812 
1813  def __init__(self, *, config=None):
1814  super().__init__(config=config)
1815  if not config.assembleStaticSkyModel.doWrite:
1816  self.outputs.remove("templateCoadd")
1817  config.validate()
1818 
1819 
1820 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1821  pipelineConnections=CompareWarpAssembleCoaddConnections):
1822  assembleStaticSkyModel = pexConfig.ConfigurableField(
1823  target=AssembleCoaddTask,
1824  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1825  " naive/first-iteration model of the static sky.",
1826  )
1827  detect = pexConfig.ConfigurableField(
1828  target=SourceDetectionTask,
1829  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1830  )
1831  detectTemplate = pexConfig.ConfigurableField(
1832  target=SourceDetectionTask,
1833  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1834  )
1835  maskStreaks = pexConfig.ConfigurableField(
1836  target=MaskStreaksTask,
1837  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1838  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1839  "streakMaskName"
1840  )
1841  streakMaskName = pexConfig.Field(
1842  dtype=str,
1843  default="STREAK",
1844  doc="Name of mask bit used for streaks"
1845  )
1846  maxNumEpochs = pexConfig.Field(
1847  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1848  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1849  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1850  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1851  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1852  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1853  "than transient and not masked.",
1854  dtype=int,
1855  default=2
1856  )
1857  maxFractionEpochsLow = pexConfig.RangeField(
1858  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1859  "Effective maxNumEpochs = "
1860  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1861  dtype=float,
1862  default=0.4,
1863  min=0., max=1.,
1864  )
1865  maxFractionEpochsHigh = pexConfig.RangeField(
1866  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1867  "Effective maxNumEpochs = "
1868  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1869  dtype=float,
1870  default=0.03,
1871  min=0., max=1.,
1872  )
1873  spatialThreshold = pexConfig.RangeField(
1874  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1875  "temporal criteria. If 0, clip all. If 1, clip none.",
1876  dtype=float,
1877  default=0.5,
1878  min=0., max=1.,
1879  inclusiveMin=True, inclusiveMax=True
1880  )
1881  doScaleWarpVariance = pexConfig.Field(
1882  doc="Rescale Warp variance plane using empirical noise?",
1883  dtype=bool,
1884  default=True,
1885  )
1886  scaleWarpVariance = pexConfig.ConfigurableField(
1887  target=ScaleVarianceTask,
1888  doc="Rescale variance on warps",
1889  )
1890  doPreserveContainedBySource = pexConfig.Field(
1891  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1892  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1893  dtype=bool,
1894  default=True,
1895  )
1896  doPrefilterArtifacts = pexConfig.Field(
1897  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1898  "because they will be excluded anyway. This prevents them from contributing "
1899  "to the outlier epoch count image and potentially being labeled as persistant."
1900  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1901  dtype=bool,
1902  default=True
1903  )
1904  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1905  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1906  dtype=str,
1907  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1908  )
1909  prefilterArtifactsRatio = pexConfig.Field(
1910  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1911  dtype=float,
1912  default=0.05
1913  )
1914  doFilterMorphological = pexConfig.Field(
1915  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1916  "be streaks.",
1917  dtype=bool,
1918  default=False
1919  )
1920 
1921  def setDefaults(self):
1922  AssembleCoaddConfig.setDefaults(self)
1923  self.statisticstatistic = 'MEAN'
1924  self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
1925 
1926  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1927  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1928  if "EDGE" in self.badMaskPlanes:
1929  self.badMaskPlanes.remove('EDGE')
1930  self.removeMaskPlanes.append('EDGE')
1931  self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1932  self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
1933  self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
1934  self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
1935  self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
1936  self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
1937  self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
1938  self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
1939  self.detectdetect.doTempLocalBackground = False
1940  self.detectdetect.reEstimateBackground = False
1941  self.detectdetect.returnOriginalFootprints = False
1942  self.detectdetect.thresholdPolarity = "both"
1943  self.detectdetect.thresholdValue = 5
1944  self.detectdetect.minPixels = 4
1945  self.detectdetect.isotropicGrow = True
1946  self.detectdetect.thresholdType = "pixel_stdev"
1947  self.detectdetect.nSigmaToGrow = 0.4
1948  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1949  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1950  self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
1951  self.detectTemplatedetectTemplate.doTempLocalBackground = False
1952  self.detectTemplatedetectTemplate.reEstimateBackground = False
1953  self.detectTemplatedetectTemplate.returnOriginalFootprints = False
1954 
1955  def validate(self):
1956  super().validate()
1957  if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
1958  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1959  "Please set assembleStaticSkyModel.doNImage=False")
1960 
1961  if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
1962  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1963  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1964  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1965  "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
1966 
1967 
1968 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1969  """Assemble a compareWarp coadded image from a set of warps
1970  by masking artifacts detected by comparing PSF-matched warps.
1971 
1972  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1973  we clip outliers). The problem with doing this is that when computing the
1974  coadd PSF at a given location, individual visit PSFs from visits with
1975  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1976  In this task, we correct for this behavior by creating a new badMaskPlane
1977  'CLIPPED' which marks pixels in the individual warps suspected to contain
1978  an artifact. We populate this plane on the input warps by comparing
1979  PSF-matched warps with a PSF-matched median coadd which serves as a
1980  model of the static sky. Any group of pixels that deviates from the
1981  PSF-matched template coadd by more than config.detect.threshold sigma,
1982  is an artifact candidate. The candidates are then filtered to remove
1983  variable sources and sources that are difficult to subtract such as
1984  bright stars. This filter is configured using the config parameters
1985  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1986  the maximum fraction of epochs that the deviation can appear in and still
1987  be considered an artifact. The spatialThreshold is the maximum fraction of
1988  pixels in the footprint of the deviation that appear in other epochs
1989  (where other epochs is defined by the temporalThreshold). If the deviant
1990  region meets this criteria of having a significant percentage of pixels
1991  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1992  set in the mask. These regions will not contribute to the final coadd.
1993  Furthermore, any routine to determine the coadd PSF can now be cognizant
1994  of clipped regions. Note that the algorithm implemented by this task is
1995  preliminary and works correctly for HSC data. Parameter modifications and
1996  or considerable redesigning of the algorithm is likley required for other
1997  surveys.
1998 
1999  ``CompareWarpAssembleCoaddTask`` sub-classes
2000  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
2001  as a subtask to generate the TemplateCoadd (the model of the static sky).
2002 
2003  Notes
2004  -----
2005  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2006  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2007  ``baseDebug`` for more about ``debug.py`` files.
2008 
2009  This task supports the following debug variables:
2010 
2011  - ``saveCountIm``
2012  If True then save the Epoch Count Image as a fits file in the `figPath`
2013  - ``figPath``
2014  Path to save the debug fits images and figures
2015 
2016  For example, put something like:
2017 
2018  .. code-block:: python
2019 
2020  import lsstDebug
2021  def DebugInfo(name):
2022  di = lsstDebug.getInfo(name)
2023  if name == "lsst.pipe.tasks.assembleCoadd":
2024  di.saveCountIm = True
2025  di.figPath = "/desired/path/to/debugging/output/images"
2026  return di
2027  lsstDebug.Info = DebugInfo
2028 
2029  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2030  ``--debug`` flag. Some subtasks may have their own debug variables;
2031  see individual Task documentation.
2032 
2033  Examples
2034  --------
2035  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2036  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2037  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2038  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2039  and filter to be coadded (specified using
2040  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2041  along with a list of coaddTempExps to attempt to coadd (specified using
2042  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2043  Only the warps that cover the specified tract and patch will be coadded.
2044  A list of the available optional arguments can be obtained by calling
2045  ``assembleCoadd.py`` with the ``--help`` command line argument:
2046 
2047  .. code-block:: none
2048 
2049  assembleCoadd.py --help
2050 
2051  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2052  context of multi-band processing, we will generate the HSC-I & -R band
2053  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2054  To begin, assuming that the lsst stack has been already set up, we must
2055  set up the ``obs_subaru`` and ``ci_hsc`` packages.
2056  This defines the environment variable ``$CI_HSC_DIR`` and points at the
2057  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2058  directory. To begin assembling the coadds, we must first
2059 
2060  - processCcd
2061  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2062  - makeSkyMap
2063  create a skymap that covers the area of the sky present in the raw exposures
2064  - makeCoaddTempExp
2065  warp the individual calibrated exposures to the tangent plane of the coadd
2066 
2067  We can perform all of these steps by running
2068 
2069  .. code-block:: none
2070 
2071  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2072 
2073  This will produce warped ``coaddTempExps`` for each visit. To coadd the
2074  warped data, we call ``assembleCoadd.py`` as follows:
2075 
2076  .. code-block:: none
2077 
2078  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2079  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2080  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2081  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2082  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2083  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2084  --selectId visit=903988 ccd=24
2085 
2086  This will process the HSC-I band data. The results are written in
2087  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2088  """
2089  ConfigClass = CompareWarpAssembleCoaddConfig
2090  _DefaultName = "compareWarpAssembleCoadd"
2091 
2092  def __init__(self, *args, **kwargs):
2093  AssembleCoaddTask.__init__(self, *args, **kwargs)
2094  self.makeSubtask("assembleStaticSkyModel")
2095  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2096  self.makeSubtask("detect", schema=detectionSchema)
2097  if self.config.doPreserveContainedBySource:
2098  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2099  if self.config.doScaleWarpVariance:
2100  self.makeSubtask("scaleWarpVariance")
2101  if self.config.doFilterMorphological:
2102  self.makeSubtask("maskStreaks")
2103 
2104  @utils.inheritDoc(AssembleCoaddTask)
2105  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2106  """
2107  Generate a templateCoadd to use as a naive model of static sky to
2108  subtract from PSF-Matched warps.
2109 
2110  Returns
2111  -------
2112  result : `lsst.pipe.base.Struct`
2113  Result struct with components:
2114 
2115  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2116  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2117  """
2118  # Ensure that psfMatchedWarps are used as input warps for template generation
2119  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2120  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2121 
2122  # Because subtasks don't have connections we have to make one.
2123  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2124  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2125  if self.config.assembleStaticSkyModel.doWrite:
2126  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2127  # Remove template coadd from both subtask's and main tasks outputs,
2128  # because it is handled by the subtask as `coaddExposure`
2129  del outputRefs.templateCoadd
2130  del staticSkyModelOutputRefs.templateCoadd
2131 
2132  # A PSF-Matched nImage does not exist as a dataset type
2133  if 'nImage' in staticSkyModelOutputRefs.keys():
2134  del staticSkyModelOutputRefs.nImage
2135 
2136  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2137  staticSkyModelOutputRefs)
2138  if templateCoadd is None:
2139  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2140 
2141  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2142  nImage=templateCoadd.nImage,
2143  warpRefList=templateCoadd.warpRefList,
2144  imageScalerList=templateCoadd.imageScalerList,
2145  weightList=templateCoadd.weightList)
2146 
2147  @utils.inheritDoc(AssembleCoaddTask)
2148  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2149  """
2150  Generate a templateCoadd to use as a naive model of static sky to
2151  subtract from PSF-Matched warps.
2152 
2153  Returns
2154  -------
2155  result : `lsst.pipe.base.Struct`
2156  Result struct with components:
2157 
2158  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2159  - ``nImage``: N Image (``lsst.afw.image.Image``)
2160  """
2161  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2162  if templateCoadd is None:
2163  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2164 
2165  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2166  nImage=templateCoadd.nImage,
2167  warpRefList=templateCoadd.warpRefList,
2168  imageScalerList=templateCoadd.imageScalerList,
2169  weightList=templateCoadd.weightList)
2170 
2171  def _noTemplateMessage(self, warpType):
2172  warpName = (warpType[0].upper() + warpType[1:])
2173  message = """No %(warpName)s warps were found to build the template coadd which is
2174  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2175  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2176  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2177 
2178  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2179  another algorithm like:
2180 
2181  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2182  config.assemble.retarget(SafeClipAssembleCoaddTask)
2183  """ % {"warpName": warpName}
2184  return message
2185 
2186  @utils.inheritDoc(AssembleCoaddTask)
2187  @pipeBase.timeMethod
2188  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2189  supplementaryData, *args, **kwargs):
2190  """Assemble the coadd.
2191 
2192  Find artifacts and apply them to the warps' masks creating a list of
2193  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2194  plane. Then pass these alternative masks to the base class's `run`
2195  method.
2196 
2197  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2198  that must contain a ``templateCoadd`` that serves as the
2199  model of the static sky.
2200  """
2201 
2202  # Check and match the order of the supplementaryData
2203  # (PSF-matched) inputs to the order of the direct inputs,
2204  # so that the artifact mask is applied to the right warp
2205  dataIds = [ref.dataId for ref in tempExpRefList]
2206  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2207 
2208  if dataIds != psfMatchedDataIds:
2209  self.log.info("Reordering and or/padding PSF-matched visit input list")
2210  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2211  psfMatchedDataIds, dataIds)
2212  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2213  psfMatchedDataIds, dataIds)
2214 
2215  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2216  spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2217  supplementaryData.warpRefList,
2218  supplementaryData.imageScalerList)
2219 
2220  badMaskPlanes = self.config.badMaskPlanes[:]
2221  badMaskPlanes.append("CLIPPED")
2222  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2223 
2224  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2225  spanSetMaskList, mask=badPixelMask)
2226 
2227  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2228  # Psf-Matching moves the real edge inwards
2229  self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2230  return result
2231 
2232  def applyAltEdgeMask(self, mask, altMaskList):
2233  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2234 
2235  Parameters
2236  ----------
2237  mask : `lsst.afw.image.Mask`
2238  Original mask.
2239  altMaskList : `list`
2240  List of Dicts containing ``spanSet`` lists.
2241  Each element contains the new mask plane name (e.g. "CLIPPED
2242  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2243  the mask.
2244  """
2245  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2246  for visitMask in altMaskList:
2247  if "EDGE" in visitMask:
2248  for spanSet in visitMask['EDGE']:
2249  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2250 
2251  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2252  """Find artifacts.
2253 
2254  Loop through warps twice. The first loop builds a map with the count
2255  of how many epochs each pixel deviates from the templateCoadd by more
2256  than ``config.chiThreshold`` sigma. The second loop takes each
2257  difference image and filters the artifacts detected in each using
2258  count map to filter out variable sources and sources that are
2259  difficult to subtract cleanly.
2260 
2261  Parameters
2262  ----------
2263  templateCoadd : `lsst.afw.image.Exposure`
2264  Exposure to serve as model of static sky.
2265  tempExpRefList : `list`
2266  List of data references to warps.
2267  imageScalerList : `list`
2268  List of image scalers.
2269 
2270  Returns
2271  -------
2272  altMasks : `list`
2273  List of dicts containing information about CLIPPED
2274  (i.e., artifacts), NO_DATA, and EDGE pixels.
2275  """
2276 
2277  self.log.debug("Generating Count Image, and mask lists.")
2278  coaddBBox = templateCoadd.getBBox()
2279  slateIm = afwImage.ImageU(coaddBBox)
2280  epochCountImage = afwImage.ImageU(coaddBBox)
2281  nImage = afwImage.ImageU(coaddBBox)
2282  spanSetArtifactList = []
2283  spanSetNoDataMaskList = []
2284  spanSetEdgeList = []
2285  spanSetBadMorphoList = []
2286  badPixelMask = self.getBadPixelMask()
2287 
2288  # mask of the warp diffs should = that of only the warp
2289  templateCoadd.mask.clearAllMaskPlanes()
2290 
2291  if self.config.doPreserveContainedBySource:
2292  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2293  else:
2294  templateFootprints = None
2295 
2296  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2297  warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2298  if warpDiffExp is not None:
2299  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2300  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2301  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2302  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2303  fpSet.positive.merge(fpSet.negative)
2304  footprints = fpSet.positive
2305  slateIm.set(0)
2306  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2307 
2308  # Remove artifacts due to defects before they contribute to the epochCountImage
2309  if self.config.doPrefilterArtifacts:
2310  spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2311 
2312  # Clear mask before adding prefiltered spanSets
2313  self.detect.clearMask(warpDiffExp.mask)
2314  for spans in spanSetList:
2315  spans.setImage(slateIm, 1, doClip=True)
2316  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2317  epochCountImage += slateIm
2318 
2319  if self.config.doFilterMorphological:
2320  maskName = self.config.streakMaskName
2321  _ = self.maskStreaks.run(warpDiffExp)
2322  streakMask = warpDiffExp.mask
2323  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2324  streakMask.getPlaneBitMask(maskName)).split()
2325 
2326  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2327  # undergo a second convolution. Pixels with data in the direct warp
2328  # but not in the PSF-matched warp will not have their artifacts detected.
2329  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2330  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2331  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2332  nansMask.setXY0(warpDiffExp.getXY0())
2333  edgeMask = warpDiffExp.mask
2334  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2335  edgeMask.getPlaneBitMask("EDGE")).split()
2336  else:
2337  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2338  # In this case, mask the whole epoch
2339  nansMask = afwImage.MaskX(coaddBBox, 1)
2340  spanSetList = []
2341  spanSetEdgeMask = []
2342  spanSetStreak = []
2343 
2344  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2345 
2346  spanSetNoDataMaskList.append(spanSetNoDataMask)
2347  spanSetArtifactList.append(spanSetList)
2348  spanSetEdgeList.append(spanSetEdgeMask)
2349  if self.config.doFilterMorphological:
2350  spanSetBadMorphoList.append(spanSetStreak)
2351 
2352  if lsstDebug.Info(__name__).saveCountIm:
2353  path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2354  epochCountImage.writeFits(path)
2355 
2356  for i, spanSetList in enumerate(spanSetArtifactList):
2357  if spanSetList:
2358  filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2359  templateFootprints)
2360  spanSetArtifactList[i] = filteredSpanSetList
2361  if self.config.doFilterMorphological:
2362  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2363 
2364  altMasks = []
2365  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2366  altMasks.append({'CLIPPED': artifacts,
2367  'NO_DATA': noData,
2368  'EDGE': edge})
2369  return altMasks
2370 
2371  def prefilterArtifacts(self, spanSetList, exp):
2372  """Remove artifact candidates covered by bad mask plane.
2373 
2374  Any future editing of the candidate list that does not depend on
2375  temporal information should go in this method.
2376 
2377  Parameters
2378  ----------
2379  spanSetList : `list`
2380  List of SpanSets representing artifact candidates.
2381  exp : `lsst.afw.image.Exposure`
2382  Exposure containing mask planes used to prefilter.
2383 
2384  Returns
2385  -------
2386  returnSpanSetList : `list`
2387  List of SpanSets with artifacts.
2388  """
2389  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2390  goodArr = (exp.mask.array & badPixelMask) == 0
2391  returnSpanSetList = []
2392  bbox = exp.getBBox()
2393  x0, y0 = exp.getXY0()
2394  for i, span in enumerate(spanSetList):
2395  y, x = span.clippedTo(bbox).indices()
2396  yIndexLocal = numpy.array(y) - y0
2397  xIndexLocal = numpy.array(x) - x0
2398  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2399  if goodRatio > self.config.prefilterArtifactsRatio:
2400  returnSpanSetList.append(span)
2401  return returnSpanSetList
2402 
2403  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2404  """Filter artifact candidates.
2405 
2406  Parameters
2407  ----------
2408  spanSetList : `list`
2409  List of SpanSets representing artifact candidates.
2410  epochCountImage : `lsst.afw.image.Image`
2411  Image of accumulated number of warpDiff detections.
2412  nImage : `lsst.afw.image.Image`
2413  Image of the accumulated number of total epochs contributing.
2414 
2415  Returns
2416  -------
2417  maskSpanSetList : `list`
2418  List of SpanSets with artifacts.
2419  """
2420 
2421  maskSpanSetList = []
2422  x0, y0 = epochCountImage.getXY0()
2423  for i, span in enumerate(spanSetList):
2424  y, x = span.indices()
2425  yIdxLocal = [y1 - y0 for y1 in y]
2426  xIdxLocal = [x1 - x0 for x1 in x]
2427  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2428  totalN = nImage.array[yIdxLocal, xIdxLocal]
2429 
2430  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2431  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2432  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2433  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2434  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2435  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2436  & (outlierN <= effectiveMaxNumEpochs))
2437  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2438  if percentBelowThreshold > self.config.spatialThreshold:
2439  maskSpanSetList.append(span)
2440 
2441  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2442  # If a candidate is contained by a footprint on the template coadd, do not clip
2443  filteredMaskSpanSetList = []
2444  for span in maskSpanSetList:
2445  doKeep = True
2446  for footprint in footprintsToExclude.positive.getFootprints():
2447  if footprint.spans.contains(span):
2448  doKeep = False
2449  break
2450  if doKeep:
2451  filteredMaskSpanSetList.append(span)
2452  maskSpanSetList = filteredMaskSpanSetList
2453 
2454  return maskSpanSetList
2455 
2456  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2457  """Fetch a warp from the butler and return a warpDiff.
2458 
2459  Parameters
2460  ----------
2461  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2462  Butler dataRef for the warp.
2463  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2464  An image scaler object.
2465  templateCoadd : `lsst.afw.image.Exposure`
2466  Exposure to be substracted from the scaled warp.
2467 
2468  Returns
2469  -------
2470  warp : `lsst.afw.image.Exposure`
2471  Exposure of the image difference between the warp and template.
2472  """
2473 
2474  # If the PSF-Matched warp did not exist for this direct warp
2475  # None is holding its place to maintain order in Gen 3
2476  if warpRef is None:
2477  return None
2478  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2479  warpName = self.getTempExpDatasetName('psfMatched')
2480  if not isinstance(warpRef, DeferredDatasetHandle):
2481  if not warpRef.datasetExists(warpName):
2482  self.log.warning("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2483  return None
2484  warp = warpRef.get(datasetType=warpName, immediate=True)
2485  # direct image scaler OK for PSF-matched Warp
2486  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2487  mi = warp.getMaskedImage()
2488  if self.config.doScaleWarpVariance:
2489  try:
2490  self.scaleWarpVariance.run(mi)
2491  except Exception as exc:
2492  self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc)
2493  mi -= templateCoadd.getMaskedImage()
2494  return warp
2495 
2496  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2497  """Return a path to which to write debugging output.
2498 
2499  Creates a hyphen-delimited string of dataId values for simple filenames.
2500 
2501  Parameters
2502  ----------
2503  prefix : `str`
2504  Prefix for filename.
2505  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2506  Butler dataRef to make the path from.
2507  coaddLevel : `bool`, optional.
2508  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2509  'filter', but no 'visit').
2510 
2511  Returns
2512  -------
2513  result : `str`
2514  Path for debugging output.
2515  """
2516  if coaddLevel:
2517  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2518  else:
2519  keys = warpRef.dataId.keys()
2520  keyList = sorted(keys, reverse=True)
2521  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2522  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2523  return os.path.join(directory, filename)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:141
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:346
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60