lsst.pipe.tasks  21.0.0-119-g1a9a5132+d29897724a
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from .healSparseMapping import HealSparseInputMapTask
48 from lsst.meas.algorithms import SourceDetectionTask
49 from lsst.daf.butler import DeferredDatasetHandle
50 
51 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
52  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
53  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
54 
55 
56 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
57  dimensions=("tract", "patch", "band", "skymap"),
58  defaultTemplates={"inputCoaddName": "deep",
59  "outputCoaddName": "deep",
60  "warpType": "direct",
61  "warpTypeSuffix": ""}):
62 
63  inputWarps = pipeBase.connectionTypes.Input(
64  doc=("Input list of warps to be assemebled i.e. stacked."
65  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
66  name="{inputCoaddName}Coadd_{warpType}Warp",
67  storageClass="ExposureF",
68  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
69  deferLoad=True,
70  multiple=True
71  )
72  skyMap = pipeBase.connectionTypes.Input(
73  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
74  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75  storageClass="SkyMap",
76  dimensions=("skymap", ),
77  )
78  selectedVisits = pipeBase.connectionTypes.Input(
79  doc="Selected visits to be coadded.",
80  name="{outputCoaddName}Visits",
81  storageClass="StructuredDataDict",
82  dimensions=("instrument", "tract", "patch", "skymap", "band")
83  )
84  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
85  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
86  " BRIGHT_OBJECT."),
87  name="brightObjectMask",
88  storageClass="ObjectMaskCatalog",
89  dimensions=("tract", "patch", "skymap", "band"),
90  )
91  coaddExposure = pipeBase.connectionTypes.Output(
92  doc="Output coadded exposure, produced by stacking input warps",
93  name="{outputCoaddName}Coadd{warpTypeSuffix}",
94  storageClass="ExposureF",
95  dimensions=("tract", "patch", "skymap", "band"),
96  )
97  nImage = pipeBase.connectionTypes.Output(
98  doc="Output image of number of input images per pixel",
99  name="{outputCoaddName}Coadd_nImage",
100  storageClass="ImageU",
101  dimensions=("tract", "patch", "skymap", "band"),
102  )
103  inputMap = pipeBase.connectionTypes.Output(
104  doc="Output healsparse map of input images",
105  name="{outputCoaddName}Coadd_inputMap",
106  storageClass="HealSparseMap",
107  dimensions=("tract", "patch", "skymap", "band"),
108  )
109 
110  def __init__(self, *, config=None):
111  super().__init__(config=config)
112 
113  # Override the connection's name template with config to replicate Gen2 behavior
114  # This duplicates some of the logic in the base class, due to wanting Gen2 and
115  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
116  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
117  templateValues['warpType'] = config.warpType
118  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
119  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
120  for name in self.allConnections}
121  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
122  # End code to remove after deprecation
123 
124  if not config.doMaskBrightObjects:
125  self.prerequisiteInputs.remove("brightObjectMask")
126 
127  if not config.doSelectVisits:
128  self.inputs.remove("selectedVisits")
129 
130  if not config.doNImage:
131  self.outputs.remove("nImage")
132 
133  if not self.config.doInputMap:
134  self.outputs.remove("inputMap")
135 
136 
137 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
138  pipelineConnections=AssembleCoaddConnections):
139  """Configuration parameters for the `AssembleCoaddTask`.
140 
141  Notes
142  -----
143  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
144  only set the bitplane config.brightObjectMaskName. To make this useful you
145  *must* also configure the flags.pixel algorithm, for example by adding
146 
147  .. code-block:: none
148 
149  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
150  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
151 
152  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
153  """
154  warpType = pexConfig.Field(
155  doc="Warp name: one of 'direct' or 'psfMatched'",
156  dtype=str,
157  default="direct",
158  )
159  subregionSize = pexConfig.ListField(
160  dtype=int,
161  doc="Width, height of stack subregion size; "
162  "make small enough that a full stack of images will fit into memory at once.",
163  length=2,
164  default=(2000, 2000),
165  )
166  statistic = pexConfig.Field(
167  dtype=str,
168  doc="Main stacking statistic for aggregating over the epochs.",
169  default="MEANCLIP",
170  )
171  doSigmaClip = pexConfig.Field(
172  dtype=bool,
173  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
174  default=False,
175  )
176  sigmaClip = pexConfig.Field(
177  dtype=float,
178  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
179  default=3.0,
180  )
181  clipIter = pexConfig.Field(
182  dtype=int,
183  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
184  default=2,
185  )
186  calcErrorFromInputVariance = pexConfig.Field(
187  dtype=bool,
188  doc="Calculate coadd variance from input variance by stacking statistic."
189  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
190  default=True,
191  )
192  scaleZeroPoint = pexConfig.ConfigurableField(
193  target=ScaleZeroPointTask,
194  doc="Task to adjust the photometric zero point of the coadd temp exposures",
195  )
196  doInterp = pexConfig.Field(
197  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
198  dtype=bool,
199  default=True,
200  )
201  interpImage = pexConfig.ConfigurableField(
202  target=InterpImageTask,
203  doc="Task to interpolate (and extrapolate) over NaN pixels",
204  )
205  doWrite = pexConfig.Field(
206  doc="Persist coadd?",
207  dtype=bool,
208  default=True,
209  )
210  doNImage = pexConfig.Field(
211  doc="Create image of number of contributing exposures for each pixel",
212  dtype=bool,
213  default=False,
214  )
215  doUsePsfMatchedPolygons = pexConfig.Field(
216  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
217  dtype=bool,
218  default=False,
219  )
220  maskPropagationThresholds = pexConfig.DictField(
221  keytype=str,
222  itemtype=float,
223  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
224  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
225  "would have contributed exceeds this value."),
226  default={"SAT": 0.1},
227  )
228  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
229  doc="Mask planes to remove before coadding")
230  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
231  doc="Set mask and flag bits for bright objects?")
232  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
233  doc="Name of mask bit used for bright objects")
234  coaddPsf = pexConfig.ConfigField(
235  doc="Configuration for CoaddPsf",
236  dtype=measAlg.CoaddPsfConfig,
237  )
238  doAttachTransmissionCurve = pexConfig.Field(
239  dtype=bool, default=False, optional=False,
240  doc=("Attach a piecewise TransmissionCurve for the coadd? "
241  "(requires all input Exposures to have TransmissionCurves).")
242  )
243  hasFakes = pexConfig.Field(
244  dtype=bool,
245  default=False,
246  doc="Should be set to True if fake sources have been inserted into the input data."
247  )
248  doSelectVisits = pexConfig.Field(
249  doc="Coadd only visits selected by a SelectVisitsTask",
250  dtype=bool,
251  default=False,
252  )
253  doInputMap = pexConfig.Field(
254  doc="Create a bitwise map of coadd inputs",
255  dtype=bool,
256  default=False,
257  )
258  inputMapper = pexConfig.ConfigurableField(
259  doc="Input map creation subtask.",
260  target=HealSparseInputMapTask,
261  )
262 
263  def setDefaults(self):
264  super().setDefaults()
265  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
266 
267  def validate(self):
268  super().validate()
269  if self.doPsfMatch:
270  # Backwards compatibility.
271  # Configs do not have loggers
272  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
273  self.warpType = 'psfMatched'
274  if self.doSigmaClip and self.statistic != "MEANCLIP":
275  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
276  self.statistic = "MEANCLIP"
277  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
278  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
279  "compute and set a non-zero coadd variance estimate." % (self.statistic))
280 
281  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
282  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
283  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
284  if str(k) not in unstackableStats]
285  raise ValueError("statistic %s is not allowed. Please choose one of %s."
286  % (self.statistic, stackableStats))
287 
288 
289 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
290  """Assemble a coadded image from a set of warps (coadded temporary exposures).
291 
292  We want to assemble a coadded image from a set of Warps (also called
293  coadded temporary exposures or ``coaddTempExps``).
294  Each input Warp covers a patch on the sky and corresponds to a single
295  run/visit/exposure of the covered patch. We provide the task with a list
296  of Warps (``selectDataList``) from which it selects Warps that cover the
297  specified patch (pointed at by ``dataRef``).
298  Each Warp that goes into a coadd will typically have an independent
299  photometric zero-point. Therefore, we must scale each Warp to set it to
300  a common photometric zeropoint. WarpType may be one of 'direct' or
301  'psfMatched', and the boolean configs `config.makeDirect` and
302  `config.makePsfMatched` set which of the warp types will be coadded.
303  The coadd is computed as a mean with optional outlier rejection.
304  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
305  Finally, Warps can have bad 'NaN' pixels which received no input from the
306  source calExps. We interpolate over these bad (NaN) pixels.
307 
308  `AssembleCoaddTask` uses several sub-tasks. These are
309 
310  - `ScaleZeroPointTask`
311  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
312  - `InterpImageTask`
313  - interpolate across bad pixels (NaN) in the final coadd
314 
315  You can retarget these subtasks if you wish.
316 
317  Notes
318  -----
319  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
320  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
321  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
322  no debug variables of its own. Some of the subtasks may support debug
323  variables. See the documentation for the subtasks for further information.
324 
325  Examples
326  --------
327  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
328  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
329  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
330  inputs: a data reference to the tract patch and filter to be coadded, and
331  a list of Warps to attempt to coadd. These are specified using ``--id`` and
332  ``--selectId``, respectively:
333 
334  .. code-block:: none
335 
336  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
337  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
338 
339  Only the Warps that cover the specified tract and patch will be coadded.
340  A list of the available optional arguments can be obtained by calling
341  ``assembleCoadd.py`` with the ``--help`` command line argument:
342 
343  .. code-block:: none
344 
345  assembleCoadd.py --help
346 
347  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
348  multi-band processing, we will generate the HSC-I & -R band coadds from
349  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
350  assuming that the lsst stack has been already set up, we must set up the
351  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
352  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
353  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
354  coadds, we must first
355 
356  - processCcd
357  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
358  - makeSkyMap
359  - create a skymap that covers the area of the sky present in the raw exposures
360  - makeCoaddTempExp
361  - warp the individual calibrated exposures to the tangent plane of the coadd
362 
363  We can perform all of these steps by running
364 
365  .. code-block:: none
366 
367  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
368 
369  This will produce warped exposures for each visit. To coadd the warped
370  data, we call assembleCoadd.py as follows:
371 
372  .. code-block:: none
373 
374  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
375  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
376  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
377  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
378  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
379  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
380  --selectId visit=903988 ccd=24
381 
382  that will process the HSC-I band data. The results are written in
383  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
384 
385  You may also choose to run:
386 
387  .. code-block:: none
388 
389  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
390  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
391  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
392  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
393  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
394  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
395  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
396  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
397 
398  to generate the coadd for the HSC-R band if you are interested in
399  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
400  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
401  rather than `AssembleCoaddTask` to make the coadd.
402  """
403  ConfigClass = AssembleCoaddConfig
404  _DefaultName = "assembleCoadd"
405 
406  def __init__(self, *args, **kwargs):
407  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
408  if args:
409  argNames = ["config", "name", "parentTask", "log"]
410  kwargs.update({k: v for k, v in zip(argNames, args)})
411  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
412  "PipelineTask will not take positional args" % argNames, FutureWarning)
413 
414  super().__init__(**kwargs)
415  self.makeSubtask("interpImage")
416  self.makeSubtask("scaleZeroPoint")
417 
418  if self.config.doMaskBrightObjects:
419  mask = afwImage.Mask()
420  try:
421  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
422  except pexExceptions.LsstCppException:
423  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
424  mask.getMaskPlaneDict().keys())
425  del mask
426 
427  if self.config.doInputMap:
428  self.makeSubtask("inputMapper")
429 
430  self.warpType = self.config.warpType
431 
432  @utils.inheritDoc(pipeBase.PipelineTask)
433  def runQuantum(self, butlerQC, inputRefs, outputRefs):
434  # Docstring to be formatted with info from PipelineTask.runQuantum
435  """
436  Notes
437  -----
438  Assemble a coadd from a set of Warps.
439 
440  PipelineTask (Gen3) entry point to Coadd a set of Warps.
441  Analogous to `runDataRef`, it prepares all the data products to be
442  passed to `run`, and processes the results before returning a struct
443  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
444  Therefore, its inputs are accessed subregion by subregion
445  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
446  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
447  correspond to an update in `runDataRef` while both entry points
448  are used.
449  """
450  inputData = butlerQC.get(inputRefs)
451 
452  # Construct skyInfo expected by run
453  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
454  skyMap = inputData["skyMap"]
455  outputDataId = butlerQC.quantum.dataId
456 
457  inputData['skyInfo'] = makeSkyInfo(skyMap,
458  tractId=outputDataId['tract'],
459  patchId=outputDataId['patch'])
460 
461  if self.config.doSelectVisits:
462  warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
463  else:
464  warpRefList = inputData['inputWarps']
465 
466  # Perform same middle steps as `runDataRef` does
467  inputs = self.prepareInputs(warpRefList)
468  self.log.info("Found %d %s", len(inputs.tempExpRefList),
469  self.getTempExpDatasetName(self.warpType))
470  if len(inputs.tempExpRefList) == 0:
471  raise pipeBase.NoWorkFound("No coadd temporary exposures found")
472 
473  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
474  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
475  inputs.weightList, supplementaryData=supplementaryData)
476 
477  inputData.setdefault('brightObjectMask', None)
478  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
479 
480  if self.config.doWrite:
481  butlerQC.put(retStruct, outputRefs)
482  return retStruct
483 
484  @pipeBase.timeMethod
485  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
486  """Assemble a coadd from a set of Warps.
487 
488  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
489  Compute weights to be applied to each Warp and
490  find scalings to match the photometric zeropoint to a reference Warp.
491  Assemble the Warps using `run`. Interpolate over NaNs and
492  optionally write the coadd to disk. Return the coadded exposure.
493 
494  Parameters
495  ----------
496  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
497  Data reference defining the patch for coaddition and the
498  reference Warp (if ``config.autoReference=False``).
499  Used to access the following data products:
500  - ``self.config.coaddName + "Coadd_skyMap"``
501  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
502  - ``self.config.coaddName + "Coadd"``
503  selectDataList : `list`
504  List of data references to Calexps. Data to be coadded will be
505  selected from this list based on overlap with the patch defined
506  by dataRef, grouped by visit, and converted to a list of data
507  references to warps.
508  warpRefList : `list`
509  List of data references to Warps to be coadded.
510  Note: `warpRefList` is just the new name for `tempExpRefList`.
511 
512  Returns
513  -------
514  retStruct : `lsst.pipe.base.Struct`
515  Result struct with components:
516 
517  - ``coaddExposure``: coadded exposure (``Exposure``).
518  - ``nImage``: exposure count image (``Image``).
519  """
520  if selectDataList and warpRefList:
521  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
522  "and which to use is ambiguous. Please pass only one.")
523 
524  skyInfo = self.getSkyInfo(dataRef)
525  if warpRefList is None:
526  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
527  if len(calExpRefList) == 0:
528  self.log.warn("No exposures to coadd")
529  return
530  self.log.info("Coadding %d exposures", len(calExpRefList))
531 
532  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
533 
534  inputData = self.prepareInputs(warpRefList)
535  self.log.info("Found %d %s", len(inputData.tempExpRefList),
536  self.getTempExpDatasetName(self.warpType))
537  if len(inputData.tempExpRefList) == 0:
538  self.log.warn("No coadd temporary exposures found")
539  return
540 
541  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
542 
543  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
544  inputData.weightList, supplementaryData=supplementaryData)
545 
546  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
547  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
548 
549  if self.config.doWrite:
550  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
551  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
552  else:
553  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
554  self.log.info("Persisting %s" % coaddDatasetName)
555  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
556  if self.config.doNImage and retStruct.nImage is not None:
557  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
558 
559  return retStruct
560 
561  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
562  """Interpolate over missing data and mask bright stars.
563 
564  Parameters
565  ----------
566  coaddExposure : `lsst.afw.image.Exposure`
567  The coadded exposure to process.
568  dataRef : `lsst.daf.persistence.ButlerDataRef`
569  Butler data reference for supplementary data.
570  """
571  if self.config.doInterp:
572  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
573  # The variance must be positive; work around for DM-3201.
574  varArray = coaddExposure.variance.array
575  with numpy.errstate(invalid="ignore"):
576  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
577 
578  if self.config.doMaskBrightObjects:
579  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
580 
581  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
582  """Make additional inputs to run() specific to subclasses (Gen2)
583 
584  Duplicates interface of `runDataRef` method
585  Available to be implemented by subclasses only if they need the
586  coadd dataRef for performing preliminary processing before
587  assembling the coadd.
588 
589  Parameters
590  ----------
591  dataRef : `lsst.daf.persistence.ButlerDataRef`
592  Butler data reference for supplementary data.
593  selectDataList : `list` (optional)
594  Optional List of data references to Calexps.
595  warpRefList : `list` (optional)
596  Optional List of data references to Warps.
597  """
598  return pipeBase.Struct()
599 
600  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
601  """Make additional inputs to run() specific to subclasses (Gen3)
602 
603  Duplicates interface of `runQuantum` method.
604  Available to be implemented by subclasses only if they need the
605  coadd dataRef for performing preliminary processing before
606  assembling the coadd.
607 
608  Parameters
609  ----------
610  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
611  Gen3 Butler object for fetching additional data products before
612  running the Task specialized for quantum being processed
613  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
614  Attributes are the names of the connections describing input dataset types.
615  Values are DatasetRefs that task consumes for corresponding dataset type.
616  DataIds are guaranteed to match data objects in ``inputData``.
617  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
618  Attributes are the names of the connections describing output dataset types.
619  Values are DatasetRefs that task is to produce
620  for corresponding dataset type.
621  """
622  return pipeBase.Struct()
623 
624  def getTempExpRefList(self, patchRef, calExpRefList):
625  """Generate list data references corresponding to warped exposures
626  that lie within the patch to be coadded.
627 
628  Parameters
629  ----------
630  patchRef : `dataRef`
631  Data reference for patch.
632  calExpRefList : `list`
633  List of data references for input calexps.
634 
635  Returns
636  -------
637  tempExpRefList : `list`
638  List of Warp/CoaddTempExp data references.
639  """
640  butler = patchRef.getButler()
641  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
642  self.getTempExpDatasetName(self.warpType))
643  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
644  g, groupData.keys) for
645  g in groupData.groups.keys()]
646  return tempExpRefList
647 
648  def prepareInputs(self, refList):
649  """Prepare the input warps for coaddition by measuring the weight for
650  each warp and the scaling for the photometric zero point.
651 
652  Each Warp has its own photometric zeropoint and background variance.
653  Before coadding these Warps together, compute a scale factor to
654  normalize the photometric zeropoint and compute the weight for each Warp.
655 
656  Parameters
657  ----------
658  refList : `list`
659  List of data references to tempExp
660 
661  Returns
662  -------
663  result : `lsst.pipe.base.Struct`
664  Result struct with components:
665 
666  - ``tempExprefList``: `list` of data references to tempExp.
667  - ``weightList``: `list` of weightings.
668  - ``imageScalerList``: `list` of image scalers.
669  """
670  statsCtrl = afwMath.StatisticsControl()
671  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
672  statsCtrl.setNumIter(self.config.clipIter)
673  statsCtrl.setAndMask(self.getBadPixelMask())
674  statsCtrl.setNanSafe(True)
675  # compute tempExpRefList: a list of tempExpRef that actually exist
676  # and weightList: a list of the weight of the associated coadd tempExp
677  # and imageScalerList: a list of scale factors for the associated coadd tempExp
678  tempExpRefList = []
679  weightList = []
680  imageScalerList = []
681  tempExpName = self.getTempExpDatasetName(self.warpType)
682  for tempExpRef in refList:
683  # Gen3's DeferredDatasetHandles are guaranteed to exist and
684  # therefore have no datasetExists() method
685  if not isinstance(tempExpRef, DeferredDatasetHandle):
686  if not tempExpRef.datasetExists(tempExpName):
687  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
688  continue
689 
690  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
691  # Ignore any input warp that is empty of data
692  if numpy.isnan(tempExp.image.array).all():
693  continue
694  maskedImage = tempExp.getMaskedImage()
695  imageScaler = self.scaleZeroPoint.computeImageScaler(
696  exposure=tempExp,
697  dataRef=tempExpRef,
698  )
699  try:
700  imageScaler.scaleMaskedImage(maskedImage)
701  except Exception as e:
702  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
703  continue
704  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
705  afwMath.MEANCLIP, statsCtrl)
706  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
707  weight = 1.0 / float(meanVar)
708  if not numpy.isfinite(weight):
709  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
710  continue
711  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
712 
713  del maskedImage
714  del tempExp
715 
716  tempExpRefList.append(tempExpRef)
717  weightList.append(weight)
718  imageScalerList.append(imageScaler)
719 
720  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
721  imageScalerList=imageScalerList)
722 
723  def prepareStats(self, mask=None):
724  """Prepare the statistics for coadding images.
725 
726  Parameters
727  ----------
728  mask : `int`, optional
729  Bit mask value to exclude from coaddition.
730 
731  Returns
732  -------
733  stats : `lsst.pipe.base.Struct`
734  Statistics structure with the following fields:
735 
736  - ``statsCtrl``: Statistics control object for coadd
737  (`lsst.afw.math.StatisticsControl`)
738  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
739  """
740  if mask is None:
741  mask = self.getBadPixelMask()
742  statsCtrl = afwMath.StatisticsControl()
743  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
744  statsCtrl.setNumIter(self.config.clipIter)
745  statsCtrl.setAndMask(mask)
746  statsCtrl.setNanSafe(True)
747  statsCtrl.setWeighted(True)
748  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
749  for plane, threshold in self.config.maskPropagationThresholds.items():
750  bit = afwImage.Mask.getMaskPlane(plane)
751  statsCtrl.setMaskPropagationThreshold(bit, threshold)
752  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
753  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
754 
755  @pipeBase.timeMethod
756  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
757  altMaskList=None, mask=None, supplementaryData=None):
758  """Assemble a coadd from input warps
759 
760  Assemble the coadd using the provided list of coaddTempExps. Since
761  the full coadd covers a patch (a large area), the assembly is
762  performed over small areas on the image at a time in order to
763  conserve memory usage. Iterate over subregions within the outer
764  bbox of the patch using `assembleSubregion` to stack the corresponding
765  subregions from the coaddTempExps with the statistic specified.
766  Set the edge bits the coadd mask based on the weight map.
767 
768  Parameters
769  ----------
770  skyInfo : `lsst.pipe.base.Struct`
771  Struct with geometric information about the patch.
772  tempExpRefList : `list`
773  List of data references to Warps (previously called CoaddTempExps).
774  imageScalerList : `list`
775  List of image scalers.
776  weightList : `list`
777  List of weights
778  altMaskList : `list`, optional
779  List of alternate masks to use rather than those stored with
780  tempExp.
781  mask : `int`, optional
782  Bit mask value to exclude from coaddition.
783  supplementaryData : lsst.pipe.base.Struct, optional
784  Struct with additional data products needed to assemble coadd.
785  Only used by subclasses that implement `makeSupplementaryData`
786  and override `run`.
787 
788  Returns
789  -------
790  result : `lsst.pipe.base.Struct`
791  Result struct with components:
792 
793  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
794  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
795  - ``inputMap``: bit-wise map of inputs, if requested.
796  - ``warpRefList``: input list of refs to the warps (
797  ``lsst.daf.butler.DeferredDatasetHandle`` or
798  ``lsst.daf.persistence.ButlerDataRef``)
799  (unmodified)
800  - ``imageScalerList``: input list of image scalers (unmodified)
801  - ``weightList``: input list of weights (unmodified)
802  """
803  tempExpName = self.getTempExpDatasetName(self.warpType)
804  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
805  stats = self.prepareStats(mask=mask)
806 
807  if altMaskList is None:
808  altMaskList = [None]*len(tempExpRefList)
809 
810  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
811  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
812  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
813  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
814  coaddMaskedImage = coaddExposure.getMaskedImage()
815  subregionSizeArr = self.config.subregionSize
816  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
817  # if nImage is requested, create a zero one which can be passed to assembleSubregion
818  if self.config.doNImage:
819  nImage = afwImage.ImageU(skyInfo.bbox)
820  else:
821  nImage = None
822  # If inputMap is requested, create the initial version that can be masked in
823  # assembleSubregion.
824  if self.config.doInputMap:
825  self.inputMapper.build_ccd_input_map(skyInfo.bbox,
826  skyInfo.wcs,
827  coaddExposure.getInfo().getCoaddInputs().ccds)
828 
829  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
830  try:
831  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
832  weightList, altMaskList, stats.flags, stats.ctrl,
833  nImage=nImage)
834  except Exception as e:
835  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
836 
837  # If inputMap is requested, we must finalize the map after the accumulation.
838  if self.config.doInputMap:
839  self.inputMapper.finalize_ccd_input_map_mask()
840  inputMap = self.inputMapper.ccd_input_map
841  else:
842  inputMap = None
843 
844  self.setInexactPsf(coaddMaskedImage.getMask())
845  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
846  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
847  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
848  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
849  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
850  weightList=weightList, inputMap=inputMap)
851 
852  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
853  """Set the metadata for the coadd.
854 
855  This basic implementation sets the filter from the first input.
856 
857  Parameters
858  ----------
859  coaddExposure : `lsst.afw.image.Exposure`
860  The target exposure for the coadd.
861  tempExpRefList : `list`
862  List of data references to tempExp.
863  weightList : `list`
864  List of weights.
865  """
866  assert len(tempExpRefList) == len(weightList), "Length mismatch"
867  tempExpName = self.getTempExpDatasetName(self.warpType)
868  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
869  # (and we need more than just the PropertySet that contains the header), which is not possible
870  # with the current butler (see #2777).
871  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
872 
873  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
874  # Gen 3 API
875  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
876  else:
877  # Gen 2 API. Delete this when Gen 2 retired
878  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
879  for tempExpRef in tempExpRefList]
880  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
881 
882  # Set the coadd FilterLabel to the band of the first input exposure:
883  # Coadds are calibrated, so the physical label is now meaningless.
884  coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
885  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
886  coaddInputs.ccds.reserve(numCcds)
887  coaddInputs.visits.reserve(len(tempExpList))
888 
889  for tempExp, weight in zip(tempExpList, weightList):
890  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
891 
892  if self.config.doUsePsfMatchedPolygons:
893  self.shrinkValidPolygons(coaddInputs)
894 
895  coaddInputs.visits.sort()
896  if self.warpType == "psfMatched":
897  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
898  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
899  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
900  # having the maximum width (sufficient because square)
901  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
902  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
903  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
904  else:
905  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
906  self.config.coaddPsf.makeControl())
907  coaddExposure.setPsf(psf)
908  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
909  coaddExposure.getWcs())
910  coaddExposure.getInfo().setApCorrMap(apCorrMap)
911  if self.config.doAttachTransmissionCurve:
912  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
913  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
914 
915  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
916  altMaskList, statsFlags, statsCtrl, nImage=None):
917  """Assemble the coadd for a sub-region.
918 
919  For each coaddTempExp, check for (and swap in) an alternative mask
920  if one is passed. Remove mask planes listed in
921  `config.removeMaskPlanes`. Finally, stack the actual exposures using
922  `lsst.afw.math.statisticsStack` with the statistic specified by
923  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
924  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
925  an N-sigma clipped mean where N and iterations are specified by
926  statsCtrl. Assign the stacked subregion back to the coadd.
927 
928  Parameters
929  ----------
930  coaddExposure : `lsst.afw.image.Exposure`
931  The target exposure for the coadd.
932  bbox : `lsst.geom.Box`
933  Sub-region to coadd.
934  tempExpRefList : `list`
935  List of data reference to tempExp.
936  imageScalerList : `list`
937  List of image scalers.
938  weightList : `list`
939  List of weights.
940  altMaskList : `list`
941  List of alternate masks to use rather than those stored with
942  tempExp, or None. Each element is dict with keys = mask plane
943  name to which to add the spans.
944  statsFlags : `lsst.afw.math.Property`
945  Property object for statistic for coadd.
946  statsCtrl : `lsst.afw.math.StatisticsControl`
947  Statistics control object for coadd.
948  nImage : `lsst.afw.image.ImageU`, optional
949  Keeps track of exposure count for each pixel.
950  """
951  self.log.debug("Computing coadd over %s", bbox)
952  tempExpName = self.getTempExpDatasetName(self.warpType)
953  coaddExposure.mask.addMaskPlane("REJECTED")
954  coaddExposure.mask.addMaskPlane("CLIPPED")
955  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
956  maskMap = self.setRejectedMaskMapping(statsCtrl)
957  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
958  maskedImageList = []
959  if nImage is not None:
960  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
961  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
962 
963  if isinstance(tempExpRef, DeferredDatasetHandle):
964  # Gen 3 API
965  exposure = tempExpRef.get(parameters={'bbox': bbox})
966  else:
967  # Gen 2 API. Delete this when Gen 2 retired
968  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
969 
970  maskedImage = exposure.getMaskedImage()
971  mask = maskedImage.getMask()
972  if altMask is not None:
973  self.applyAltMaskPlanes(mask, altMask)
974  imageScaler.scaleMaskedImage(maskedImage)
975 
976  # Add 1 for each pixel which is not excluded by the exclude mask.
977  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
978  if nImage is not None:
979  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
980  if self.config.removeMaskPlanes:
981  self.removeMaskPlanes(maskedImage)
982  maskedImageList.append(maskedImage)
983 
984  if self.config.doInputMap:
985  visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
986  self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
987 
988  with self.timer("stack"):
989  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
990  clipped, # also set output to CLIPPED if sigma-clipped
991  maskMap)
992  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
993  if nImage is not None:
994  nImage.assign(subNImage, bbox)
995 
996  def removeMaskPlanes(self, maskedImage):
997  """Unset the mask of an image for mask planes specified in the config.
998 
999  Parameters
1000  ----------
1001  maskedImage : `lsst.afw.image.MaskedImage`
1002  The masked image to be modified.
1003  """
1004  mask = maskedImage.getMask()
1005  for maskPlane in self.config.removeMaskPlanes:
1006  try:
1007  mask &= ~mask.getPlaneBitMask(maskPlane)
1008  except pexExceptions.InvalidParameterError:
1009  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
1010  maskPlane)
1011 
1012  @staticmethod
1013  def setRejectedMaskMapping(statsCtrl):
1014  """Map certain mask planes of the warps to new planes for the coadd.
1015 
1016  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1017  or CLIPPED, set it to REJECTED on the coadd.
1018  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1019  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1020 
1021  Parameters
1022  ----------
1023  statsCtrl : `lsst.afw.math.StatisticsControl`
1024  Statistics control object for coadd
1025 
1026  Returns
1027  -------
1028  maskMap : `list` of `tuple` of `int`
1029  A list of mappings of mask planes of the warped exposures to
1030  mask planes of the coadd.
1031  """
1032  edge = afwImage.Mask.getPlaneBitMask("EDGE")
1033  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1034  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1035  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1036  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1037  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1038  (clipped, clipped)]
1039  return maskMap
1040 
1041  def applyAltMaskPlanes(self, mask, altMaskSpans):
1042  """Apply in place alt mask formatted as SpanSets to a mask.
1043 
1044  Parameters
1045  ----------
1046  mask : `lsst.afw.image.Mask`
1047  Original mask.
1048  altMaskSpans : `dict`
1049  SpanSet lists to apply. Each element contains the new mask
1050  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1051  and list of SpanSets to apply to the mask.
1052 
1053  Returns
1054  -------
1055  mask : `lsst.afw.image.Mask`
1056  Updated mask.
1057  """
1058  if self.config.doUsePsfMatchedPolygons:
1059  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1060  # Clear away any other masks outside the validPolygons. These pixels are no longer
1061  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1062  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1063  # This mask-clearing step must only occur *before* applying the new masks below
1064  for spanSet in altMaskSpans['NO_DATA']:
1065  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1066 
1067  for plane, spanSetList in altMaskSpans.items():
1068  maskClipValue = mask.addMaskPlane(plane)
1069  for spanSet in spanSetList:
1070  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1071  return mask
1072 
1073  def shrinkValidPolygons(self, coaddInputs):
1074  """Shrink coaddInputs' ccds' ValidPolygons in place.
1075 
1076  Either modify each ccd's validPolygon in place, or if CoaddInputs
1077  does not have a validPolygon, create one from its bbox.
1078 
1079  Parameters
1080  ----------
1081  coaddInputs : `lsst.afw.image.coaddInputs`
1082  Original mask.
1083 
1084  """
1085  for ccd in coaddInputs.ccds:
1086  polyOrig = ccd.getValidPolygon()
1087  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1088  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1089  if polyOrig:
1090  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1091  else:
1092  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1093  ccd.setValidPolygon(validPolygon)
1094 
1095  def readBrightObjectMasks(self, dataRef):
1096  """Retrieve the bright object masks.
1097 
1098  Returns None on failure.
1099 
1100  Parameters
1101  ----------
1102  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1103  A Butler dataRef.
1104 
1105  Returns
1106  -------
1107  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1108  Bright object mask from the Butler object, or None if it cannot
1109  be retrieved.
1110  """
1111  try:
1112  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1113  except Exception as e:
1114  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1115  return None
1116 
1117  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1118  """Set the bright object masks.
1119 
1120  Parameters
1121  ----------
1122  exposure : `lsst.afw.image.Exposure`
1123  Exposure under consideration.
1124  dataId : `lsst.daf.persistence.dataId`
1125  Data identifier dict for patch.
1126  brightObjectMasks : `lsst.afw.table`
1127  Table of bright objects to mask.
1128  """
1129 
1130  if brightObjectMasks is None:
1131  self.log.warn("Unable to apply bright object mask: none supplied")
1132  return
1133  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1134  mask = exposure.getMaskedImage().getMask()
1135  wcs = exposure.getWcs()
1136  plateScale = wcs.getPixelScale().asArcseconds()
1137 
1138  for rec in brightObjectMasks:
1139  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1140  if rec["type"] == "box":
1141  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1142  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1143  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1144 
1145  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1146  bbox = geom.Box2I(center - halfSize, center + halfSize)
1147 
1148  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1149  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1150  spans = afwGeom.SpanSet(bbox)
1151  elif rec["type"] == "circle":
1152  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1153  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1154  else:
1155  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1156  continue
1157  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1158 
1159  def setInexactPsf(self, mask):
1160  """Set INEXACT_PSF mask plane.
1161 
1162  If any of the input images isn't represented in the coadd (due to
1163  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1164  these pixels.
1165 
1166  Parameters
1167  ----------
1168  mask : `lsst.afw.image.Mask`
1169  Coadded exposure's mask, modified in-place.
1170  """
1171  mask.addMaskPlane("INEXACT_PSF")
1172  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1173  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1174  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1175  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1176  array = mask.getArray()
1177  selected = array & (sensorEdge | clipped | rejected) > 0
1178  array[selected] |= inexactPsf
1179 
1180  @classmethod
1181  def _makeArgumentParser(cls):
1182  """Create an argument parser.
1183  """
1184  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1185  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1186  + cls.ConfigClass().warpType + "Warp",
1187  help="data ID, e.g. --id tract=12345 patch=1,2",
1188  ContainerClass=AssembleCoaddDataIdContainer)
1189  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1190  ContainerClass=SelectDataIdContainer)
1191  return parser
1192 
1193  @staticmethod
1194  def _subBBoxIter(bbox, subregionSize):
1195  """Iterate over subregions of a bbox.
1196 
1197  Parameters
1198  ----------
1199  bbox : `lsst.geom.Box2I`
1200  Bounding box over which to iterate.
1201  subregionSize: `lsst.geom.Extent2I`
1202  Size of sub-bboxes.
1203 
1204  Yields
1205  ------
1206  subBBox : `lsst.geom.Box2I`
1207  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1208  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1209  the edges of ``bbox``, but it will never be empty.
1210  """
1211  if bbox.isEmpty():
1212  raise RuntimeError("bbox %s is empty" % (bbox,))
1213  if subregionSize[0] < 1 or subregionSize[1] < 1:
1214  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1215 
1216  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1217  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1218  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1219  subBBox.clip(bbox)
1220  if subBBox.isEmpty():
1221  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1222  "colShift=%s, rowShift=%s" %
1223  (bbox, subregionSize, colShift, rowShift))
1224  yield subBBox
1225 
1226  def filterWarps(self, inputs, goodVisits):
1227  """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1228 
1229  Parameters
1230  ----------
1231  inputs : list
1232  List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1233  goodVisit : `dict`
1234  Dictionary with good visitIds as the keys. Value ignored.
1235 
1236  Returns:
1237  --------
1238  filteredInputs : `list`
1239  Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1240  """
1241  inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1242  filteredInputs = []
1243  for visit in goodVisits.keys():
1244  filteredInputs.append(inputWarpDict[visit])
1245  return filteredInputs
1246 
1247 
1248 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1249  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1250  """
1251 
1252  def makeDataRefList(self, namespace):
1253  """Make self.refList from self.idList.
1254 
1255  Parameters
1256  ----------
1257  namespace
1258  Results of parsing command-line (with ``butler`` and ``log`` elements).
1259  """
1260  datasetType = namespace.config.coaddName + "Coadd"
1261  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1262 
1263  for dataId in self.idList:
1264  # tract and patch are required
1265  for key in keysCoadd:
1266  if key not in dataId:
1267  raise RuntimeError("--id must include " + key)
1268 
1269  dataRef = namespace.butler.dataRef(
1270  datasetType=datasetType,
1271  dataId=dataId,
1272  )
1273  self.refList.append(dataRef)
1274 
1275 
1276 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1277  """Function to count the number of pixels with a specific mask in a
1278  footprint.
1279 
1280  Find the intersection of mask & footprint. Count all pixels in the mask
1281  that are in the intersection that have bitmask set but do not have
1282  ignoreMask set. Return the count.
1283 
1284  Parameters
1285  ----------
1286  mask : `lsst.afw.image.Mask`
1287  Mask to define intersection region by.
1288  footprint : `lsst.afw.detection.Footprint`
1289  Footprint to define the intersection region by.
1290  bitmask
1291  Specific mask that we wish to count the number of occurances of.
1292  ignoreMask
1293  Pixels to not consider.
1294 
1295  Returns
1296  -------
1297  result : `int`
1298  Count of number of pixels in footprint with specified mask.
1299  """
1300  bbox = footprint.getBBox()
1301  bbox.clip(mask.getBBox(afwImage.PARENT))
1302  fp = afwImage.Mask(bbox)
1303  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1304  footprint.spans.setMask(fp, bitmask)
1305  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1306  (subMask.getArray() & ignoreMask) == 0).sum()
1307 
1308 
1309 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1310  """Configuration parameters for the SafeClipAssembleCoaddTask.
1311  """
1312  clipDetection = pexConfig.ConfigurableField(
1313  target=SourceDetectionTask,
1314  doc="Detect sources on difference between unclipped and clipped coadd")
1315  minClipFootOverlap = pexConfig.Field(
1316  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1317  dtype=float,
1318  default=0.6
1319  )
1320  minClipFootOverlapSingle = pexConfig.Field(
1321  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1322  "clipped when only one visit overlaps",
1323  dtype=float,
1324  default=0.5
1325  )
1326  minClipFootOverlapDouble = pexConfig.Field(
1327  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1328  "clipped when two visits overlap",
1329  dtype=float,
1330  default=0.45
1331  )
1332  maxClipFootOverlapDouble = pexConfig.Field(
1333  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1334  "considering two visits",
1335  dtype=float,
1336  default=0.15
1337  )
1338  minBigOverlap = pexConfig.Field(
1339  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1340  "when labeling clipped footprints",
1341  dtype=int,
1342  default=100
1343  )
1344 
1345  def setDefaults(self):
1346  """Set default values for clipDetection.
1347 
1348  Notes
1349  -----
1350  The numeric values for these configuration parameters were
1351  empirically determined, future work may further refine them.
1352  """
1353  AssembleCoaddConfig.setDefaults(self)
1354  self.clipDetectionclipDetection.doTempLocalBackground = False
1355  self.clipDetectionclipDetection.reEstimateBackground = False
1356  self.clipDetectionclipDetection.returnOriginalFootprints = False
1357  self.clipDetectionclipDetection.thresholdPolarity = "both"
1358  self.clipDetectionclipDetection.thresholdValue = 2
1359  self.clipDetectionclipDetection.nSigmaToGrow = 2
1360  self.clipDetectionclipDetection.minPixels = 4
1361  self.clipDetectionclipDetection.isotropicGrow = True
1362  self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1363  self.sigmaClipsigmaClip = 1.5
1364  self.clipIterclipIter = 3
1365  self.statisticstatistic = "MEAN"
1366 
1367  def validate(self):
1368  if self.doSigmaClipdoSigmaClip:
1369  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1370  "Ignoring doSigmaClip.")
1371  self.doSigmaClipdoSigmaClip = False
1372  if self.statisticstatistic != "MEAN":
1373  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1374  "(%s chosen). Please set statistic to MEAN."
1375  % (self.statisticstatistic))
1376  AssembleCoaddTask.ConfigClass.validate(self)
1377 
1378 
1379 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1380  """Assemble a coadded image from a set of coadded temporary exposures,
1381  being careful to clip & flag areas with potential artifacts.
1382 
1383  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1384  we clip outliers). The problem with doing this is that when computing the
1385  coadd PSF at a given location, individual visit PSFs from visits with
1386  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1387  In this task, we correct for this behavior by creating a new
1388  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1389  coaddTempExps and the final coadd where
1390 
1391  i. difference imaging suggests that there is an outlier and
1392  ii. this outlier appears on only one or two images.
1393 
1394  Such regions will not contribute to the final coadd. Furthermore, any
1395  routine to determine the coadd PSF can now be cognizant of clipped regions.
1396  Note that the algorithm implemented by this task is preliminary and works
1397  correctly for HSC data. Parameter modifications and or considerable
1398  redesigning of the algorithm is likley required for other surveys.
1399 
1400  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1401  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1402  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1403  if you wish.
1404 
1405  Notes
1406  -----
1407  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1408  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1409  see `baseDebug` for more about ``debug.py`` files.
1410  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1411  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1412  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1413  for further information.
1414 
1415  Examples
1416  --------
1417  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1418  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1419  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1420 
1421  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1422  and filter to be coadded (specified using
1423  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1424  along with a list of coaddTempExps to attempt to coadd (specified using
1425  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1426  Only the coaddTempExps that cover the specified tract and patch will be
1427  coadded. A list of the available optional arguments can be obtained by
1428  calling assembleCoadd.py with the --help command line argument:
1429 
1430  .. code-block:: none
1431 
1432  assembleCoadd.py --help
1433 
1434  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1435  context of multi-band processing, we will generate the HSC-I & -R band
1436  coadds from HSC engineering test data provided in the ci_hsc package.
1437  To begin, assuming that the lsst stack has been already set up, we must
1438  set up the obs_subaru and ci_hsc packages. This defines the environment
1439  variable $CI_HSC_DIR and points at the location of the package. The raw
1440  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1441  the coadds, we must first
1442 
1443  - ``processCcd``
1444  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1445  - ``makeSkyMap``
1446  create a skymap that covers the area of the sky present in the raw exposures
1447  - ``makeCoaddTempExp``
1448  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1449 
1450  We can perform all of these steps by running
1451 
1452  .. code-block:: none
1453 
1454  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1455 
1456  This will produce warped coaddTempExps for each visit. To coadd the
1457  warped data, we call ``assembleCoadd.py`` as follows:
1458 
1459  .. code-block:: none
1460 
1461  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1462  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1463  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1464  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1465  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1466  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1467  --selectId visit=903988 ccd=24
1468 
1469  This will process the HSC-I band data. The results are written in
1470  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1471 
1472  You may also choose to run:
1473 
1474  .. code-block:: none
1475 
1476  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1477  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1478  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1479  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1480  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1481  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1482  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1483  --selectId visit=903346 ccd=12
1484 
1485  to generate the coadd for the HSC-R band if you are interested in following
1486  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1487  """
1488  ConfigClass = SafeClipAssembleCoaddConfig
1489  _DefaultName = "safeClipAssembleCoadd"
1490 
1491  def __init__(self, *args, **kwargs):
1492  AssembleCoaddTask.__init__(self, *args, **kwargs)
1493  schema = afwTable.SourceTable.makeMinimalSchema()
1494  self.makeSubtask("clipDetection", schema=schema)
1495 
1496  @utils.inheritDoc(AssembleCoaddTask)
1497  @pipeBase.timeMethod
1498  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1499  """Assemble the coadd for a region.
1500 
1501  Compute the difference of coadds created with and without outlier
1502  rejection to identify coadd pixels that have outlier values in some
1503  individual visits.
1504  Detect clipped regions on the difference image and mark these regions
1505  on the one or two individual coaddTempExps where they occur if there
1506  is significant overlap between the clipped region and a source. This
1507  leaves us with a set of footprints from the difference image that have
1508  been identified as having occured on just one or two individual visits.
1509  However, these footprints were generated from a difference image. It
1510  is conceivable for a large diffuse source to have become broken up
1511  into multiple footprints acrosss the coadd difference in this process.
1512  Determine the clipped region from all overlapping footprints from the
1513  detected sources in each visit - these are big footprints.
1514  Combine the small and big clipped footprints and mark them on a new
1515  bad mask plane.
1516  Generate the coadd using `AssembleCoaddTask.run` without outlier
1517  removal. Clipped footprints will no longer make it into the coadd
1518  because they are marked in the new bad mask plane.
1519 
1520  Notes
1521  -----
1522  args and kwargs are passed but ignored in order to match the call
1523  signature expected by the parent task.
1524  """
1525  exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1526  mask = exp.getMaskedImage().getMask()
1527  mask.addMaskPlane("CLIPPED")
1528 
1529  result = self.detectClipdetectClip(exp, tempExpRefList)
1530 
1531  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1532 
1533  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1534  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1535  # Append big footprints from individual Warps to result.clipSpans
1536  bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1537  result.detectionFootprints, maskClipValue, maskDetValue,
1538  exp.getBBox())
1539  # Create mask of the current clipped footprints
1540  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1541  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1542 
1543  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1544  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1545  maskClip |= maskClipBig
1546 
1547  # Assemble coadd from base class, but ignoring CLIPPED pixels
1548  badMaskPlanes = self.config.badMaskPlanes[:]
1549  badMaskPlanes.append("CLIPPED")
1550  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1551  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1552  result.clipSpans, mask=badPixelMask)
1553 
1554  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1555  """Return an exposure that contains the difference between unclipped
1556  and clipped coadds.
1557 
1558  Generate a difference image between clipped and unclipped coadds.
1559  Compute the difference image by subtracting an outlier-clipped coadd
1560  from an outlier-unclipped coadd. Return the difference image.
1561 
1562  Parameters
1563  ----------
1564  skyInfo : `lsst.pipe.base.Struct`
1565  Patch geometry information, from getSkyInfo
1566  tempExpRefList : `list`
1567  List of data reference to tempExp
1568  imageScalerList : `list`
1569  List of image scalers
1570  weightList : `list`
1571  List of weights
1572 
1573  Returns
1574  -------
1575  exp : `lsst.afw.image.Exposure`
1576  Difference image of unclipped and clipped coadd wrapped in an Exposure
1577  """
1578  config = AssembleCoaddConfig()
1579  # getattr necessary because subtasks do not survive Config.toDict()
1580  # exclude connections because the class of self.config.connections is not
1581  # the same as AssembleCoaddConfig.connections, and the connections are not
1582  # needed to run this task anyway.
1583  configIntersection = {k: getattr(self.config, k)
1584  for k, v in self.config.toDict().items()
1585  if (k in config.keys() and k != "connections")}
1586  configIntersection['doInputMap'] = False
1587  configIntersection['doNImage'] = False
1588  config.update(**configIntersection)
1589 
1590  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1591  config.statistic = 'MEAN'
1592  task = AssembleCoaddTask(config=config)
1593  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1594 
1595  config.statistic = 'MEANCLIP'
1596  task = AssembleCoaddTask(config=config)
1597  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1598 
1599  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1600  coaddDiff -= coaddClip.getMaskedImage()
1601  exp = afwImage.ExposureF(coaddDiff)
1602  exp.setPsf(coaddMean.getPsf())
1603  return exp
1604 
1605  def detectClip(self, exp, tempExpRefList):
1606  """Detect clipped regions on an exposure and set the mask on the
1607  individual tempExp masks.
1608 
1609  Detect footprints in the difference image after smoothing the
1610  difference image with a Gaussian kernal. Identify footprints that
1611  overlap with one or two input ``coaddTempExps`` by comparing the
1612  computed overlap fraction to thresholds set in the config. A different
1613  threshold is applied depending on the number of overlapping visits
1614  (restricted to one or two). If the overlap exceeds the thresholds,
1615  the footprint is considered "CLIPPED" and is marked as such on the
1616  coaddTempExp. Return a struct with the clipped footprints, the indices
1617  of the ``coaddTempExps`` that end up overlapping with the clipped
1618  footprints, and a list of new masks for the ``coaddTempExps``.
1619 
1620  Parameters
1621  ----------
1622  exp : `lsst.afw.image.Exposure`
1623  Exposure to run detection on.
1624  tempExpRefList : `list`
1625  List of data reference to tempExp.
1626 
1627  Returns
1628  -------
1629  result : `lsst.pipe.base.Struct`
1630  Result struct with components:
1631 
1632  - ``clipFootprints``: list of clipped footprints.
1633  - ``clipIndices``: indices for each ``clippedFootprint`` in
1634  ``tempExpRefList``.
1635  - ``clipSpans``: List of dictionaries containing spanSet lists
1636  to clip. Each element contains the new maskplane name
1637  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1638  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1639  compressed into footprints.
1640  """
1641  mask = exp.getMaskedImage().getMask()
1642  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1643  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1644  # Merge positive and negative together footprints together
1645  fpSet.positive.merge(fpSet.negative)
1646  footprints = fpSet.positive
1647  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1648  ignoreMask = self.getBadPixelMask()
1649 
1650  clipFootprints = []
1651  clipIndices = []
1652  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1653 
1654  # for use by detectClipBig
1655  visitDetectionFootprints = []
1656 
1657  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1658  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1659  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1660 
1661  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1662  for i, warpRef in enumerate(tempExpRefList):
1663  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1664  immediate=True).getMaskedImage().getMask()
1665  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1666  afwImage.PARENT, True)
1667  maskVisitDet &= maskDetValue
1668  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1669  visitDetectionFootprints.append(visitFootprints)
1670 
1671  for j, footprint in enumerate(footprints.getFootprints()):
1672  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1673  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1674 
1675  # build a list of clipped spans for each visit
1676  for j, footprint in enumerate(footprints.getFootprints()):
1677  nPixel = footprint.getArea()
1678  overlap = [] # hold the overlap with each visit
1679  indexList = [] # index of visit in global list
1680  for i in range(len(tempExpRefList)):
1681  ignore = ignoreArr[i, j]
1682  overlapDet = overlapDetArr[i, j]
1683  totPixel = nPixel - ignore
1684 
1685  # If we have more bad pixels than detection skip
1686  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1687  continue
1688  overlap.append(overlapDet/float(totPixel))
1689  indexList.append(i)
1690 
1691  overlap = numpy.array(overlap)
1692  if not len(overlap):
1693  continue
1694 
1695  keep = False # Should this footprint be marked as clipped?
1696  keepIndex = [] # Which tempExps does the clipped footprint belong to
1697 
1698  # If footprint only has one overlap use a lower threshold
1699  if len(overlap) == 1:
1700  if overlap[0] > self.config.minClipFootOverlapSingle:
1701  keep = True
1702  keepIndex = [0]
1703  else:
1704  # This is the general case where only visit should be clipped
1705  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1706  if len(clipIndex) == 1:
1707  keep = True
1708  keepIndex = [clipIndex[0]]
1709 
1710  # Test if there are clipped objects that overlap two different visits
1711  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1712  if len(clipIndex) == 2 and len(overlap) > 3:
1713  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1714  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1715  keep = True
1716  keepIndex = clipIndex
1717 
1718  if not keep:
1719  continue
1720 
1721  for index in keepIndex:
1722  globalIndex = indexList[index]
1723  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1724 
1725  clipIndices.append(numpy.array(indexList)[keepIndex])
1726  clipFootprints.append(footprint)
1727 
1728  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1729  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1730 
1731  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1732  maskClipValue, maskDetValue, coaddBBox):
1733  """Return individual warp footprints for large artifacts and append
1734  them to ``clipList`` in place.
1735 
1736  Identify big footprints composed of many sources in the coadd
1737  difference that may have originated in a large diffuse source in the
1738  coadd. We do this by indentifying all clipped footprints that overlap
1739  significantly with each source in all the coaddTempExps.
1740 
1741  Parameters
1742  ----------
1743  clipList : `list`
1744  List of alt mask SpanSets with clipping information. Modified.
1745  clipFootprints : `list`
1746  List of clipped footprints.
1747  clipIndices : `list`
1748  List of which entries in tempExpClipList each footprint belongs to.
1749  maskClipValue
1750  Mask value of clipped pixels.
1751  maskDetValue
1752  Mask value of detected pixels.
1753  coaddBBox : `lsst.geom.Box`
1754  BBox of the coadd and warps.
1755 
1756  Returns
1757  -------
1758  bigFootprintsCoadd : `list`
1759  List of big footprints
1760  """
1761  bigFootprintsCoadd = []
1762  ignoreMask = self.getBadPixelMask()
1763  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1764  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1765  for footprint in visitFootprints.getFootprints():
1766  footprint.spans.setMask(maskVisitDet, maskDetValue)
1767 
1768  # build a mask of clipped footprints that are in this visit
1769  clippedFootprintsVisit = []
1770  for foot, clipIndex in zip(clipFootprints, clipIndices):
1771  if index not in clipIndex:
1772  continue
1773  clippedFootprintsVisit.append(foot)
1774  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1775  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1776 
1777  bigFootprintsVisit = []
1778  for foot in visitFootprints.getFootprints():
1779  if foot.getArea() < self.config.minBigOverlap:
1780  continue
1781  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1782  if nCount > self.config.minBigOverlap:
1783  bigFootprintsVisit.append(foot)
1784  bigFootprintsCoadd.append(foot)
1785 
1786  for footprint in bigFootprintsVisit:
1787  clippedSpans["CLIPPED"].append(footprint.spans)
1788 
1789  return bigFootprintsCoadd
1790 
1791 
1793  psfMatchedWarps = pipeBase.connectionTypes.Input(
1794  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1795  "Only PSF-Matched Warps make sense for image subtraction. "
1796  "Therefore, they must be an additional declared input."),
1797  name="{inputCoaddName}Coadd_psfMatchedWarp",
1798  storageClass="ExposureF",
1799  dimensions=("tract", "patch", "skymap", "visit"),
1800  deferLoad=True,
1801  multiple=True
1802  )
1803  templateCoadd = pipeBase.connectionTypes.Output(
1804  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1805  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1806  name="{outputCoaddName}CoaddPsfMatched",
1807  storageClass="ExposureF",
1808  dimensions=("tract", "patch", "skymap", "band"),
1809  )
1810 
1811  def __init__(self, *, config=None):
1812  super().__init__(config=config)
1813  if not config.assembleStaticSkyModel.doWrite:
1814  self.outputs.remove("templateCoadd")
1815  config.validate()
1816 
1817 
1818 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1819  pipelineConnections=CompareWarpAssembleCoaddConnections):
1820  assembleStaticSkyModel = pexConfig.ConfigurableField(
1821  target=AssembleCoaddTask,
1822  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1823  " naive/first-iteration model of the static sky.",
1824  )
1825  detect = pexConfig.ConfigurableField(
1826  target=SourceDetectionTask,
1827  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1828  )
1829  detectTemplate = pexConfig.ConfigurableField(
1830  target=SourceDetectionTask,
1831  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1832  )
1833  maskStreaks = pexConfig.ConfigurableField(
1834  target=MaskStreaksTask,
1835  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1836  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1837  "streakMaskName"
1838  )
1839  streakMaskName = pexConfig.Field(
1840  dtype=str,
1841  default="STREAK",
1842  doc="Name of mask bit used for streaks"
1843  )
1844  maxNumEpochs = pexConfig.Field(
1845  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1846  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1847  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1848  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1849  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1850  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1851  "than transient and not masked.",
1852  dtype=int,
1853  default=2
1854  )
1855  maxFractionEpochsLow = pexConfig.RangeField(
1856  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1857  "Effective maxNumEpochs = "
1858  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1859  dtype=float,
1860  default=0.4,
1861  min=0., max=1.,
1862  )
1863  maxFractionEpochsHigh = pexConfig.RangeField(
1864  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1865  "Effective maxNumEpochs = "
1866  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1867  dtype=float,
1868  default=0.03,
1869  min=0., max=1.,
1870  )
1871  spatialThreshold = pexConfig.RangeField(
1872  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1873  "temporal criteria. If 0, clip all. If 1, clip none.",
1874  dtype=float,
1875  default=0.5,
1876  min=0., max=1.,
1877  inclusiveMin=True, inclusiveMax=True
1878  )
1879  doScaleWarpVariance = pexConfig.Field(
1880  doc="Rescale Warp variance plane using empirical noise?",
1881  dtype=bool,
1882  default=True,
1883  )
1884  scaleWarpVariance = pexConfig.ConfigurableField(
1885  target=ScaleVarianceTask,
1886  doc="Rescale variance on warps",
1887  )
1888  doPreserveContainedBySource = pexConfig.Field(
1889  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1890  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1891  dtype=bool,
1892  default=True,
1893  )
1894  doPrefilterArtifacts = pexConfig.Field(
1895  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1896  "because they will be excluded anyway. This prevents them from contributing "
1897  "to the outlier epoch count image and potentially being labeled as persistant."
1898  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1899  dtype=bool,
1900  default=True
1901  )
1902  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1903  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1904  dtype=str,
1905  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1906  )
1907  prefilterArtifactsRatio = pexConfig.Field(
1908  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1909  dtype=float,
1910  default=0.05
1911  )
1912  doFilterMorphological = pexConfig.Field(
1913  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1914  "be streaks.",
1915  dtype=bool,
1916  default=False
1917  )
1918 
1919  def setDefaults(self):
1920  AssembleCoaddConfig.setDefaults(self)
1921  self.statisticstatistic = 'MEAN'
1922  self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
1923 
1924  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1925  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1926  if "EDGE" in self.badMaskPlanes:
1927  self.badMaskPlanes.remove('EDGE')
1928  self.removeMaskPlanes.append('EDGE')
1929  self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1930  self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
1931  self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
1932  self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
1933  self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
1934  self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
1935  self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
1936  self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
1937  self.detectdetect.doTempLocalBackground = False
1938  self.detectdetect.reEstimateBackground = False
1939  self.detectdetect.returnOriginalFootprints = False
1940  self.detectdetect.thresholdPolarity = "both"
1941  self.detectdetect.thresholdValue = 5
1942  self.detectdetect.minPixels = 4
1943  self.detectdetect.isotropicGrow = True
1944  self.detectdetect.thresholdType = "pixel_stdev"
1945  self.detectdetect.nSigmaToGrow = 0.4
1946  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1947  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1948  self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
1949  self.detectTemplatedetectTemplate.doTempLocalBackground = False
1950  self.detectTemplatedetectTemplate.reEstimateBackground = False
1951  self.detectTemplatedetectTemplate.returnOriginalFootprints = False
1952 
1953  def validate(self):
1954  super().validate()
1955  if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
1956  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1957  "Please set assembleStaticSkyModel.doNImage=False")
1958 
1959  if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
1960  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1961  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1962  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1963  "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
1964 
1965 
1966 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1967  """Assemble a compareWarp coadded image from a set of warps
1968  by masking artifacts detected by comparing PSF-matched warps.
1969 
1970  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1971  we clip outliers). The problem with doing this is that when computing the
1972  coadd PSF at a given location, individual visit PSFs from visits with
1973  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1974  In this task, we correct for this behavior by creating a new badMaskPlane
1975  'CLIPPED' which marks pixels in the individual warps suspected to contain
1976  an artifact. We populate this plane on the input warps by comparing
1977  PSF-matched warps with a PSF-matched median coadd which serves as a
1978  model of the static sky. Any group of pixels that deviates from the
1979  PSF-matched template coadd by more than config.detect.threshold sigma,
1980  is an artifact candidate. The candidates are then filtered to remove
1981  variable sources and sources that are difficult to subtract such as
1982  bright stars. This filter is configured using the config parameters
1983  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1984  the maximum fraction of epochs that the deviation can appear in and still
1985  be considered an artifact. The spatialThreshold is the maximum fraction of
1986  pixels in the footprint of the deviation that appear in other epochs
1987  (where other epochs is defined by the temporalThreshold). If the deviant
1988  region meets this criteria of having a significant percentage of pixels
1989  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1990  set in the mask. These regions will not contribute to the final coadd.
1991  Furthermore, any routine to determine the coadd PSF can now be cognizant
1992  of clipped regions. Note that the algorithm implemented by this task is
1993  preliminary and works correctly for HSC data. Parameter modifications and
1994  or considerable redesigning of the algorithm is likley required for other
1995  surveys.
1996 
1997  ``CompareWarpAssembleCoaddTask`` sub-classes
1998  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1999  as a subtask to generate the TemplateCoadd (the model of the static sky).
2000 
2001  Notes
2002  -----
2003  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2004  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2005  ``baseDebug`` for more about ``debug.py`` files.
2006 
2007  This task supports the following debug variables:
2008 
2009  - ``saveCountIm``
2010  If True then save the Epoch Count Image as a fits file in the `figPath`
2011  - ``figPath``
2012  Path to save the debug fits images and figures
2013 
2014  For example, put something like:
2015 
2016  .. code-block:: python
2017 
2018  import lsstDebug
2019  def DebugInfo(name):
2020  di = lsstDebug.getInfo(name)
2021  if name == "lsst.pipe.tasks.assembleCoadd":
2022  di.saveCountIm = True
2023  di.figPath = "/desired/path/to/debugging/output/images"
2024  return di
2025  lsstDebug.Info = DebugInfo
2026 
2027  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2028  ``--debug`` flag. Some subtasks may have their own debug variables;
2029  see individual Task documentation.
2030 
2031  Examples
2032  --------
2033  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2034  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2035  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2036  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2037  and filter to be coadded (specified using
2038  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2039  along with a list of coaddTempExps to attempt to coadd (specified using
2040  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2041  Only the warps that cover the specified tract and patch will be coadded.
2042  A list of the available optional arguments can be obtained by calling
2043  ``assembleCoadd.py`` with the ``--help`` command line argument:
2044 
2045  .. code-block:: none
2046 
2047  assembleCoadd.py --help
2048 
2049  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2050  context of multi-band processing, we will generate the HSC-I & -R band
2051  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2052  To begin, assuming that the lsst stack has been already set up, we must
2053  set up the ``obs_subaru`` and ``ci_hsc`` packages.
2054  This defines the environment variable ``$CI_HSC_DIR`` and points at the
2055  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2056  directory. To begin assembling the coadds, we must first
2057 
2058  - processCcd
2059  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2060  - makeSkyMap
2061  create a skymap that covers the area of the sky present in the raw exposures
2062  - makeCoaddTempExp
2063  warp the individual calibrated exposures to the tangent plane of the coadd
2064 
2065  We can perform all of these steps by running
2066 
2067  .. code-block:: none
2068 
2069  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2070 
2071  This will produce warped ``coaddTempExps`` for each visit. To coadd the
2072  warped data, we call ``assembleCoadd.py`` as follows:
2073 
2074  .. code-block:: none
2075 
2076  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2077  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2078  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2079  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2080  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2081  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2082  --selectId visit=903988 ccd=24
2083 
2084  This will process the HSC-I band data. The results are written in
2085  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2086  """
2087  ConfigClass = CompareWarpAssembleCoaddConfig
2088  _DefaultName = "compareWarpAssembleCoadd"
2089 
2090  def __init__(self, *args, **kwargs):
2091  AssembleCoaddTask.__init__(self, *args, **kwargs)
2092  self.makeSubtask("assembleStaticSkyModel")
2093  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2094  self.makeSubtask("detect", schema=detectionSchema)
2095  if self.config.doPreserveContainedBySource:
2096  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2097  if self.config.doScaleWarpVariance:
2098  self.makeSubtask("scaleWarpVariance")
2099  if self.config.doFilterMorphological:
2100  self.makeSubtask("maskStreaks")
2101 
2102  @utils.inheritDoc(AssembleCoaddTask)
2103  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2104  """
2105  Generate a templateCoadd to use as a naive model of static sky to
2106  subtract from PSF-Matched warps.
2107 
2108  Returns
2109  -------
2110  result : `lsst.pipe.base.Struct`
2111  Result struct with components:
2112 
2113  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2114  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2115  """
2116  # Ensure that psfMatchedWarps are used as input warps for template generation
2117  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2118  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2119 
2120  # Because subtasks don't have connections we have to make one.
2121  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2122  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2123  if self.config.assembleStaticSkyModel.doWrite:
2124  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2125  # Remove template coadd from both subtask's and main tasks outputs,
2126  # because it is handled by the subtask as `coaddExposure`
2127  del outputRefs.templateCoadd
2128  del staticSkyModelOutputRefs.templateCoadd
2129 
2130  # A PSF-Matched nImage does not exist as a dataset type
2131  if 'nImage' in staticSkyModelOutputRefs.keys():
2132  del staticSkyModelOutputRefs.nImage
2133 
2134  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2135  staticSkyModelOutputRefs)
2136  if templateCoadd is None:
2137  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2138 
2139  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2140  nImage=templateCoadd.nImage,
2141  warpRefList=templateCoadd.warpRefList,
2142  imageScalerList=templateCoadd.imageScalerList,
2143  weightList=templateCoadd.weightList)
2144 
2145  @utils.inheritDoc(AssembleCoaddTask)
2146  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2147  """
2148  Generate a templateCoadd to use as a naive model of static sky to
2149  subtract from PSF-Matched warps.
2150 
2151  Returns
2152  -------
2153  result : `lsst.pipe.base.Struct`
2154  Result struct with components:
2155 
2156  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2157  - ``nImage``: N Image (``lsst.afw.image.Image``)
2158  """
2159  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2160  if templateCoadd is None:
2161  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2162 
2163  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2164  nImage=templateCoadd.nImage,
2165  warpRefList=templateCoadd.warpRefList,
2166  imageScalerList=templateCoadd.imageScalerList,
2167  weightList=templateCoadd.weightList)
2168 
2169  def _noTemplateMessage(self, warpType):
2170  warpName = (warpType[0].upper() + warpType[1:])
2171  message = """No %(warpName)s warps were found to build the template coadd which is
2172  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2173  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2174  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2175 
2176  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2177  another algorithm like:
2178 
2179  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2180  config.assemble.retarget(SafeClipAssembleCoaddTask)
2181  """ % {"warpName": warpName}
2182  return message
2183 
2184  @utils.inheritDoc(AssembleCoaddTask)
2185  @pipeBase.timeMethod
2186  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2187  supplementaryData, *args, **kwargs):
2188  """Assemble the coadd.
2189 
2190  Find artifacts and apply them to the warps' masks creating a list of
2191  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2192  plane. Then pass these alternative masks to the base class's `run`
2193  method.
2194 
2195  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2196  that must contain a ``templateCoadd`` that serves as the
2197  model of the static sky.
2198  """
2199 
2200  # Check and match the order of the supplementaryData
2201  # (PSF-matched) inputs to the order of the direct inputs,
2202  # so that the artifact mask is applied to the right warp
2203  dataIds = [ref.dataId for ref in tempExpRefList]
2204  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2205 
2206  if dataIds != psfMatchedDataIds:
2207  self.log.info("Reordering and or/padding PSF-matched visit input list")
2208  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2209  psfMatchedDataIds, dataIds)
2210  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2211  psfMatchedDataIds, dataIds)
2212 
2213  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2214  spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2215  supplementaryData.warpRefList,
2216  supplementaryData.imageScalerList)
2217 
2218  badMaskPlanes = self.config.badMaskPlanes[:]
2219  badMaskPlanes.append("CLIPPED")
2220  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2221 
2222  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2223  spanSetMaskList, mask=badPixelMask)
2224 
2225  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2226  # Psf-Matching moves the real edge inwards
2227  self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2228  return result
2229 
2230  def applyAltEdgeMask(self, mask, altMaskList):
2231  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2232 
2233  Parameters
2234  ----------
2235  mask : `lsst.afw.image.Mask`
2236  Original mask.
2237  altMaskList : `list`
2238  List of Dicts containing ``spanSet`` lists.
2239  Each element contains the new mask plane name (e.g. "CLIPPED
2240  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2241  the mask.
2242  """
2243  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2244  for visitMask in altMaskList:
2245  if "EDGE" in visitMask:
2246  for spanSet in visitMask['EDGE']:
2247  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2248 
2249  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2250  """Find artifacts.
2251 
2252  Loop through warps twice. The first loop builds a map with the count
2253  of how many epochs each pixel deviates from the templateCoadd by more
2254  than ``config.chiThreshold`` sigma. The second loop takes each
2255  difference image and filters the artifacts detected in each using
2256  count map to filter out variable sources and sources that are
2257  difficult to subtract cleanly.
2258 
2259  Parameters
2260  ----------
2261  templateCoadd : `lsst.afw.image.Exposure`
2262  Exposure to serve as model of static sky.
2263  tempExpRefList : `list`
2264  List of data references to warps.
2265  imageScalerList : `list`
2266  List of image scalers.
2267 
2268  Returns
2269  -------
2270  altMasks : `list`
2271  List of dicts containing information about CLIPPED
2272  (i.e., artifacts), NO_DATA, and EDGE pixels.
2273  """
2274 
2275  self.log.debug("Generating Count Image, and mask lists.")
2276  coaddBBox = templateCoadd.getBBox()
2277  slateIm = afwImage.ImageU(coaddBBox)
2278  epochCountImage = afwImage.ImageU(coaddBBox)
2279  nImage = afwImage.ImageU(coaddBBox)
2280  spanSetArtifactList = []
2281  spanSetNoDataMaskList = []
2282  spanSetEdgeList = []
2283  spanSetBadMorphoList = []
2284  badPixelMask = self.getBadPixelMask()
2285 
2286  # mask of the warp diffs should = that of only the warp
2287  templateCoadd.mask.clearAllMaskPlanes()
2288 
2289  if self.config.doPreserveContainedBySource:
2290  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2291  else:
2292  templateFootprints = None
2293 
2294  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2295  warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2296  if warpDiffExp is not None:
2297  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2298  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2299  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2300  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2301  fpSet.positive.merge(fpSet.negative)
2302  footprints = fpSet.positive
2303  slateIm.set(0)
2304  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2305 
2306  # Remove artifacts due to defects before they contribute to the epochCountImage
2307  if self.config.doPrefilterArtifacts:
2308  spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2309 
2310  # Clear mask before adding prefiltered spanSets
2311  self.detect.clearMask(warpDiffExp.mask)
2312  for spans in spanSetList:
2313  spans.setImage(slateIm, 1, doClip=True)
2314  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2315  epochCountImage += slateIm
2316 
2317  if self.config.doFilterMorphological:
2318  maskName = self.config.streakMaskName
2319  _ = self.maskStreaks.run(warpDiffExp)
2320  streakMask = warpDiffExp.mask
2321  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2322  streakMask.getPlaneBitMask(maskName)).split()
2323 
2324  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2325  # undergo a second convolution. Pixels with data in the direct warp
2326  # but not in the PSF-matched warp will not have their artifacts detected.
2327  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2328  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2329  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2330  nansMask.setXY0(warpDiffExp.getXY0())
2331  edgeMask = warpDiffExp.mask
2332  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2333  edgeMask.getPlaneBitMask("EDGE")).split()
2334  else:
2335  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2336  # In this case, mask the whole epoch
2337  nansMask = afwImage.MaskX(coaddBBox, 1)
2338  spanSetList = []
2339  spanSetEdgeMask = []
2340  spanSetStreak = []
2341 
2342  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2343 
2344  spanSetNoDataMaskList.append(spanSetNoDataMask)
2345  spanSetArtifactList.append(spanSetList)
2346  spanSetEdgeList.append(spanSetEdgeMask)
2347  if self.config.doFilterMorphological:
2348  spanSetBadMorphoList.append(spanSetStreak)
2349 
2350  if lsstDebug.Info(__name__).saveCountIm:
2351  path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2352  epochCountImage.writeFits(path)
2353 
2354  for i, spanSetList in enumerate(spanSetArtifactList):
2355  if spanSetList:
2356  filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2357  templateFootprints)
2358  spanSetArtifactList[i] = filteredSpanSetList
2359  if self.config.doFilterMorphological:
2360  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2361 
2362  altMasks = []
2363  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2364  altMasks.append({'CLIPPED': artifacts,
2365  'NO_DATA': noData,
2366  'EDGE': edge})
2367  return altMasks
2368 
2369  def prefilterArtifacts(self, spanSetList, exp):
2370  """Remove artifact candidates covered by bad mask plane.
2371 
2372  Any future editing of the candidate list that does not depend on
2373  temporal information should go in this method.
2374 
2375  Parameters
2376  ----------
2377  spanSetList : `list`
2378  List of SpanSets representing artifact candidates.
2379  exp : `lsst.afw.image.Exposure`
2380  Exposure containing mask planes used to prefilter.
2381 
2382  Returns
2383  -------
2384  returnSpanSetList : `list`
2385  List of SpanSets with artifacts.
2386  """
2387  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2388  goodArr = (exp.mask.array & badPixelMask) == 0
2389  returnSpanSetList = []
2390  bbox = exp.getBBox()
2391  x0, y0 = exp.getXY0()
2392  for i, span in enumerate(spanSetList):
2393  y, x = span.clippedTo(bbox).indices()
2394  yIndexLocal = numpy.array(y) - y0
2395  xIndexLocal = numpy.array(x) - x0
2396  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2397  if goodRatio > self.config.prefilterArtifactsRatio:
2398  returnSpanSetList.append(span)
2399  return returnSpanSetList
2400 
2401  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2402  """Filter artifact candidates.
2403 
2404  Parameters
2405  ----------
2406  spanSetList : `list`
2407  List of SpanSets representing artifact candidates.
2408  epochCountImage : `lsst.afw.image.Image`
2409  Image of accumulated number of warpDiff detections.
2410  nImage : `lsst.afw.image.Image`
2411  Image of the accumulated number of total epochs contributing.
2412 
2413  Returns
2414  -------
2415  maskSpanSetList : `list`
2416  List of SpanSets with artifacts.
2417  """
2418 
2419  maskSpanSetList = []
2420  x0, y0 = epochCountImage.getXY0()
2421  for i, span in enumerate(spanSetList):
2422  y, x = span.indices()
2423  yIdxLocal = [y1 - y0 for y1 in y]
2424  xIdxLocal = [x1 - x0 for x1 in x]
2425  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2426  totalN = nImage.array[yIdxLocal, xIdxLocal]
2427 
2428  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2429  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2430  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2431  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2432  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2433  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2434  & (outlierN <= effectiveMaxNumEpochs))
2435  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2436  if percentBelowThreshold > self.config.spatialThreshold:
2437  maskSpanSetList.append(span)
2438 
2439  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2440  # If a candidate is contained by a footprint on the template coadd, do not clip
2441  filteredMaskSpanSetList = []
2442  for span in maskSpanSetList:
2443  doKeep = True
2444  for footprint in footprintsToExclude.positive.getFootprints():
2445  if footprint.spans.contains(span):
2446  doKeep = False
2447  break
2448  if doKeep:
2449  filteredMaskSpanSetList.append(span)
2450  maskSpanSetList = filteredMaskSpanSetList
2451 
2452  return maskSpanSetList
2453 
2454  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2455  """Fetch a warp from the butler and return a warpDiff.
2456 
2457  Parameters
2458  ----------
2459  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2460  Butler dataRef for the warp.
2461  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2462  An image scaler object.
2463  templateCoadd : `lsst.afw.image.Exposure`
2464  Exposure to be substracted from the scaled warp.
2465 
2466  Returns
2467  -------
2468  warp : `lsst.afw.image.Exposure`
2469  Exposure of the image difference between the warp and template.
2470  """
2471 
2472  # If the PSF-Matched warp did not exist for this direct warp
2473  # None is holding its place to maintain order in Gen 3
2474  if warpRef is None:
2475  return None
2476  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2477  warpName = self.getTempExpDatasetName('psfMatched')
2478  if not isinstance(warpRef, DeferredDatasetHandle):
2479  if not warpRef.datasetExists(warpName):
2480  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2481  return None
2482  warp = warpRef.get(datasetType=warpName, immediate=True)
2483  # direct image scaler OK for PSF-matched Warp
2484  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2485  mi = warp.getMaskedImage()
2486  if self.config.doScaleWarpVariance:
2487  try:
2488  self.scaleWarpVariance.run(mi)
2489  except Exception as exc:
2490  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2491  mi -= templateCoadd.getMaskedImage()
2492  return warp
2493 
2494  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2495  """Return a path to which to write debugging output.
2496 
2497  Creates a hyphen-delimited string of dataId values for simple filenames.
2498 
2499  Parameters
2500  ----------
2501  prefix : `str`
2502  Prefix for filename.
2503  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2504  Butler dataRef to make the path from.
2505  coaddLevel : `bool`, optional.
2506  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2507  'filter', but no 'visit').
2508 
2509  Returns
2510  -------
2511  result : `str`
2512  Path for debugging output.
2513  """
2514  if coaddLevel:
2515  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2516  else:
2517  keys = warpRef.dataId.keys()
2518  keyList = sorted(keys, reverse=True)
2519  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2520  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2521  return os.path.join(directory, filename)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:141
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:346
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60