lsst.pipe.tasks  21.0.0-89-g04f5858e+73638ba61f
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from .healSparseMapping import HealSparseInputMapTask
48 from lsst.meas.algorithms import SourceDetectionTask
49 from lsst.daf.butler import DeferredDatasetHandle
50 
51 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
52  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
53  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
54 
55 
56 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
57  dimensions=("tract", "patch", "band", "skymap"),
58  defaultTemplates={"inputCoaddName": "deep",
59  "outputCoaddName": "deep",
60  "warpType": "direct",
61  "warpTypeSuffix": ""}):
62 
63  inputWarps = pipeBase.connectionTypes.Input(
64  doc=("Input list of warps to be assemebled i.e. stacked."
65  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
66  name="{inputCoaddName}Coadd_{warpType}Warp",
67  storageClass="ExposureF",
68  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
69  deferLoad=True,
70  multiple=True
71  )
72  skyMap = pipeBase.connectionTypes.Input(
73  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
74  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75  storageClass="SkyMap",
76  dimensions=("skymap", ),
77  )
78  selectedVisits = pipeBase.connectionTypes.Input(
79  doc="Selected visits to be coadded.",
80  name="{outputCoaddName}Visits",
81  storageClass="StructuredDataDict",
82  dimensions=("instrument", "tract", "patch", "skymap", "band")
83  )
84  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
85  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
86  " BRIGHT_OBJECT."),
87  name="brightObjectMask",
88  storageClass="ObjectMaskCatalog",
89  dimensions=("tract", "patch", "skymap", "band"),
90  )
91  coaddExposure = pipeBase.connectionTypes.Output(
92  doc="Output coadded exposure, produced by stacking input warps",
93  name="{outputCoaddName}Coadd{warpTypeSuffix}",
94  storageClass="ExposureF",
95  dimensions=("tract", "patch", "skymap", "band"),
96  )
97  nImage = pipeBase.connectionTypes.Output(
98  doc="Output image of number of input images per pixel",
99  name="{outputCoaddName}Coadd_nImage",
100  storageClass="ImageU",
101  dimensions=("tract", "patch", "skymap", "band"),
102  )
103  inputMap = pipeBase.connectionTypes.Output(
104  doc="Output healsparse map of input images",
105  name="{outputCoaddName}Coadd_inputMap",
106  storageClass="HealSparseMap",
107  dimensions=("tract", "patch", "skymap", "band"),
108  )
109 
110  def __init__(self, *, config=None):
111  super().__init__(config=config)
112 
113  # Override the connection's name template with config to replicate Gen2 behavior
114  # This duplicates some of the logic in the base class, due to wanting Gen2 and
115  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
116  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
117  templateValues['warpType'] = config.warpType
118  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
119  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
120  for name in self.allConnections}
121  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
122  # End code to remove after deprecation
123 
124  if not config.doMaskBrightObjects:
125  self.prerequisiteInputs.remove("brightObjectMask")
126 
127  if not config.doSelectVisits:
128  self.inputs.remove("selectedVisits")
129 
130  if not config.doNImage:
131  self.outputs.remove("nImage")
132 
133  if not self.config.doInputMap:
134  self.outputs.remove("inputMap")
135 
136 
137 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
138  pipelineConnections=AssembleCoaddConnections):
139  """Configuration parameters for the `AssembleCoaddTask`.
140 
141  Notes
142  -----
143  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
144  only set the bitplane config.brightObjectMaskName. To make this useful you
145  *must* also configure the flags.pixel algorithm, for example by adding
146 
147  .. code-block:: none
148 
149  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
150  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
151 
152  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
153  """
154  warpType = pexConfig.Field(
155  doc="Warp name: one of 'direct' or 'psfMatched'",
156  dtype=str,
157  default="direct",
158  )
159  subregionSize = pexConfig.ListField(
160  dtype=int,
161  doc="Width, height of stack subregion size; "
162  "make small enough that a full stack of images will fit into memory at once.",
163  length=2,
164  default=(2000, 2000),
165  )
166  statistic = pexConfig.Field(
167  dtype=str,
168  doc="Main stacking statistic for aggregating over the epochs.",
169  default="MEANCLIP",
170  )
171  doSigmaClip = pexConfig.Field(
172  dtype=bool,
173  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
174  default=False,
175  )
176  sigmaClip = pexConfig.Field(
177  dtype=float,
178  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
179  default=3.0,
180  )
181  clipIter = pexConfig.Field(
182  dtype=int,
183  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
184  default=2,
185  )
186  calcErrorFromInputVariance = pexConfig.Field(
187  dtype=bool,
188  doc="Calculate coadd variance from input variance by stacking statistic."
189  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
190  default=True,
191  )
192  scaleZeroPoint = pexConfig.ConfigurableField(
193  target=ScaleZeroPointTask,
194  doc="Task to adjust the photometric zero point of the coadd temp exposures",
195  )
196  doInterp = pexConfig.Field(
197  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
198  dtype=bool,
199  default=True,
200  )
201  interpImage = pexConfig.ConfigurableField(
202  target=InterpImageTask,
203  doc="Task to interpolate (and extrapolate) over NaN pixels",
204  )
205  doWrite = pexConfig.Field(
206  doc="Persist coadd?",
207  dtype=bool,
208  default=True,
209  )
210  doNImage = pexConfig.Field(
211  doc="Create image of number of contributing exposures for each pixel",
212  dtype=bool,
213  default=False,
214  )
215  doUsePsfMatchedPolygons = pexConfig.Field(
216  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
217  dtype=bool,
218  default=False,
219  )
220  maskPropagationThresholds = pexConfig.DictField(
221  keytype=str,
222  itemtype=float,
223  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
224  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
225  "would have contributed exceeds this value."),
226  default={"SAT": 0.1},
227  )
228  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
229  doc="Mask planes to remove before coadding")
230  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
231  doc="Set mask and flag bits for bright objects?")
232  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
233  doc="Name of mask bit used for bright objects")
234  coaddPsf = pexConfig.ConfigField(
235  doc="Configuration for CoaddPsf",
236  dtype=measAlg.CoaddPsfConfig,
237  )
238  doAttachTransmissionCurve = pexConfig.Field(
239  dtype=bool, default=False, optional=False,
240  doc=("Attach a piecewise TransmissionCurve for the coadd? "
241  "(requires all input Exposures to have TransmissionCurves).")
242  )
243  hasFakes = pexConfig.Field(
244  dtype=bool,
245  default=False,
246  doc="Should be set to True if fake sources have been inserted into the input data."
247  )
248  doSelectVisits = pexConfig.Field(
249  doc="Coadd only visits selected by a SelectVisitsTask",
250  dtype=bool,
251  default=False,
252  )
253  doInputMap = pexConfig.Field(
254  doc="Create a bitwise map of coadd inputs",
255  dtype=bool,
256  default=False,
257  )
258  inputMapper = pexConfig.ConfigurableField(
259  doc="Input map creation subtask.",
260  target=HealSparseInputMapTask,
261  )
262 
263  def setDefaults(self):
264  super().setDefaults()
265  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
266 
267  def validate(self):
268  super().validate()
269  if self.doPsfMatch:
270  # Backwards compatibility.
271  # Configs do not have loggers
272  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
273  self.warpType = 'psfMatched'
274  if self.doSigmaClip and self.statistic != "MEANCLIP":
275  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
276  self.statistic = "MEANCLIP"
277  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
278  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
279  "compute and set a non-zero coadd variance estimate." % (self.statistic))
280 
281  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
282  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
283  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
284  if str(k) not in unstackableStats]
285  raise ValueError("statistic %s is not allowed. Please choose one of %s."
286  % (self.statistic, stackableStats))
287 
288 
289 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
290  """Assemble a coadded image from a set of warps (coadded temporary exposures).
291 
292  We want to assemble a coadded image from a set of Warps (also called
293  coadded temporary exposures or ``coaddTempExps``).
294  Each input Warp covers a patch on the sky and corresponds to a single
295  run/visit/exposure of the covered patch. We provide the task with a list
296  of Warps (``selectDataList``) from which it selects Warps that cover the
297  specified patch (pointed at by ``dataRef``).
298  Each Warp that goes into a coadd will typically have an independent
299  photometric zero-point. Therefore, we must scale each Warp to set it to
300  a common photometric zeropoint. WarpType may be one of 'direct' or
301  'psfMatched', and the boolean configs `config.makeDirect` and
302  `config.makePsfMatched` set which of the warp types will be coadded.
303  The coadd is computed as a mean with optional outlier rejection.
304  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
305  Finally, Warps can have bad 'NaN' pixels which received no input from the
306  source calExps. We interpolate over these bad (NaN) pixels.
307 
308  `AssembleCoaddTask` uses several sub-tasks. These are
309 
310  - `ScaleZeroPointTask`
311  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
312  - `InterpImageTask`
313  - interpolate across bad pixels (NaN) in the final coadd
314 
315  You can retarget these subtasks if you wish.
316 
317  Notes
318  -----
319  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
320  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
321  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
322  no debug variables of its own. Some of the subtasks may support debug
323  variables. See the documentation for the subtasks for further information.
324 
325  Examples
326  --------
327  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
328  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
329  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
330  inputs: a data reference to the tract patch and filter to be coadded, and
331  a list of Warps to attempt to coadd. These are specified using ``--id`` and
332  ``--selectId``, respectively:
333 
334  .. code-block:: none
335 
336  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
337  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
338 
339  Only the Warps that cover the specified tract and patch will be coadded.
340  A list of the available optional arguments can be obtained by calling
341  ``assembleCoadd.py`` with the ``--help`` command line argument:
342 
343  .. code-block:: none
344 
345  assembleCoadd.py --help
346 
347  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
348  multi-band processing, we will generate the HSC-I & -R band coadds from
349  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
350  assuming that the lsst stack has been already set up, we must set up the
351  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
352  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
353  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
354  coadds, we must first
355 
356  - processCcd
357  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
358  - makeSkyMap
359  - create a skymap that covers the area of the sky present in the raw exposures
360  - makeCoaddTempExp
361  - warp the individual calibrated exposures to the tangent plane of the coadd
362 
363  We can perform all of these steps by running
364 
365  .. code-block:: none
366 
367  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
368 
369  This will produce warped exposures for each visit. To coadd the warped
370  data, we call assembleCoadd.py as follows:
371 
372  .. code-block:: none
373 
374  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
375  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
376  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
377  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
378  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
379  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
380  --selectId visit=903988 ccd=24
381 
382  that will process the HSC-I band data. The results are written in
383  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
384 
385  You may also choose to run:
386 
387  .. code-block:: none
388 
389  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
390  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
391  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
392  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
393  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
394  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
395  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
396  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
397 
398  to generate the coadd for the HSC-R band if you are interested in
399  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
400  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
401  rather than `AssembleCoaddTask` to make the coadd.
402  """
403  ConfigClass = AssembleCoaddConfig
404  _DefaultName = "assembleCoadd"
405 
406  def __init__(self, *args, **kwargs):
407  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
408  if args:
409  argNames = ["config", "name", "parentTask", "log"]
410  kwargs.update({k: v for k, v in zip(argNames, args)})
411  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
412  "PipelineTask will not take positional args" % argNames, FutureWarning)
413 
414  super().__init__(**kwargs)
415  self.makeSubtask("interpImage")
416  self.makeSubtask("scaleZeroPoint")
417 
418  if self.config.doMaskBrightObjects:
419  mask = afwImage.Mask()
420  try:
421  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
422  except pexExceptions.LsstCppException:
423  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
424  mask.getMaskPlaneDict().keys())
425  del mask
426 
427  if self.config.doInputMap:
428  self.makeSubtask("inputMapper")
429 
430  self.warpType = self.config.warpType
431 
432  @utils.inheritDoc(pipeBase.PipelineTask)
433  def runQuantum(self, butlerQC, inputRefs, outputRefs):
434  # Docstring to be formatted with info from PipelineTask.runQuantum
435  """
436  Notes
437  -----
438  Assemble a coadd from a set of Warps.
439 
440  PipelineTask (Gen3) entry point to Coadd a set of Warps.
441  Analogous to `runDataRef`, it prepares all the data products to be
442  passed to `run`, and processes the results before returning a struct
443  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
444  Therefore, its inputs are accessed subregion by subregion
445  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
446  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
447  correspond to an update in `runDataRef` while both entry points
448  are used.
449  """
450  inputData = butlerQC.get(inputRefs)
451 
452  # Construct skyInfo expected by run
453  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
454  skyMap = inputData["skyMap"]
455  outputDataId = butlerQC.quantum.dataId
456 
457  inputData['skyInfo'] = makeSkyInfo(skyMap,
458  tractId=outputDataId['tract'],
459  patchId=outputDataId['patch'])
460 
461  if self.config.doSelectVisits:
462  warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
463  else:
464  warpRefList = inputData['inputWarps']
465 
466  # Perform same middle steps as `runDataRef` does
467  inputs = self.prepareInputs(warpRefList)
468  self.log.info("Found %d %s", len(inputs.tempExpRefList),
469  self.getTempExpDatasetName(self.warpType))
470  if len(inputs.tempExpRefList) == 0:
471  self.log.warn("No coadd temporary exposures found")
472  return
473 
474  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
475  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
476  inputs.weightList, supplementaryData=supplementaryData)
477 
478  inputData.setdefault('brightObjectMask', None)
479  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
480 
481  if self.config.doWrite:
482  butlerQC.put(retStruct, outputRefs)
483  return retStruct
484 
485  @pipeBase.timeMethod
486  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
487  """Assemble a coadd from a set of Warps.
488 
489  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
490  Compute weights to be applied to each Warp and
491  find scalings to match the photometric zeropoint to a reference Warp.
492  Assemble the Warps using `run`. Interpolate over NaNs and
493  optionally write the coadd to disk. Return the coadded exposure.
494 
495  Parameters
496  ----------
497  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
498  Data reference defining the patch for coaddition and the
499  reference Warp (if ``config.autoReference=False``).
500  Used to access the following data products:
501  - ``self.config.coaddName + "Coadd_skyMap"``
502  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
503  - ``self.config.coaddName + "Coadd"``
504  selectDataList : `list`
505  List of data references to Calexps. Data to be coadded will be
506  selected from this list based on overlap with the patch defined
507  by dataRef, grouped by visit, and converted to a list of data
508  references to warps.
509  warpRefList : `list`
510  List of data references to Warps to be coadded.
511  Note: `warpRefList` is just the new name for `tempExpRefList`.
512 
513  Returns
514  -------
515  retStruct : `lsst.pipe.base.Struct`
516  Result struct with components:
517 
518  - ``coaddExposure``: coadded exposure (``Exposure``).
519  - ``nImage``: exposure count image (``Image``).
520  """
521  if selectDataList and warpRefList:
522  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
523  "and which to use is ambiguous. Please pass only one.")
524 
525  skyInfo = self.getSkyInfo(dataRef)
526  if warpRefList is None:
527  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
528  if len(calExpRefList) == 0:
529  self.log.warn("No exposures to coadd")
530  return
531  self.log.info("Coadding %d exposures", len(calExpRefList))
532 
533  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
534 
535  inputData = self.prepareInputs(warpRefList)
536  self.log.info("Found %d %s", len(inputData.tempExpRefList),
537  self.getTempExpDatasetName(self.warpType))
538  if len(inputData.tempExpRefList) == 0:
539  self.log.warn("No coadd temporary exposures found")
540  return
541 
542  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
543 
544  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
545  inputData.weightList, supplementaryData=supplementaryData)
546 
547  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
548  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
549 
550  if self.config.doWrite:
551  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
552  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
553  else:
554  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
555  self.log.info("Persisting %s" % coaddDatasetName)
556  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
557  if self.config.doNImage and retStruct.nImage is not None:
558  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
559 
560  return retStruct
561 
562  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
563  """Interpolate over missing data and mask bright stars.
564 
565  Parameters
566  ----------
567  coaddExposure : `lsst.afw.image.Exposure`
568  The coadded exposure to process.
569  dataRef : `lsst.daf.persistence.ButlerDataRef`
570  Butler data reference for supplementary data.
571  """
572  if self.config.doInterp:
573  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
574  # The variance must be positive; work around for DM-3201.
575  varArray = coaddExposure.variance.array
576  with numpy.errstate(invalid="ignore"):
577  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
578 
579  if self.config.doMaskBrightObjects:
580  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
581 
582  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
583  """Make additional inputs to run() specific to subclasses (Gen2)
584 
585  Duplicates interface of `runDataRef` method
586  Available to be implemented by subclasses only if they need the
587  coadd dataRef for performing preliminary processing before
588  assembling the coadd.
589 
590  Parameters
591  ----------
592  dataRef : `lsst.daf.persistence.ButlerDataRef`
593  Butler data reference for supplementary data.
594  selectDataList : `list` (optional)
595  Optional List of data references to Calexps.
596  warpRefList : `list` (optional)
597  Optional List of data references to Warps.
598  """
599  return pipeBase.Struct()
600 
601  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
602  """Make additional inputs to run() specific to subclasses (Gen3)
603 
604  Duplicates interface of `runQuantum` method.
605  Available to be implemented by subclasses only if they need the
606  coadd dataRef for performing preliminary processing before
607  assembling the coadd.
608 
609  Parameters
610  ----------
611  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
612  Gen3 Butler object for fetching additional data products before
613  running the Task specialized for quantum being processed
614  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
615  Attributes are the names of the connections describing input dataset types.
616  Values are DatasetRefs that task consumes for corresponding dataset type.
617  DataIds are guaranteed to match data objects in ``inputData``.
618  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
619  Attributes are the names of the connections describing output dataset types.
620  Values are DatasetRefs that task is to produce
621  for corresponding dataset type.
622  """
623  return pipeBase.Struct()
624 
625  def getTempExpRefList(self, patchRef, calExpRefList):
626  """Generate list data references corresponding to warped exposures
627  that lie within the patch to be coadded.
628 
629  Parameters
630  ----------
631  patchRef : `dataRef`
632  Data reference for patch.
633  calExpRefList : `list`
634  List of data references for input calexps.
635 
636  Returns
637  -------
638  tempExpRefList : `list`
639  List of Warp/CoaddTempExp data references.
640  """
641  butler = patchRef.getButler()
642  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
643  self.getTempExpDatasetName(self.warpType))
644  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
645  g, groupData.keys) for
646  g in groupData.groups.keys()]
647  return tempExpRefList
648 
649  def prepareInputs(self, refList):
650  """Prepare the input warps for coaddition by measuring the weight for
651  each warp and the scaling for the photometric zero point.
652 
653  Each Warp has its own photometric zeropoint and background variance.
654  Before coadding these Warps together, compute a scale factor to
655  normalize the photometric zeropoint and compute the weight for each Warp.
656 
657  Parameters
658  ----------
659  refList : `list`
660  List of data references to tempExp
661 
662  Returns
663  -------
664  result : `lsst.pipe.base.Struct`
665  Result struct with components:
666 
667  - ``tempExprefList``: `list` of data references to tempExp.
668  - ``weightList``: `list` of weightings.
669  - ``imageScalerList``: `list` of image scalers.
670  """
671  statsCtrl = afwMath.StatisticsControl()
672  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
673  statsCtrl.setNumIter(self.config.clipIter)
674  statsCtrl.setAndMask(self.getBadPixelMask())
675  statsCtrl.setNanSafe(True)
676  # compute tempExpRefList: a list of tempExpRef that actually exist
677  # and weightList: a list of the weight of the associated coadd tempExp
678  # and imageScalerList: a list of scale factors for the associated coadd tempExp
679  tempExpRefList = []
680  weightList = []
681  imageScalerList = []
682  tempExpName = self.getTempExpDatasetName(self.warpType)
683  for tempExpRef in refList:
684  # Gen3's DeferredDatasetHandles are guaranteed to exist and
685  # therefore have no datasetExists() method
686  if not isinstance(tempExpRef, DeferredDatasetHandle):
687  if not tempExpRef.datasetExists(tempExpName):
688  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
689  continue
690 
691  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
692  # Ignore any input warp that is empty of data
693  if numpy.isnan(tempExp.image.array).all():
694  continue
695  maskedImage = tempExp.getMaskedImage()
696  imageScaler = self.scaleZeroPoint.computeImageScaler(
697  exposure=tempExp,
698  dataRef=tempExpRef,
699  )
700  try:
701  imageScaler.scaleMaskedImage(maskedImage)
702  except Exception as e:
703  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
704  continue
705  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
706  afwMath.MEANCLIP, statsCtrl)
707  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
708  weight = 1.0 / float(meanVar)
709  if not numpy.isfinite(weight):
710  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
711  continue
712  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
713 
714  del maskedImage
715  del tempExp
716 
717  tempExpRefList.append(tempExpRef)
718  weightList.append(weight)
719  imageScalerList.append(imageScaler)
720 
721  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
722  imageScalerList=imageScalerList)
723 
724  def prepareStats(self, mask=None):
725  """Prepare the statistics for coadding images.
726 
727  Parameters
728  ----------
729  mask : `int`, optional
730  Bit mask value to exclude from coaddition.
731 
732  Returns
733  -------
734  stats : `lsst.pipe.base.Struct`
735  Statistics structure with the following fields:
736 
737  - ``statsCtrl``: Statistics control object for coadd
738  (`lsst.afw.math.StatisticsControl`)
739  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
740  """
741  if mask is None:
742  mask = self.getBadPixelMask()
743  statsCtrl = afwMath.StatisticsControl()
744  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
745  statsCtrl.setNumIter(self.config.clipIter)
746  statsCtrl.setAndMask(mask)
747  statsCtrl.setNanSafe(True)
748  statsCtrl.setWeighted(True)
749  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
750  for plane, threshold in self.config.maskPropagationThresholds.items():
751  bit = afwImage.Mask.getMaskPlane(plane)
752  statsCtrl.setMaskPropagationThreshold(bit, threshold)
753  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
754  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
755 
756  @pipeBase.timeMethod
757  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
758  altMaskList=None, mask=None, supplementaryData=None):
759  """Assemble a coadd from input warps
760 
761  Assemble the coadd using the provided list of coaddTempExps. Since
762  the full coadd covers a patch (a large area), the assembly is
763  performed over small areas on the image at a time in order to
764  conserve memory usage. Iterate over subregions within the outer
765  bbox of the patch using `assembleSubregion` to stack the corresponding
766  subregions from the coaddTempExps with the statistic specified.
767  Set the edge bits the coadd mask based on the weight map.
768 
769  Parameters
770  ----------
771  skyInfo : `lsst.pipe.base.Struct`
772  Struct with geometric information about the patch.
773  tempExpRefList : `list`
774  List of data references to Warps (previously called CoaddTempExps).
775  imageScalerList : `list`
776  List of image scalers.
777  weightList : `list`
778  List of weights
779  altMaskList : `list`, optional
780  List of alternate masks to use rather than those stored with
781  tempExp.
782  mask : `int`, optional
783  Bit mask value to exclude from coaddition.
784  supplementaryData : lsst.pipe.base.Struct, optional
785  Struct with additional data products needed to assemble coadd.
786  Only used by subclasses that implement `makeSupplementaryData`
787  and override `run`.
788 
789  Returns
790  -------
791  result : `lsst.pipe.base.Struct`
792  Result struct with components:
793 
794  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
795  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
796  - ``inputMap``: bit-wise map of inputs, if requested.
797  - ``warpRefList``: input list of refs to the warps (
798  ``lsst.daf.butler.DeferredDatasetHandle`` or
799  ``lsst.daf.persistence.ButlerDataRef``)
800  (unmodified)
801  - ``imageScalerList``: input list of image scalers (unmodified)
802  - ``weightList``: input list of weights (unmodified)
803  """
804  tempExpName = self.getTempExpDatasetName(self.warpType)
805  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
806  stats = self.prepareStats(mask=mask)
807 
808  if altMaskList is None:
809  altMaskList = [None]*len(tempExpRefList)
810 
811  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
812  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
813  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
814  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
815  coaddMaskedImage = coaddExposure.getMaskedImage()
816  subregionSizeArr = self.config.subregionSize
817  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
818  # if nImage is requested, create a zero one which can be passed to assembleSubregion
819  if self.config.doNImage:
820  nImage = afwImage.ImageU(skyInfo.bbox)
821  else:
822  nImage = None
823  # If inputMap is requested, create the initial version that can be masked in
824  # assembleSubregion.
825  if self.config.doInputMap:
826  self.inputMapper.build_ccd_input_map(skyInfo.bbox,
827  skyInfo.wcs,
828  coaddExposure.getInfo().getCoaddInputs().ccds)
829 
830  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
831  try:
832  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
833  weightList, altMaskList, stats.flags, stats.ctrl,
834  nImage=nImage)
835  except Exception as e:
836  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
837 
838  # If inputMap is requested, we must finalize the map after the accumulation.
839  if self.config.doInputMap:
840  self.inputMapper.finalize_ccd_input_map_mask()
841  inputMap = self.inputMapper.ccd_input_map
842  else:
843  inputMap = None
844 
845  self.setInexactPsf(coaddMaskedImage.getMask())
846  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
847  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
848  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
849  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
850  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
851  weightList=weightList, inputMap=inputMap)
852 
853  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
854  """Set the metadata for the coadd.
855 
856  This basic implementation sets the filter from the first input.
857 
858  Parameters
859  ----------
860  coaddExposure : `lsst.afw.image.Exposure`
861  The target exposure for the coadd.
862  tempExpRefList : `list`
863  List of data references to tempExp.
864  weightList : `list`
865  List of weights.
866  """
867  assert len(tempExpRefList) == len(weightList), "Length mismatch"
868  tempExpName = self.getTempExpDatasetName(self.warpType)
869  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
870  # (and we need more than just the PropertySet that contains the header), which is not possible
871  # with the current butler (see #2777).
872  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
873 
874  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
875  # Gen 3 API
876  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
877  else:
878  # Gen 2 API. Delete this when Gen 2 retired
879  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
880  for tempExpRef in tempExpRefList]
881  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
882 
883  # Set the coadd FilterLabel to the band of the first input exposure:
884  # Coadds are calibrated, so the physical label is now meaningless.
885  coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
886  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
887  coaddInputs.ccds.reserve(numCcds)
888  coaddInputs.visits.reserve(len(tempExpList))
889 
890  for tempExp, weight in zip(tempExpList, weightList):
891  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
892 
893  if self.config.doUsePsfMatchedPolygons:
894  self.shrinkValidPolygons(coaddInputs)
895 
896  coaddInputs.visits.sort()
897  if self.warpType == "psfMatched":
898  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
899  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
900  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
901  # having the maximum width (sufficient because square)
902  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
903  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
904  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
905  else:
906  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
907  self.config.coaddPsf.makeControl())
908  coaddExposure.setPsf(psf)
909  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
910  coaddExposure.getWcs())
911  coaddExposure.getInfo().setApCorrMap(apCorrMap)
912  if self.config.doAttachTransmissionCurve:
913  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
914  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
915 
916  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
917  altMaskList, statsFlags, statsCtrl, nImage=None):
918  """Assemble the coadd for a sub-region.
919 
920  For each coaddTempExp, check for (and swap in) an alternative mask
921  if one is passed. Remove mask planes listed in
922  `config.removeMaskPlanes`. Finally, stack the actual exposures using
923  `lsst.afw.math.statisticsStack` with the statistic specified by
924  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
925  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
926  an N-sigma clipped mean where N and iterations are specified by
927  statsCtrl. Assign the stacked subregion back to the coadd.
928 
929  Parameters
930  ----------
931  coaddExposure : `lsst.afw.image.Exposure`
932  The target exposure for the coadd.
933  bbox : `lsst.geom.Box`
934  Sub-region to coadd.
935  tempExpRefList : `list`
936  List of data reference to tempExp.
937  imageScalerList : `list`
938  List of image scalers.
939  weightList : `list`
940  List of weights.
941  altMaskList : `list`
942  List of alternate masks to use rather than those stored with
943  tempExp, or None. Each element is dict with keys = mask plane
944  name to which to add the spans.
945  statsFlags : `lsst.afw.math.Property`
946  Property object for statistic for coadd.
947  statsCtrl : `lsst.afw.math.StatisticsControl`
948  Statistics control object for coadd.
949  nImage : `lsst.afw.image.ImageU`, optional
950  Keeps track of exposure count for each pixel.
951  """
952  self.log.debug("Computing coadd over %s", bbox)
953  tempExpName = self.getTempExpDatasetName(self.warpType)
954  coaddExposure.mask.addMaskPlane("REJECTED")
955  coaddExposure.mask.addMaskPlane("CLIPPED")
956  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
957  maskMap = self.setRejectedMaskMapping(statsCtrl)
958  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
959  maskedImageList = []
960  if nImage is not None:
961  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
962  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
963 
964  if isinstance(tempExpRef, DeferredDatasetHandle):
965  # Gen 3 API
966  exposure = tempExpRef.get(parameters={'bbox': bbox})
967  else:
968  # Gen 2 API. Delete this when Gen 2 retired
969  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
970 
971  maskedImage = exposure.getMaskedImage()
972  mask = maskedImage.getMask()
973  if altMask is not None:
974  self.applyAltMaskPlanes(mask, altMask)
975  imageScaler.scaleMaskedImage(maskedImage)
976 
977  # Add 1 for each pixel which is not excluded by the exclude mask.
978  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
979  if nImage is not None:
980  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
981  if self.config.removeMaskPlanes:
982  self.removeMaskPlanes(maskedImage)
983  maskedImageList.append(maskedImage)
984 
985  if self.config.doInputMap:
986  visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
987  self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
988 
989  with self.timer("stack"):
990  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
991  clipped, # also set output to CLIPPED if sigma-clipped
992  maskMap)
993  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
994  if nImage is not None:
995  nImage.assign(subNImage, bbox)
996 
997  def removeMaskPlanes(self, maskedImage):
998  """Unset the mask of an image for mask planes specified in the config.
999 
1000  Parameters
1001  ----------
1002  maskedImage : `lsst.afw.image.MaskedImage`
1003  The masked image to be modified.
1004  """
1005  mask = maskedImage.getMask()
1006  for maskPlane in self.config.removeMaskPlanes:
1007  try:
1008  mask &= ~mask.getPlaneBitMask(maskPlane)
1009  except pexExceptions.InvalidParameterError:
1010  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
1011  maskPlane)
1012 
1013  @staticmethod
1014  def setRejectedMaskMapping(statsCtrl):
1015  """Map certain mask planes of the warps to new planes for the coadd.
1016 
1017  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1018  or CLIPPED, set it to REJECTED on the coadd.
1019  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1020  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1021 
1022  Parameters
1023  ----------
1024  statsCtrl : `lsst.afw.math.StatisticsControl`
1025  Statistics control object for coadd
1026 
1027  Returns
1028  -------
1029  maskMap : `list` of `tuple` of `int`
1030  A list of mappings of mask planes of the warped exposures to
1031  mask planes of the coadd.
1032  """
1033  edge = afwImage.Mask.getPlaneBitMask("EDGE")
1034  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1035  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1036  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1037  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1038  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1039  (clipped, clipped)]
1040  return maskMap
1041 
1042  def applyAltMaskPlanes(self, mask, altMaskSpans):
1043  """Apply in place alt mask formatted as SpanSets to a mask.
1044 
1045  Parameters
1046  ----------
1047  mask : `lsst.afw.image.Mask`
1048  Original mask.
1049  altMaskSpans : `dict`
1050  SpanSet lists to apply. Each element contains the new mask
1051  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1052  and list of SpanSets to apply to the mask.
1053 
1054  Returns
1055  -------
1056  mask : `lsst.afw.image.Mask`
1057  Updated mask.
1058  """
1059  if self.config.doUsePsfMatchedPolygons:
1060  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1061  # Clear away any other masks outside the validPolygons. These pixels are no longer
1062  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1063  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1064  # This mask-clearing step must only occur *before* applying the new masks below
1065  for spanSet in altMaskSpans['NO_DATA']:
1066  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1067 
1068  for plane, spanSetList in altMaskSpans.items():
1069  maskClipValue = mask.addMaskPlane(plane)
1070  for spanSet in spanSetList:
1071  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1072  return mask
1073 
1074  def shrinkValidPolygons(self, coaddInputs):
1075  """Shrink coaddInputs' ccds' ValidPolygons in place.
1076 
1077  Either modify each ccd's validPolygon in place, or if CoaddInputs
1078  does not have a validPolygon, create one from its bbox.
1079 
1080  Parameters
1081  ----------
1082  coaddInputs : `lsst.afw.image.coaddInputs`
1083  Original mask.
1084 
1085  """
1086  for ccd in coaddInputs.ccds:
1087  polyOrig = ccd.getValidPolygon()
1088  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1089  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1090  if polyOrig:
1091  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1092  else:
1093  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1094  ccd.setValidPolygon(validPolygon)
1095 
1096  def readBrightObjectMasks(self, dataRef):
1097  """Retrieve the bright object masks.
1098 
1099  Returns None on failure.
1100 
1101  Parameters
1102  ----------
1103  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1104  A Butler dataRef.
1105 
1106  Returns
1107  -------
1108  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1109  Bright object mask from the Butler object, or None if it cannot
1110  be retrieved.
1111  """
1112  try:
1113  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1114  except Exception as e:
1115  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1116  return None
1117 
1118  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1119  """Set the bright object masks.
1120 
1121  Parameters
1122  ----------
1123  exposure : `lsst.afw.image.Exposure`
1124  Exposure under consideration.
1125  dataId : `lsst.daf.persistence.dataId`
1126  Data identifier dict for patch.
1127  brightObjectMasks : `lsst.afw.table`
1128  Table of bright objects to mask.
1129  """
1130 
1131  if brightObjectMasks is None:
1132  self.log.warn("Unable to apply bright object mask: none supplied")
1133  return
1134  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1135  mask = exposure.getMaskedImage().getMask()
1136  wcs = exposure.getWcs()
1137  plateScale = wcs.getPixelScale().asArcseconds()
1138 
1139  for rec in brightObjectMasks:
1140  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1141  if rec["type"] == "box":
1142  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1143  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1144  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1145 
1146  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1147  bbox = geom.Box2I(center - halfSize, center + halfSize)
1148 
1149  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1150  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1151  spans = afwGeom.SpanSet(bbox)
1152  elif rec["type"] == "circle":
1153  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1154  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1155  else:
1156  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1157  continue
1158  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1159 
1160  def setInexactPsf(self, mask):
1161  """Set INEXACT_PSF mask plane.
1162 
1163  If any of the input images isn't represented in the coadd (due to
1164  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1165  these pixels.
1166 
1167  Parameters
1168  ----------
1169  mask : `lsst.afw.image.Mask`
1170  Coadded exposure's mask, modified in-place.
1171  """
1172  mask.addMaskPlane("INEXACT_PSF")
1173  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1174  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1175  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1176  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1177  array = mask.getArray()
1178  selected = array & (sensorEdge | clipped | rejected) > 0
1179  array[selected] |= inexactPsf
1180 
1181  @classmethod
1182  def _makeArgumentParser(cls):
1183  """Create an argument parser.
1184  """
1185  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1186  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1187  + cls.ConfigClass().warpType + "Warp",
1188  help="data ID, e.g. --id tract=12345 patch=1,2",
1189  ContainerClass=AssembleCoaddDataIdContainer)
1190  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1191  ContainerClass=SelectDataIdContainer)
1192  return parser
1193 
1194  @staticmethod
1195  def _subBBoxIter(bbox, subregionSize):
1196  """Iterate over subregions of a bbox.
1197 
1198  Parameters
1199  ----------
1200  bbox : `lsst.geom.Box2I`
1201  Bounding box over which to iterate.
1202  subregionSize: `lsst.geom.Extent2I`
1203  Size of sub-bboxes.
1204 
1205  Yields
1206  ------
1207  subBBox : `lsst.geom.Box2I`
1208  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1209  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1210  the edges of ``bbox``, but it will never be empty.
1211  """
1212  if bbox.isEmpty():
1213  raise RuntimeError("bbox %s is empty" % (bbox,))
1214  if subregionSize[0] < 1 or subregionSize[1] < 1:
1215  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1216 
1217  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1218  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1219  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1220  subBBox.clip(bbox)
1221  if subBBox.isEmpty():
1222  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1223  "colShift=%s, rowShift=%s" %
1224  (bbox, subregionSize, colShift, rowShift))
1225  yield subBBox
1226 
1227  def filterWarps(self, inputs, goodVisits):
1228  """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1229 
1230  Parameters
1231  ----------
1232  inputs : list
1233  List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1234  goodVisit : `dict`
1235  Dictionary with good visitIds as the keys. Value ignored.
1236 
1237  Returns:
1238  --------
1239  filteredInputs : `list`
1240  Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1241  """
1242  inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1243  filteredInputs = []
1244  for visit in goodVisits.keys():
1245  filteredInputs.append(inputWarpDict[visit])
1246  return filteredInputs
1247 
1248 
1249 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1250  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1251  """
1252 
1253  def makeDataRefList(self, namespace):
1254  """Make self.refList from self.idList.
1255 
1256  Parameters
1257  ----------
1258  namespace
1259  Results of parsing command-line (with ``butler`` and ``log`` elements).
1260  """
1261  datasetType = namespace.config.coaddName + "Coadd"
1262  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1263 
1264  for dataId in self.idList:
1265  # tract and patch are required
1266  for key in keysCoadd:
1267  if key not in dataId:
1268  raise RuntimeError("--id must include " + key)
1269 
1270  dataRef = namespace.butler.dataRef(
1271  datasetType=datasetType,
1272  dataId=dataId,
1273  )
1274  self.refList.append(dataRef)
1275 
1276 
1277 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1278  """Function to count the number of pixels with a specific mask in a
1279  footprint.
1280 
1281  Find the intersection of mask & footprint. Count all pixels in the mask
1282  that are in the intersection that have bitmask set but do not have
1283  ignoreMask set. Return the count.
1284 
1285  Parameters
1286  ----------
1287  mask : `lsst.afw.image.Mask`
1288  Mask to define intersection region by.
1289  footprint : `lsst.afw.detection.Footprint`
1290  Footprint to define the intersection region by.
1291  bitmask
1292  Specific mask that we wish to count the number of occurances of.
1293  ignoreMask
1294  Pixels to not consider.
1295 
1296  Returns
1297  -------
1298  result : `int`
1299  Count of number of pixels in footprint with specified mask.
1300  """
1301  bbox = footprint.getBBox()
1302  bbox.clip(mask.getBBox(afwImage.PARENT))
1303  fp = afwImage.Mask(bbox)
1304  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1305  footprint.spans.setMask(fp, bitmask)
1306  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1307  (subMask.getArray() & ignoreMask) == 0).sum()
1308 
1309 
1310 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1311  """Configuration parameters for the SafeClipAssembleCoaddTask.
1312  """
1313  clipDetection = pexConfig.ConfigurableField(
1314  target=SourceDetectionTask,
1315  doc="Detect sources on difference between unclipped and clipped coadd")
1316  minClipFootOverlap = pexConfig.Field(
1317  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1318  dtype=float,
1319  default=0.6
1320  )
1321  minClipFootOverlapSingle = pexConfig.Field(
1322  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1323  "clipped when only one visit overlaps",
1324  dtype=float,
1325  default=0.5
1326  )
1327  minClipFootOverlapDouble = pexConfig.Field(
1328  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1329  "clipped when two visits overlap",
1330  dtype=float,
1331  default=0.45
1332  )
1333  maxClipFootOverlapDouble = pexConfig.Field(
1334  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1335  "considering two visits",
1336  dtype=float,
1337  default=0.15
1338  )
1339  minBigOverlap = pexConfig.Field(
1340  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1341  "when labeling clipped footprints",
1342  dtype=int,
1343  default=100
1344  )
1345 
1346  def setDefaults(self):
1347  """Set default values for clipDetection.
1348 
1349  Notes
1350  -----
1351  The numeric values for these configuration parameters were
1352  empirically determined, future work may further refine them.
1353  """
1354  AssembleCoaddConfig.setDefaults(self)
1355  self.clipDetectionclipDetection.doTempLocalBackground = False
1356  self.clipDetectionclipDetection.reEstimateBackground = False
1357  self.clipDetectionclipDetection.returnOriginalFootprints = False
1358  self.clipDetectionclipDetection.thresholdPolarity = "both"
1359  self.clipDetectionclipDetection.thresholdValue = 2
1360  self.clipDetectionclipDetection.nSigmaToGrow = 2
1361  self.clipDetectionclipDetection.minPixels = 4
1362  self.clipDetectionclipDetection.isotropicGrow = True
1363  self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1364  self.sigmaClipsigmaClip = 1.5
1365  self.clipIterclipIter = 3
1366  self.statisticstatistic = "MEAN"
1367 
1368  def validate(self):
1369  if self.doSigmaClipdoSigmaClip:
1370  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1371  "Ignoring doSigmaClip.")
1372  self.doSigmaClipdoSigmaClip = False
1373  if self.statisticstatistic != "MEAN":
1374  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1375  "(%s chosen). Please set statistic to MEAN."
1376  % (self.statisticstatistic))
1377  AssembleCoaddTask.ConfigClass.validate(self)
1378 
1379 
1380 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1381  """Assemble a coadded image from a set of coadded temporary exposures,
1382  being careful to clip & flag areas with potential artifacts.
1383 
1384  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1385  we clip outliers). The problem with doing this is that when computing the
1386  coadd PSF at a given location, individual visit PSFs from visits with
1387  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1388  In this task, we correct for this behavior by creating a new
1389  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1390  coaddTempExps and the final coadd where
1391 
1392  i. difference imaging suggests that there is an outlier and
1393  ii. this outlier appears on only one or two images.
1394 
1395  Such regions will not contribute to the final coadd. Furthermore, any
1396  routine to determine the coadd PSF can now be cognizant of clipped regions.
1397  Note that the algorithm implemented by this task is preliminary and works
1398  correctly for HSC data. Parameter modifications and or considerable
1399  redesigning of the algorithm is likley required for other surveys.
1400 
1401  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1402  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1403  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1404  if you wish.
1405 
1406  Notes
1407  -----
1408  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1409  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1410  see `baseDebug` for more about ``debug.py`` files.
1411  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1412  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1413  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1414  for further information.
1415 
1416  Examples
1417  --------
1418  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1419  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1420  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1421 
1422  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1423  and filter to be coadded (specified using
1424  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1425  along with a list of coaddTempExps to attempt to coadd (specified using
1426  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1427  Only the coaddTempExps that cover the specified tract and patch will be
1428  coadded. A list of the available optional arguments can be obtained by
1429  calling assembleCoadd.py with the --help command line argument:
1430 
1431  .. code-block:: none
1432 
1433  assembleCoadd.py --help
1434 
1435  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1436  context of multi-band processing, we will generate the HSC-I & -R band
1437  coadds from HSC engineering test data provided in the ci_hsc package.
1438  To begin, assuming that the lsst stack has been already set up, we must
1439  set up the obs_subaru and ci_hsc packages. This defines the environment
1440  variable $CI_HSC_DIR and points at the location of the package. The raw
1441  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1442  the coadds, we must first
1443 
1444  - ``processCcd``
1445  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1446  - ``makeSkyMap``
1447  create a skymap that covers the area of the sky present in the raw exposures
1448  - ``makeCoaddTempExp``
1449  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1450 
1451  We can perform all of these steps by running
1452 
1453  .. code-block:: none
1454 
1455  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1456 
1457  This will produce warped coaddTempExps for each visit. To coadd the
1458  warped data, we call ``assembleCoadd.py`` as follows:
1459 
1460  .. code-block:: none
1461 
1462  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1463  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1464  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1465  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1466  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1467  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1468  --selectId visit=903988 ccd=24
1469 
1470  This will process the HSC-I band data. The results are written in
1471  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1472 
1473  You may also choose to run:
1474 
1475  .. code-block:: none
1476 
1477  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1478  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1479  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1480  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1481  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1482  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1483  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1484  --selectId visit=903346 ccd=12
1485 
1486  to generate the coadd for the HSC-R band if you are interested in following
1487  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1488  """
1489  ConfigClass = SafeClipAssembleCoaddConfig
1490  _DefaultName = "safeClipAssembleCoadd"
1491 
1492  def __init__(self, *args, **kwargs):
1493  AssembleCoaddTask.__init__(self, *args, **kwargs)
1494  schema = afwTable.SourceTable.makeMinimalSchema()
1495  self.makeSubtask("clipDetection", schema=schema)
1496 
1497  @utils.inheritDoc(AssembleCoaddTask)
1498  @pipeBase.timeMethod
1499  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1500  """Assemble the coadd for a region.
1501 
1502  Compute the difference of coadds created with and without outlier
1503  rejection to identify coadd pixels that have outlier values in some
1504  individual visits.
1505  Detect clipped regions on the difference image and mark these regions
1506  on the one or two individual coaddTempExps where they occur if there
1507  is significant overlap between the clipped region and a source. This
1508  leaves us with a set of footprints from the difference image that have
1509  been identified as having occured on just one or two individual visits.
1510  However, these footprints were generated from a difference image. It
1511  is conceivable for a large diffuse source to have become broken up
1512  into multiple footprints acrosss the coadd difference in this process.
1513  Determine the clipped region from all overlapping footprints from the
1514  detected sources in each visit - these are big footprints.
1515  Combine the small and big clipped footprints and mark them on a new
1516  bad mask plane.
1517  Generate the coadd using `AssembleCoaddTask.run` without outlier
1518  removal. Clipped footprints will no longer make it into the coadd
1519  because they are marked in the new bad mask plane.
1520 
1521  Notes
1522  -----
1523  args and kwargs are passed but ignored in order to match the call
1524  signature expected by the parent task.
1525  """
1526  exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1527  mask = exp.getMaskedImage().getMask()
1528  mask.addMaskPlane("CLIPPED")
1529 
1530  result = self.detectClipdetectClip(exp, tempExpRefList)
1531 
1532  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1533 
1534  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1535  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1536  # Append big footprints from individual Warps to result.clipSpans
1537  bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1538  result.detectionFootprints, maskClipValue, maskDetValue,
1539  exp.getBBox())
1540  # Create mask of the current clipped footprints
1541  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1542  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1543 
1544  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1545  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1546  maskClip |= maskClipBig
1547 
1548  # Assemble coadd from base class, but ignoring CLIPPED pixels
1549  badMaskPlanes = self.config.badMaskPlanes[:]
1550  badMaskPlanes.append("CLIPPED")
1551  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1552  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1553  result.clipSpans, mask=badPixelMask)
1554 
1555  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1556  """Return an exposure that contains the difference between unclipped
1557  and clipped coadds.
1558 
1559  Generate a difference image between clipped and unclipped coadds.
1560  Compute the difference image by subtracting an outlier-clipped coadd
1561  from an outlier-unclipped coadd. Return the difference image.
1562 
1563  Parameters
1564  ----------
1565  skyInfo : `lsst.pipe.base.Struct`
1566  Patch geometry information, from getSkyInfo
1567  tempExpRefList : `list`
1568  List of data reference to tempExp
1569  imageScalerList : `list`
1570  List of image scalers
1571  weightList : `list`
1572  List of weights
1573 
1574  Returns
1575  -------
1576  exp : `lsst.afw.image.Exposure`
1577  Difference image of unclipped and clipped coadd wrapped in an Exposure
1578  """
1579  config = AssembleCoaddConfig()
1580  # getattr necessary because subtasks do not survive Config.toDict()
1581  # exclude connections because the class of self.config.connections is not
1582  # the same as AssembleCoaddConfig.connections, and the connections are not
1583  # needed to run this task anyway.
1584  configIntersection = {k: getattr(self.config, k)
1585  for k, v in self.config.toDict().items()
1586  if (k in config.keys() and k != "connections")}
1587  configIntersection['doInputMap'] = False
1588  configIntersection['doNImage'] = False
1589  config.update(**configIntersection)
1590 
1591  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1592  config.statistic = 'MEAN'
1593  task = AssembleCoaddTask(config=config)
1594  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1595 
1596  config.statistic = 'MEANCLIP'
1597  task = AssembleCoaddTask(config=config)
1598  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1599 
1600  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1601  coaddDiff -= coaddClip.getMaskedImage()
1602  exp = afwImage.ExposureF(coaddDiff)
1603  exp.setPsf(coaddMean.getPsf())
1604  return exp
1605 
1606  def detectClip(self, exp, tempExpRefList):
1607  """Detect clipped regions on an exposure and set the mask on the
1608  individual tempExp masks.
1609 
1610  Detect footprints in the difference image after smoothing the
1611  difference image with a Gaussian kernal. Identify footprints that
1612  overlap with one or two input ``coaddTempExps`` by comparing the
1613  computed overlap fraction to thresholds set in the config. A different
1614  threshold is applied depending on the number of overlapping visits
1615  (restricted to one or two). If the overlap exceeds the thresholds,
1616  the footprint is considered "CLIPPED" and is marked as such on the
1617  coaddTempExp. Return a struct with the clipped footprints, the indices
1618  of the ``coaddTempExps`` that end up overlapping with the clipped
1619  footprints, and a list of new masks for the ``coaddTempExps``.
1620 
1621  Parameters
1622  ----------
1623  exp : `lsst.afw.image.Exposure`
1624  Exposure to run detection on.
1625  tempExpRefList : `list`
1626  List of data reference to tempExp.
1627 
1628  Returns
1629  -------
1630  result : `lsst.pipe.base.Struct`
1631  Result struct with components:
1632 
1633  - ``clipFootprints``: list of clipped footprints.
1634  - ``clipIndices``: indices for each ``clippedFootprint`` in
1635  ``tempExpRefList``.
1636  - ``clipSpans``: List of dictionaries containing spanSet lists
1637  to clip. Each element contains the new maskplane name
1638  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1639  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1640  compressed into footprints.
1641  """
1642  mask = exp.getMaskedImage().getMask()
1643  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1644  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1645  # Merge positive and negative together footprints together
1646  fpSet.positive.merge(fpSet.negative)
1647  footprints = fpSet.positive
1648  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1649  ignoreMask = self.getBadPixelMask()
1650 
1651  clipFootprints = []
1652  clipIndices = []
1653  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1654 
1655  # for use by detectClipBig
1656  visitDetectionFootprints = []
1657 
1658  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1659  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1660  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1661 
1662  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1663  for i, warpRef in enumerate(tempExpRefList):
1664  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1665  immediate=True).getMaskedImage().getMask()
1666  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1667  afwImage.PARENT, True)
1668  maskVisitDet &= maskDetValue
1669  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1670  visitDetectionFootprints.append(visitFootprints)
1671 
1672  for j, footprint in enumerate(footprints.getFootprints()):
1673  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1674  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1675 
1676  # build a list of clipped spans for each visit
1677  for j, footprint in enumerate(footprints.getFootprints()):
1678  nPixel = footprint.getArea()
1679  overlap = [] # hold the overlap with each visit
1680  indexList = [] # index of visit in global list
1681  for i in range(len(tempExpRefList)):
1682  ignore = ignoreArr[i, j]
1683  overlapDet = overlapDetArr[i, j]
1684  totPixel = nPixel - ignore
1685 
1686  # If we have more bad pixels than detection skip
1687  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1688  continue
1689  overlap.append(overlapDet/float(totPixel))
1690  indexList.append(i)
1691 
1692  overlap = numpy.array(overlap)
1693  if not len(overlap):
1694  continue
1695 
1696  keep = False # Should this footprint be marked as clipped?
1697  keepIndex = [] # Which tempExps does the clipped footprint belong to
1698 
1699  # If footprint only has one overlap use a lower threshold
1700  if len(overlap) == 1:
1701  if overlap[0] > self.config.minClipFootOverlapSingle:
1702  keep = True
1703  keepIndex = [0]
1704  else:
1705  # This is the general case where only visit should be clipped
1706  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1707  if len(clipIndex) == 1:
1708  keep = True
1709  keepIndex = [clipIndex[0]]
1710 
1711  # Test if there are clipped objects that overlap two different visits
1712  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1713  if len(clipIndex) == 2 and len(overlap) > 3:
1714  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1715  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1716  keep = True
1717  keepIndex = clipIndex
1718 
1719  if not keep:
1720  continue
1721 
1722  for index in keepIndex:
1723  globalIndex = indexList[index]
1724  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1725 
1726  clipIndices.append(numpy.array(indexList)[keepIndex])
1727  clipFootprints.append(footprint)
1728 
1729  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1730  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1731 
1732  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1733  maskClipValue, maskDetValue, coaddBBox):
1734  """Return individual warp footprints for large artifacts and append
1735  them to ``clipList`` in place.
1736 
1737  Identify big footprints composed of many sources in the coadd
1738  difference that may have originated in a large diffuse source in the
1739  coadd. We do this by indentifying all clipped footprints that overlap
1740  significantly with each source in all the coaddTempExps.
1741 
1742  Parameters
1743  ----------
1744  clipList : `list`
1745  List of alt mask SpanSets with clipping information. Modified.
1746  clipFootprints : `list`
1747  List of clipped footprints.
1748  clipIndices : `list`
1749  List of which entries in tempExpClipList each footprint belongs to.
1750  maskClipValue
1751  Mask value of clipped pixels.
1752  maskDetValue
1753  Mask value of detected pixels.
1754  coaddBBox : `lsst.geom.Box`
1755  BBox of the coadd and warps.
1756 
1757  Returns
1758  -------
1759  bigFootprintsCoadd : `list`
1760  List of big footprints
1761  """
1762  bigFootprintsCoadd = []
1763  ignoreMask = self.getBadPixelMask()
1764  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1765  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1766  for footprint in visitFootprints.getFootprints():
1767  footprint.spans.setMask(maskVisitDet, maskDetValue)
1768 
1769  # build a mask of clipped footprints that are in this visit
1770  clippedFootprintsVisit = []
1771  for foot, clipIndex in zip(clipFootprints, clipIndices):
1772  if index not in clipIndex:
1773  continue
1774  clippedFootprintsVisit.append(foot)
1775  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1776  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1777 
1778  bigFootprintsVisit = []
1779  for foot in visitFootprints.getFootprints():
1780  if foot.getArea() < self.config.minBigOverlap:
1781  continue
1782  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1783  if nCount > self.config.minBigOverlap:
1784  bigFootprintsVisit.append(foot)
1785  bigFootprintsCoadd.append(foot)
1786 
1787  for footprint in bigFootprintsVisit:
1788  clippedSpans["CLIPPED"].append(footprint.spans)
1789 
1790  return bigFootprintsCoadd
1791 
1792 
1794  psfMatchedWarps = pipeBase.connectionTypes.Input(
1795  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1796  "Only PSF-Matched Warps make sense for image subtraction. "
1797  "Therefore, they must be an additional declared input."),
1798  name="{inputCoaddName}Coadd_psfMatchedWarp",
1799  storageClass="ExposureF",
1800  dimensions=("tract", "patch", "skymap", "visit"),
1801  deferLoad=True,
1802  multiple=True
1803  )
1804  templateCoadd = pipeBase.connectionTypes.Output(
1805  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1806  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1807  name="{outputCoaddName}CoaddPsfMatched",
1808  storageClass="ExposureF",
1809  dimensions=("tract", "patch", "skymap", "band"),
1810  )
1811 
1812  def __init__(self, *, config=None):
1813  super().__init__(config=config)
1814  if not config.assembleStaticSkyModel.doWrite:
1815  self.outputs.remove("templateCoadd")
1816  config.validate()
1817 
1818 
1819 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1820  pipelineConnections=CompareWarpAssembleCoaddConnections):
1821  assembleStaticSkyModel = pexConfig.ConfigurableField(
1822  target=AssembleCoaddTask,
1823  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1824  " naive/first-iteration model of the static sky.",
1825  )
1826  detect = pexConfig.ConfigurableField(
1827  target=SourceDetectionTask,
1828  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1829  )
1830  detectTemplate = pexConfig.ConfigurableField(
1831  target=SourceDetectionTask,
1832  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1833  )
1834  maskStreaks = pexConfig.ConfigurableField(
1835  target=MaskStreaksTask,
1836  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1837  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1838  "streakMaskName"
1839  )
1840  streakMaskName = pexConfig.Field(
1841  dtype=str,
1842  default="STREAK",
1843  doc="Name of mask bit used for streaks"
1844  )
1845  maxNumEpochs = pexConfig.Field(
1846  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1847  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1848  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1849  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1850  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1851  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1852  "than transient and not masked.",
1853  dtype=int,
1854  default=2
1855  )
1856  maxFractionEpochsLow = pexConfig.RangeField(
1857  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1858  "Effective maxNumEpochs = "
1859  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1860  dtype=float,
1861  default=0.4,
1862  min=0., max=1.,
1863  )
1864  maxFractionEpochsHigh = pexConfig.RangeField(
1865  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1866  "Effective maxNumEpochs = "
1867  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1868  dtype=float,
1869  default=0.03,
1870  min=0., max=1.,
1871  )
1872  spatialThreshold = pexConfig.RangeField(
1873  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1874  "temporal criteria. If 0, clip all. If 1, clip none.",
1875  dtype=float,
1876  default=0.5,
1877  min=0., max=1.,
1878  inclusiveMin=True, inclusiveMax=True
1879  )
1880  doScaleWarpVariance = pexConfig.Field(
1881  doc="Rescale Warp variance plane using empirical noise?",
1882  dtype=bool,
1883  default=True,
1884  )
1885  scaleWarpVariance = pexConfig.ConfigurableField(
1886  target=ScaleVarianceTask,
1887  doc="Rescale variance on warps",
1888  )
1889  doPreserveContainedBySource = pexConfig.Field(
1890  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1891  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1892  dtype=bool,
1893  default=True,
1894  )
1895  doPrefilterArtifacts = pexConfig.Field(
1896  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1897  "because they will be excluded anyway. This prevents them from contributing "
1898  "to the outlier epoch count image and potentially being labeled as persistant."
1899  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1900  dtype=bool,
1901  default=True
1902  )
1903  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1904  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1905  dtype=str,
1906  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1907  )
1908  prefilterArtifactsRatio = pexConfig.Field(
1909  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1910  dtype=float,
1911  default=0.05
1912  )
1913  doFilterMorphological = pexConfig.Field(
1914  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1915  "be streaks.",
1916  dtype=bool,
1917  default=False
1918  )
1919 
1920  def setDefaults(self):
1921  AssembleCoaddConfig.setDefaults(self)
1922  self.statisticstatistic = 'MEAN'
1923  self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
1924 
1925  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1926  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1927  if "EDGE" in self.badMaskPlanes:
1928  self.badMaskPlanes.remove('EDGE')
1929  self.removeMaskPlanes.append('EDGE')
1930  self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1931  self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
1932  self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
1933  self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
1934  self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
1935  self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
1936  self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
1937  self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
1938  self.detectdetect.doTempLocalBackground = False
1939  self.detectdetect.reEstimateBackground = False
1940  self.detectdetect.returnOriginalFootprints = False
1941  self.detectdetect.thresholdPolarity = "both"
1942  self.detectdetect.thresholdValue = 5
1943  self.detectdetect.minPixels = 4
1944  self.detectdetect.isotropicGrow = True
1945  self.detectdetect.thresholdType = "pixel_stdev"
1946  self.detectdetect.nSigmaToGrow = 0.4
1947  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1948  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1949  self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
1950  self.detectTemplatedetectTemplate.doTempLocalBackground = False
1951  self.detectTemplatedetectTemplate.reEstimateBackground = False
1952  self.detectTemplatedetectTemplate.returnOriginalFootprints = False
1953 
1954  def validate(self):
1955  super().validate()
1956  if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
1957  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1958  "Please set assembleStaticSkyModel.doNImage=False")
1959 
1960  if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
1961  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1962  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1963  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1964  "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
1965 
1966 
1967 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1968  """Assemble a compareWarp coadded image from a set of warps
1969  by masking artifacts detected by comparing PSF-matched warps.
1970 
1971  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1972  we clip outliers). The problem with doing this is that when computing the
1973  coadd PSF at a given location, individual visit PSFs from visits with
1974  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1975  In this task, we correct for this behavior by creating a new badMaskPlane
1976  'CLIPPED' which marks pixels in the individual warps suspected to contain
1977  an artifact. We populate this plane on the input warps by comparing
1978  PSF-matched warps with a PSF-matched median coadd which serves as a
1979  model of the static sky. Any group of pixels that deviates from the
1980  PSF-matched template coadd by more than config.detect.threshold sigma,
1981  is an artifact candidate. The candidates are then filtered to remove
1982  variable sources and sources that are difficult to subtract such as
1983  bright stars. This filter is configured using the config parameters
1984  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1985  the maximum fraction of epochs that the deviation can appear in and still
1986  be considered an artifact. The spatialThreshold is the maximum fraction of
1987  pixels in the footprint of the deviation that appear in other epochs
1988  (where other epochs is defined by the temporalThreshold). If the deviant
1989  region meets this criteria of having a significant percentage of pixels
1990  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1991  set in the mask. These regions will not contribute to the final coadd.
1992  Furthermore, any routine to determine the coadd PSF can now be cognizant
1993  of clipped regions. Note that the algorithm implemented by this task is
1994  preliminary and works correctly for HSC data. Parameter modifications and
1995  or considerable redesigning of the algorithm is likley required for other
1996  surveys.
1997 
1998  ``CompareWarpAssembleCoaddTask`` sub-classes
1999  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
2000  as a subtask to generate the TemplateCoadd (the model of the static sky).
2001 
2002  Notes
2003  -----
2004  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2005  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2006  ``baseDebug`` for more about ``debug.py`` files.
2007 
2008  This task supports the following debug variables:
2009 
2010  - ``saveCountIm``
2011  If True then save the Epoch Count Image as a fits file in the `figPath`
2012  - ``figPath``
2013  Path to save the debug fits images and figures
2014 
2015  For example, put something like:
2016 
2017  .. code-block:: python
2018 
2019  import lsstDebug
2020  def DebugInfo(name):
2021  di = lsstDebug.getInfo(name)
2022  if name == "lsst.pipe.tasks.assembleCoadd":
2023  di.saveCountIm = True
2024  di.figPath = "/desired/path/to/debugging/output/images"
2025  return di
2026  lsstDebug.Info = DebugInfo
2027 
2028  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2029  ``--debug`` flag. Some subtasks may have their own debug variables;
2030  see individual Task documentation.
2031 
2032  Examples
2033  --------
2034  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2035  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2036  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2037  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2038  and filter to be coadded (specified using
2039  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2040  along with a list of coaddTempExps to attempt to coadd (specified using
2041  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2042  Only the warps that cover the specified tract and patch will be coadded.
2043  A list of the available optional arguments can be obtained by calling
2044  ``assembleCoadd.py`` with the ``--help`` command line argument:
2045 
2046  .. code-block:: none
2047 
2048  assembleCoadd.py --help
2049 
2050  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2051  context of multi-band processing, we will generate the HSC-I & -R band
2052  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2053  To begin, assuming that the lsst stack has been already set up, we must
2054  set up the ``obs_subaru`` and ``ci_hsc`` packages.
2055  This defines the environment variable ``$CI_HSC_DIR`` and points at the
2056  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2057  directory. To begin assembling the coadds, we must first
2058 
2059  - processCcd
2060  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2061  - makeSkyMap
2062  create a skymap that covers the area of the sky present in the raw exposures
2063  - makeCoaddTempExp
2064  warp the individual calibrated exposures to the tangent plane of the coadd
2065 
2066  We can perform all of these steps by running
2067 
2068  .. code-block:: none
2069 
2070  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2071 
2072  This will produce warped ``coaddTempExps`` for each visit. To coadd the
2073  warped data, we call ``assembleCoadd.py`` as follows:
2074 
2075  .. code-block:: none
2076 
2077  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2078  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2079  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2080  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2081  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2082  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2083  --selectId visit=903988 ccd=24
2084 
2085  This will process the HSC-I band data. The results are written in
2086  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2087  """
2088  ConfigClass = CompareWarpAssembleCoaddConfig
2089  _DefaultName = "compareWarpAssembleCoadd"
2090 
2091  def __init__(self, *args, **kwargs):
2092  AssembleCoaddTask.__init__(self, *args, **kwargs)
2093  self.makeSubtask("assembleStaticSkyModel")
2094  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2095  self.makeSubtask("detect", schema=detectionSchema)
2096  if self.config.doPreserveContainedBySource:
2097  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2098  if self.config.doScaleWarpVariance:
2099  self.makeSubtask("scaleWarpVariance")
2100  if self.config.doFilterMorphological:
2101  self.makeSubtask("maskStreaks")
2102 
2103  @utils.inheritDoc(AssembleCoaddTask)
2104  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2105  """
2106  Generate a templateCoadd to use as a naive model of static sky to
2107  subtract from PSF-Matched warps.
2108 
2109  Returns
2110  -------
2111  result : `lsst.pipe.base.Struct`
2112  Result struct with components:
2113 
2114  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2115  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2116  """
2117  # Ensure that psfMatchedWarps are used as input warps for template generation
2118  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2119  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2120 
2121  # Because subtasks don't have connections we have to make one.
2122  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2123  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2124  if self.config.assembleStaticSkyModel.doWrite:
2125  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2126  # Remove template coadd from both subtask's and main tasks outputs,
2127  # because it is handled by the subtask as `coaddExposure`
2128  del outputRefs.templateCoadd
2129  del staticSkyModelOutputRefs.templateCoadd
2130 
2131  # A PSF-Matched nImage does not exist as a dataset type
2132  if 'nImage' in staticSkyModelOutputRefs.keys():
2133  del staticSkyModelOutputRefs.nImage
2134 
2135  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2136  staticSkyModelOutputRefs)
2137  if templateCoadd is None:
2138  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2139 
2140  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2141  nImage=templateCoadd.nImage,
2142  warpRefList=templateCoadd.warpRefList,
2143  imageScalerList=templateCoadd.imageScalerList,
2144  weightList=templateCoadd.weightList)
2145 
2146  @utils.inheritDoc(AssembleCoaddTask)
2147  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2148  """
2149  Generate a templateCoadd to use as a naive model of static sky to
2150  subtract from PSF-Matched warps.
2151 
2152  Returns
2153  -------
2154  result : `lsst.pipe.base.Struct`
2155  Result struct with components:
2156 
2157  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2158  - ``nImage``: N Image (``lsst.afw.image.Image``)
2159  """
2160  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2161  if templateCoadd is None:
2162  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2163 
2164  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2165  nImage=templateCoadd.nImage,
2166  warpRefList=templateCoadd.warpRefList,
2167  imageScalerList=templateCoadd.imageScalerList,
2168  weightList=templateCoadd.weightList)
2169 
2170  def _noTemplateMessage(self, warpType):
2171  warpName = (warpType[0].upper() + warpType[1:])
2172  message = """No %(warpName)s warps were found to build the template coadd which is
2173  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2174  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2175  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2176 
2177  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2178  another algorithm like:
2179 
2180  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2181  config.assemble.retarget(SafeClipAssembleCoaddTask)
2182  """ % {"warpName": warpName}
2183  return message
2184 
2185  @utils.inheritDoc(AssembleCoaddTask)
2186  @pipeBase.timeMethod
2187  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2188  supplementaryData, *args, **kwargs):
2189  """Assemble the coadd.
2190 
2191  Find artifacts and apply them to the warps' masks creating a list of
2192  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2193  plane. Then pass these alternative masks to the base class's `run`
2194  method.
2195 
2196  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2197  that must contain a ``templateCoadd`` that serves as the
2198  model of the static sky.
2199  """
2200 
2201  # Check and match the order of the supplementaryData
2202  # (PSF-matched) inputs to the order of the direct inputs,
2203  # so that the artifact mask is applied to the right warp
2204  dataIds = [ref.dataId for ref in tempExpRefList]
2205  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2206 
2207  if dataIds != psfMatchedDataIds:
2208  self.log.info("Reordering and or/padding PSF-matched visit input list")
2209  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2210  psfMatchedDataIds, dataIds)
2211  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2212  psfMatchedDataIds, dataIds)
2213 
2214  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2215  spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2216  supplementaryData.warpRefList,
2217  supplementaryData.imageScalerList)
2218 
2219  badMaskPlanes = self.config.badMaskPlanes[:]
2220  badMaskPlanes.append("CLIPPED")
2221  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2222 
2223  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2224  spanSetMaskList, mask=badPixelMask)
2225 
2226  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2227  # Psf-Matching moves the real edge inwards
2228  self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2229  return result
2230 
2231  def applyAltEdgeMask(self, mask, altMaskList):
2232  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2233 
2234  Parameters
2235  ----------
2236  mask : `lsst.afw.image.Mask`
2237  Original mask.
2238  altMaskList : `list`
2239  List of Dicts containing ``spanSet`` lists.
2240  Each element contains the new mask plane name (e.g. "CLIPPED
2241  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2242  the mask.
2243  """
2244  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2245  for visitMask in altMaskList:
2246  if "EDGE" in visitMask:
2247  for spanSet in visitMask['EDGE']:
2248  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2249 
2250  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2251  """Find artifacts.
2252 
2253  Loop through warps twice. The first loop builds a map with the count
2254  of how many epochs each pixel deviates from the templateCoadd by more
2255  than ``config.chiThreshold`` sigma. The second loop takes each
2256  difference image and filters the artifacts detected in each using
2257  count map to filter out variable sources and sources that are
2258  difficult to subtract cleanly.
2259 
2260  Parameters
2261  ----------
2262  templateCoadd : `lsst.afw.image.Exposure`
2263  Exposure to serve as model of static sky.
2264  tempExpRefList : `list`
2265  List of data references to warps.
2266  imageScalerList : `list`
2267  List of image scalers.
2268 
2269  Returns
2270  -------
2271  altMasks : `list`
2272  List of dicts containing information about CLIPPED
2273  (i.e., artifacts), NO_DATA, and EDGE pixels.
2274  """
2275 
2276  self.log.debug("Generating Count Image, and mask lists.")
2277  coaddBBox = templateCoadd.getBBox()
2278  slateIm = afwImage.ImageU(coaddBBox)
2279  epochCountImage = afwImage.ImageU(coaddBBox)
2280  nImage = afwImage.ImageU(coaddBBox)
2281  spanSetArtifactList = []
2282  spanSetNoDataMaskList = []
2283  spanSetEdgeList = []
2284  spanSetBadMorphoList = []
2285  badPixelMask = self.getBadPixelMask()
2286 
2287  # mask of the warp diffs should = that of only the warp
2288  templateCoadd.mask.clearAllMaskPlanes()
2289 
2290  if self.config.doPreserveContainedBySource:
2291  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2292  else:
2293  templateFootprints = None
2294 
2295  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2296  warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2297  if warpDiffExp is not None:
2298  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2299  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2300  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2301  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2302  fpSet.positive.merge(fpSet.negative)
2303  footprints = fpSet.positive
2304  slateIm.set(0)
2305  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2306 
2307  # Remove artifacts due to defects before they contribute to the epochCountImage
2308  if self.config.doPrefilterArtifacts:
2309  spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2310 
2311  # Clear mask before adding prefiltered spanSets
2312  self.detect.clearMask(warpDiffExp.mask)
2313  for spans in spanSetList:
2314  spans.setImage(slateIm, 1, doClip=True)
2315  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2316  epochCountImage += slateIm
2317 
2318  if self.config.doFilterMorphological:
2319  maskName = self.config.streakMaskName
2320  _ = self.maskStreaks.run(warpDiffExp)
2321  streakMask = warpDiffExp.mask
2322  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2323  streakMask.getPlaneBitMask(maskName)).split()
2324 
2325  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2326  # undergo a second convolution. Pixels with data in the direct warp
2327  # but not in the PSF-matched warp will not have their artifacts detected.
2328  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2329  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2330  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2331  nansMask.setXY0(warpDiffExp.getXY0())
2332  edgeMask = warpDiffExp.mask
2333  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2334  edgeMask.getPlaneBitMask("EDGE")).split()
2335  else:
2336  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2337  # In this case, mask the whole epoch
2338  nansMask = afwImage.MaskX(coaddBBox, 1)
2339  spanSetList = []
2340  spanSetEdgeMask = []
2341  spanSetStreak = []
2342 
2343  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2344 
2345  spanSetNoDataMaskList.append(spanSetNoDataMask)
2346  spanSetArtifactList.append(spanSetList)
2347  spanSetEdgeList.append(spanSetEdgeMask)
2348  if self.config.doFilterMorphological:
2349  spanSetBadMorphoList.append(spanSetStreak)
2350 
2351  if lsstDebug.Info(__name__).saveCountIm:
2352  path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2353  epochCountImage.writeFits(path)
2354 
2355  for i, spanSetList in enumerate(spanSetArtifactList):
2356  if spanSetList:
2357  filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2358  templateFootprints)
2359  spanSetArtifactList[i] = filteredSpanSetList
2360  if self.config.doFilterMorphological:
2361  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2362 
2363  altMasks = []
2364  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2365  altMasks.append({'CLIPPED': artifacts,
2366  'NO_DATA': noData,
2367  'EDGE': edge})
2368  return altMasks
2369 
2370  def prefilterArtifacts(self, spanSetList, exp):
2371  """Remove artifact candidates covered by bad mask plane.
2372 
2373  Any future editing of the candidate list that does not depend on
2374  temporal information should go in this method.
2375 
2376  Parameters
2377  ----------
2378  spanSetList : `list`
2379  List of SpanSets representing artifact candidates.
2380  exp : `lsst.afw.image.Exposure`
2381  Exposure containing mask planes used to prefilter.
2382 
2383  Returns
2384  -------
2385  returnSpanSetList : `list`
2386  List of SpanSets with artifacts.
2387  """
2388  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2389  goodArr = (exp.mask.array & badPixelMask) == 0
2390  returnSpanSetList = []
2391  bbox = exp.getBBox()
2392  x0, y0 = exp.getXY0()
2393  for i, span in enumerate(spanSetList):
2394  y, x = span.clippedTo(bbox).indices()
2395  yIndexLocal = numpy.array(y) - y0
2396  xIndexLocal = numpy.array(x) - x0
2397  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2398  if goodRatio > self.config.prefilterArtifactsRatio:
2399  returnSpanSetList.append(span)
2400  return returnSpanSetList
2401 
2402  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2403  """Filter artifact candidates.
2404 
2405  Parameters
2406  ----------
2407  spanSetList : `list`
2408  List of SpanSets representing artifact candidates.
2409  epochCountImage : `lsst.afw.image.Image`
2410  Image of accumulated number of warpDiff detections.
2411  nImage : `lsst.afw.image.Image`
2412  Image of the accumulated number of total epochs contributing.
2413 
2414  Returns
2415  -------
2416  maskSpanSetList : `list`
2417  List of SpanSets with artifacts.
2418  """
2419 
2420  maskSpanSetList = []
2421  x0, y0 = epochCountImage.getXY0()
2422  for i, span in enumerate(spanSetList):
2423  y, x = span.indices()
2424  yIdxLocal = [y1 - y0 for y1 in y]
2425  xIdxLocal = [x1 - x0 for x1 in x]
2426  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2427  totalN = nImage.array[yIdxLocal, xIdxLocal]
2428 
2429  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2430  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2431  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2432  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2433  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2434  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2435  & (outlierN <= effectiveMaxNumEpochs))
2436  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2437  if percentBelowThreshold > self.config.spatialThreshold:
2438  maskSpanSetList.append(span)
2439 
2440  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2441  # If a candidate is contained by a footprint on the template coadd, do not clip
2442  filteredMaskSpanSetList = []
2443  for span in maskSpanSetList:
2444  doKeep = True
2445  for footprint in footprintsToExclude.positive.getFootprints():
2446  if footprint.spans.contains(span):
2447  doKeep = False
2448  break
2449  if doKeep:
2450  filteredMaskSpanSetList.append(span)
2451  maskSpanSetList = filteredMaskSpanSetList
2452 
2453  return maskSpanSetList
2454 
2455  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2456  """Fetch a warp from the butler and return a warpDiff.
2457 
2458  Parameters
2459  ----------
2460  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2461  Butler dataRef for the warp.
2462  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2463  An image scaler object.
2464  templateCoadd : `lsst.afw.image.Exposure`
2465  Exposure to be substracted from the scaled warp.
2466 
2467  Returns
2468  -------
2469  warp : `lsst.afw.image.Exposure`
2470  Exposure of the image difference between the warp and template.
2471  """
2472 
2473  # If the PSF-Matched warp did not exist for this direct warp
2474  # None is holding its place to maintain order in Gen 3
2475  if warpRef is None:
2476  return None
2477  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2478  warpName = self.getTempExpDatasetName('psfMatched')
2479  if not isinstance(warpRef, DeferredDatasetHandle):
2480  if not warpRef.datasetExists(warpName):
2481  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2482  return None
2483  warp = warpRef.get(datasetType=warpName, immediate=True)
2484  # direct image scaler OK for PSF-matched Warp
2485  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2486  mi = warp.getMaskedImage()
2487  if self.config.doScaleWarpVariance:
2488  try:
2489  self.scaleWarpVariance.run(mi)
2490  except Exception as exc:
2491  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2492  mi -= templateCoadd.getMaskedImage()
2493  return warp
2494 
2495  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2496  """Return a path to which to write debugging output.
2497 
2498  Creates a hyphen-delimited string of dataId values for simple filenames.
2499 
2500  Parameters
2501  ----------
2502  prefix : `str`
2503  Prefix for filename.
2504  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2505  Butler dataRef to make the path from.
2506  coaddLevel : `bool`, optional.
2507  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2508  'filter', but no 'visit').
2509 
2510  Returns
2511  -------
2512  result : `str`
2513  Path for debugging output.
2514  """
2515  if coaddLevel:
2516  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2517  else:
2518  keys = warpRef.dataId.keys()
2519  keyList = sorted(keys, reverse=True)
2520  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2521  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2522  return os.path.join(directory, filename)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:141
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:346
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60