lsst.pipe.tasks  21.0.0-80-g62ad60b1+450980b591
assembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import copy
24 import numpy
25 import warnings
26 import lsst.pex.config as pexConfig
27 import lsst.pex.exceptions as pexExceptions
28 import lsst.geom as geom
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.afw.detection as afwDet
34 import lsst.coadd.utils as coaddUtils
35 import lsst.pipe.base as pipeBase
36 import lsst.meas.algorithms as measAlg
37 import lsst.log as log
38 import lsstDebug
39 import lsst.utils as utils
40 from lsst.skymap import BaseSkyMap
41 from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42 from .interpImage import InterpImageTask
43 from .scaleZeroPoint import ScaleZeroPointTask
44 from .coaddHelpers import groupPatchExposures, getGroupDataRef
45 from .scaleVariance import ScaleVarianceTask
46 from .maskStreaks import MaskStreaksTask
47 from .healSparseMapping import HealSparseInputMapTask
48 from lsst.meas.algorithms import SourceDetectionTask
49 from lsst.daf.butler import DeferredDatasetHandle
50 
51 __all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
52  "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
53  "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
54 
55 
56 class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
57  dimensions=("tract", "patch", "band", "skymap"),
58  defaultTemplates={"inputCoaddName": "deep",
59  "outputCoaddName": "deep",
60  "warpType": "direct",
61  "warpTypeSuffix": "",
62  "fakesType": ""}):
63  inputWarps = pipeBase.connectionTypes.Input(
64  doc=("Input list of warps to be assemebled i.e. stacked."
65  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
66  name="{inputCoaddName}Coadd_{warpType}Warp",
67  storageClass="ExposureF",
68  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
69  deferLoad=True,
70  multiple=True
71  )
72  skyMap = pipeBase.connectionTypes.Input(
73  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
74  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75  storageClass="SkyMap",
76  dimensions=("skymap", ),
77  )
78  selectedVisits = pipeBase.connectionTypes.Input(
79  doc="Selected visits to be coadded.",
80  name="{outputCoaddName}Visits",
81  storageClass="StructuredDataDict",
82  dimensions=("instrument", "tract", "patch", "skymap", "band")
83  )
84  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
85  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
86  " BRIGHT_OBJECT."),
87  name="brightObjectMask",
88  storageClass="ObjectMaskCatalog",
89  dimensions=("tract", "patch", "skymap", "band"),
90  )
91  coaddExposure = pipeBase.connectionTypes.Output(
92  doc="Output coadded exposure, produced by stacking input warps",
93  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
94  storageClass="ExposureF",
95  dimensions=("tract", "patch", "skymap", "band"),
96  )
97  nImage = pipeBase.connectionTypes.Output(
98  doc="Output image of number of input images per pixel",
99  name="{outputCoaddName}Coadd_nImage",
100  storageClass="ImageU",
101  dimensions=("tract", "patch", "skymap", "band"),
102  )
103  inputMap = pipeBase.connectionTypes.Output(
104  doc="Output healsparse map of input images",
105  name="{outputCoaddName}Coadd_inputMap",
106  storageClass="HealSparseMap",
107  dimensions=("tract", "patch", "skymap", "band"),
108  )
109 
110  def __init__(self, *, config=None):
111  super().__init__(config=config)
112 
113  # Override the connection's name template with config to replicate Gen2 behavior
114  # This duplicates some of the logic in the base class, due to wanting Gen2 and
115  # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
116  templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
117  templateValues['warpType'] = config.warpType
118  templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
119  if config.hasFakes:
120  templateValues['fakesType'] = "_fakes"
121  self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
122  for name in self.allConnections}
123  self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
124  # End code to remove after deprecation
125 
126  if not config.doMaskBrightObjects:
127  self.prerequisiteInputs.remove("brightObjectMask")
128 
129  if not config.doSelectVisits:
130  self.inputs.remove("selectedVisits")
131 
132  if not config.doNImage:
133  self.outputs.remove("nImage")
134 
135  if not self.config.doInputMap:
136  self.outputs.remove("inputMap")
137 
138 
139 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
140  pipelineConnections=AssembleCoaddConnections):
141  """Configuration parameters for the `AssembleCoaddTask`.
142 
143  Notes
144  -----
145  The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
146  only set the bitplane config.brightObjectMaskName. To make this useful you
147  *must* also configure the flags.pixel algorithm, for example by adding
148 
149  .. code-block:: none
150 
151  config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
152  config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
153 
154  to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
155  """
156  warpType = pexConfig.Field(
157  doc="Warp name: one of 'direct' or 'psfMatched'",
158  dtype=str,
159  default="direct",
160  )
161  subregionSize = pexConfig.ListField(
162  dtype=int,
163  doc="Width, height of stack subregion size; "
164  "make small enough that a full stack of images will fit into memory at once.",
165  length=2,
166  default=(2000, 2000),
167  )
168  statistic = pexConfig.Field(
169  dtype=str,
170  doc="Main stacking statistic for aggregating over the epochs.",
171  default="MEANCLIP",
172  )
173  doSigmaClip = pexConfig.Field(
174  dtype=bool,
175  doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
176  default=False,
177  )
178  sigmaClip = pexConfig.Field(
179  dtype=float,
180  doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
181  default=3.0,
182  )
183  clipIter = pexConfig.Field(
184  dtype=int,
185  doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
186  default=2,
187  )
188  calcErrorFromInputVariance = pexConfig.Field(
189  dtype=bool,
190  doc="Calculate coadd variance from input variance by stacking statistic."
191  "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
192  default=True,
193  )
194  scaleZeroPoint = pexConfig.ConfigurableField(
195  target=ScaleZeroPointTask,
196  doc="Task to adjust the photometric zero point of the coadd temp exposures",
197  )
198  doInterp = pexConfig.Field(
199  doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
200  dtype=bool,
201  default=True,
202  )
203  interpImage = pexConfig.ConfigurableField(
204  target=InterpImageTask,
205  doc="Task to interpolate (and extrapolate) over NaN pixels",
206  )
207  doWrite = pexConfig.Field(
208  doc="Persist coadd?",
209  dtype=bool,
210  default=True,
211  )
212  doNImage = pexConfig.Field(
213  doc="Create image of number of contributing exposures for each pixel",
214  dtype=bool,
215  default=False,
216  )
217  doUsePsfMatchedPolygons = pexConfig.Field(
218  doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
219  dtype=bool,
220  default=False,
221  )
222  maskPropagationThresholds = pexConfig.DictField(
223  keytype=str,
224  itemtype=float,
225  doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
226  "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
227  "would have contributed exceeds this value."),
228  default={"SAT": 0.1},
229  )
230  removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
231  doc="Mask planes to remove before coadding")
232  doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
233  doc="Set mask and flag bits for bright objects?")
234  brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
235  doc="Name of mask bit used for bright objects")
236  coaddPsf = pexConfig.ConfigField(
237  doc="Configuration for CoaddPsf",
238  dtype=measAlg.CoaddPsfConfig,
239  )
240  doAttachTransmissionCurve = pexConfig.Field(
241  dtype=bool, default=False, optional=False,
242  doc=("Attach a piecewise TransmissionCurve for the coadd? "
243  "(requires all input Exposures to have TransmissionCurves).")
244  )
245  hasFakes = pexConfig.Field(
246  dtype=bool,
247  default=False,
248  doc="Should be set to True if fake sources have been inserted into the input data."
249  )
250  doSelectVisits = pexConfig.Field(
251  doc="Coadd only visits selected by a SelectVisitsTask",
252  dtype=bool,
253  default=False,
254  )
255  doInputMap = pexConfig.Field(
256  doc="Create a bitwise map of coadd inputs",
257  dtype=bool,
258  default=False,
259  )
260  inputMapper = pexConfig.ConfigurableField(
261  doc="Input map creation subtask.",
262  target=HealSparseInputMapTask,
263  )
264 
265  def setDefaults(self):
266  super().setDefaults()
267  self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
268 
269  def validate(self):
270  super().validate()
271  if self.doPsfMatch:
272  # Backwards compatibility.
273  # Configs do not have loggers
274  log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
275  self.warpType = 'psfMatched'
276  if self.doSigmaClip and self.statistic != "MEANCLIP":
277  log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
278  self.statistic = "MEANCLIP"
279  if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
280  raise ValueError("Must set doInterp=False for statistic=%s, which does not "
281  "compute and set a non-zero coadd variance estimate." % (self.statistic))
282 
283  unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
284  if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
285  stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
286  if str(k) not in unstackableStats]
287  raise ValueError("statistic %s is not allowed. Please choose one of %s."
288  % (self.statistic, stackableStats))
289 
290 
291 class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
292  """Assemble a coadded image from a set of warps (coadded temporary exposures).
293 
294  We want to assemble a coadded image from a set of Warps (also called
295  coadded temporary exposures or ``coaddTempExps``).
296  Each input Warp covers a patch on the sky and corresponds to a single
297  run/visit/exposure of the covered patch. We provide the task with a list
298  of Warps (``selectDataList``) from which it selects Warps that cover the
299  specified patch (pointed at by ``dataRef``).
300  Each Warp that goes into a coadd will typically have an independent
301  photometric zero-point. Therefore, we must scale each Warp to set it to
302  a common photometric zeropoint. WarpType may be one of 'direct' or
303  'psfMatched', and the boolean configs `config.makeDirect` and
304  `config.makePsfMatched` set which of the warp types will be coadded.
305  The coadd is computed as a mean with optional outlier rejection.
306  Criteria for outlier rejection are set in `AssembleCoaddConfig`.
307  Finally, Warps can have bad 'NaN' pixels which received no input from the
308  source calExps. We interpolate over these bad (NaN) pixels.
309 
310  `AssembleCoaddTask` uses several sub-tasks. These are
311 
312  - `ScaleZeroPointTask`
313  - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
314  - `InterpImageTask`
315  - interpolate across bad pixels (NaN) in the final coadd
316 
317  You can retarget these subtasks if you wish.
318 
319  Notes
320  -----
321  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
322  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
323  `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
324  no debug variables of its own. Some of the subtasks may support debug
325  variables. See the documentation for the subtasks for further information.
326 
327  Examples
328  --------
329  `AssembleCoaddTask` assembles a set of warped images into a coadded image.
330  The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
331  with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
332  inputs: a data reference to the tract patch and filter to be coadded, and
333  a list of Warps to attempt to coadd. These are specified using ``--id`` and
334  ``--selectId``, respectively:
335 
336  .. code-block:: none
337 
338  --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
339  --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
340 
341  Only the Warps that cover the specified tract and patch will be coadded.
342  A list of the available optional arguments can be obtained by calling
343  ``assembleCoadd.py`` with the ``--help`` command line argument:
344 
345  .. code-block:: none
346 
347  assembleCoadd.py --help
348 
349  To demonstrate usage of the `AssembleCoaddTask` in the larger context of
350  multi-band processing, we will generate the HSC-I & -R band coadds from
351  HSC engineering test data provided in the ``ci_hsc`` package. To begin,
352  assuming that the lsst stack has been already set up, we must set up the
353  obs_subaru and ``ci_hsc`` packages. This defines the environment variable
354  ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
355  data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
356  coadds, we must first
357 
358  - processCcd
359  - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
360  - makeSkyMap
361  - create a skymap that covers the area of the sky present in the raw exposures
362  - makeCoaddTempExp
363  - warp the individual calibrated exposures to the tangent plane of the coadd
364 
365  We can perform all of these steps by running
366 
367  .. code-block:: none
368 
369  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
370 
371  This will produce warped exposures for each visit. To coadd the warped
372  data, we call assembleCoadd.py as follows:
373 
374  .. code-block:: none
375 
376  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
377  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
378  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
379  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
380  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
381  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
382  --selectId visit=903988 ccd=24
383 
384  that will process the HSC-I band data. The results are written in
385  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
386 
387  You may also choose to run:
388 
389  .. code-block:: none
390 
391  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
392  assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
393  --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
394  --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
395  --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
396  --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
397  --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
398  --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
399 
400  to generate the coadd for the HSC-R band if you are interested in
401  following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
402  (but note that normally, one would use the `SafeClipAssembleCoaddTask`
403  rather than `AssembleCoaddTask` to make the coadd.
404  """
405  ConfigClass = AssembleCoaddConfig
406  _DefaultName = "assembleCoadd"
407 
408  def __init__(self, *args, **kwargs):
409  # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
410  if args:
411  argNames = ["config", "name", "parentTask", "log"]
412  kwargs.update({k: v for k, v in zip(argNames, args)})
413  warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
414  "PipelineTask will not take positional args" % argNames, FutureWarning)
415 
416  super().__init__(**kwargs)
417  self.makeSubtask("interpImage")
418  self.makeSubtask("scaleZeroPoint")
419 
420  if self.config.doMaskBrightObjects:
421  mask = afwImage.Mask()
422  try:
423  self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
424  except pexExceptions.LsstCppException:
425  raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
426  mask.getMaskPlaneDict().keys())
427  del mask
428 
429  if self.config.doInputMap:
430  self.makeSubtask("inputMapper")
431 
432  self.warpType = self.config.warpType
433 
434  @utils.inheritDoc(pipeBase.PipelineTask)
435  def runQuantum(self, butlerQC, inputRefs, outputRefs):
436  # Docstring to be formatted with info from PipelineTask.runQuantum
437  """
438  Notes
439  -----
440  Assemble a coadd from a set of Warps.
441 
442  PipelineTask (Gen3) entry point to Coadd a set of Warps.
443  Analogous to `runDataRef`, it prepares all the data products to be
444  passed to `run`, and processes the results before returning a struct
445  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
446  Therefore, its inputs are accessed subregion by subregion
447  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
448  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
449  correspond to an update in `runDataRef` while both entry points
450  are used.
451  """
452  inputData = butlerQC.get(inputRefs)
453 
454  # Construct skyInfo expected by run
455  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
456  skyMap = inputData["skyMap"]
457  outputDataId = butlerQC.quantum.dataId
458 
459  inputData['skyInfo'] = makeSkyInfo(skyMap,
460  tractId=outputDataId['tract'],
461  patchId=outputDataId['patch'])
462 
463  if self.config.doSelectVisits:
464  warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
465  else:
466  warpRefList = inputData['inputWarps']
467 
468  # Perform same middle steps as `runDataRef` does
469  inputs = self.prepareInputs(warpRefList)
470  self.log.info("Found %d %s", len(inputs.tempExpRefList),
471  self.getTempExpDatasetName(self.warpType))
472  if len(inputs.tempExpRefList) == 0:
473  self.log.warn("No coadd temporary exposures found")
474  return
475 
476  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
477  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
478  inputs.weightList, supplementaryData=supplementaryData)
479 
480  inputData.setdefault('brightObjectMask', None)
481  self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
482 
483  if self.config.doWrite:
484  butlerQC.put(retStruct, outputRefs)
485  return retStruct
486 
487  @pipeBase.timeMethod
488  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
489  """Assemble a coadd from a set of Warps.
490 
491  Pipebase.CmdlineTask entry point to Coadd a set of Warps.
492  Compute weights to be applied to each Warp and
493  find scalings to match the photometric zeropoint to a reference Warp.
494  Assemble the Warps using `run`. Interpolate over NaNs and
495  optionally write the coadd to disk. Return the coadded exposure.
496 
497  Parameters
498  ----------
499  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
500  Data reference defining the patch for coaddition and the
501  reference Warp (if ``config.autoReference=False``).
502  Used to access the following data products:
503  - ``self.config.coaddName + "Coadd_skyMap"``
504  - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
505  - ``self.config.coaddName + "Coadd"``
506  selectDataList : `list`
507  List of data references to Calexps. Data to be coadded will be
508  selected from this list based on overlap with the patch defined
509  by dataRef, grouped by visit, and converted to a list of data
510  references to warps.
511  warpRefList : `list`
512  List of data references to Warps to be coadded.
513  Note: `warpRefList` is just the new name for `tempExpRefList`.
514 
515  Returns
516  -------
517  retStruct : `lsst.pipe.base.Struct`
518  Result struct with components:
519 
520  - ``coaddExposure``: coadded exposure (``Exposure``).
521  - ``nImage``: exposure count image (``Image``).
522  """
523  if selectDataList and warpRefList:
524  raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
525  "and which to use is ambiguous. Please pass only one.")
526 
527  skyInfo = self.getSkyInfo(dataRef)
528  if warpRefList is None:
529  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
530  if len(calExpRefList) == 0:
531  self.log.warn("No exposures to coadd")
532  return
533  self.log.info("Coadding %d exposures", len(calExpRefList))
534 
535  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
536 
537  inputData = self.prepareInputs(warpRefList)
538  self.log.info("Found %d %s", len(inputData.tempExpRefList),
539  self.getTempExpDatasetName(self.warpType))
540  if len(inputData.tempExpRefList) == 0:
541  self.log.warn("No coadd temporary exposures found")
542  return
543 
544  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
545 
546  retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
547  inputData.weightList, supplementaryData=supplementaryData)
548 
549  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
550  self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
551 
552  if self.config.doWrite:
553  if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
554  coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
555  else:
556  coaddDatasetName = self.getCoaddDatasetName(self.warpType)
557  self.log.info("Persisting %s" % coaddDatasetName)
558  dataRef.put(retStruct.coaddExposure, coaddDatasetName)
559  if self.config.doNImage and retStruct.nImage is not None:
560  dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
561 
562  return retStruct
563 
564  def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
565  """Interpolate over missing data and mask bright stars.
566 
567  Parameters
568  ----------
569  coaddExposure : `lsst.afw.image.Exposure`
570  The coadded exposure to process.
571  dataRef : `lsst.daf.persistence.ButlerDataRef`
572  Butler data reference for supplementary data.
573  """
574  if self.config.doInterp:
575  self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
576  # The variance must be positive; work around for DM-3201.
577  varArray = coaddExposure.variance.array
578  with numpy.errstate(invalid="ignore"):
579  varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
580 
581  if self.config.doMaskBrightObjects:
582  self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
583 
584  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
585  """Make additional inputs to run() specific to subclasses (Gen2)
586 
587  Duplicates interface of `runDataRef` method
588  Available to be implemented by subclasses only if they need the
589  coadd dataRef for performing preliminary processing before
590  assembling the coadd.
591 
592  Parameters
593  ----------
594  dataRef : `lsst.daf.persistence.ButlerDataRef`
595  Butler data reference for supplementary data.
596  selectDataList : `list` (optional)
597  Optional List of data references to Calexps.
598  warpRefList : `list` (optional)
599  Optional List of data references to Warps.
600  """
601  return pipeBase.Struct()
602 
603  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
604  """Make additional inputs to run() specific to subclasses (Gen3)
605 
606  Duplicates interface of `runQuantum` method.
607  Available to be implemented by subclasses only if they need the
608  coadd dataRef for performing preliminary processing before
609  assembling the coadd.
610 
611  Parameters
612  ----------
613  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
614  Gen3 Butler object for fetching additional data products before
615  running the Task specialized for quantum being processed
616  inputRefs : `lsst.pipe.base.InputQuantizedConnection`
617  Attributes are the names of the connections describing input dataset types.
618  Values are DatasetRefs that task consumes for corresponding dataset type.
619  DataIds are guaranteed to match data objects in ``inputData``.
620  outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
621  Attributes are the names of the connections describing output dataset types.
622  Values are DatasetRefs that task is to produce
623  for corresponding dataset type.
624  """
625  return pipeBase.Struct()
626 
627  def getTempExpRefList(self, patchRef, calExpRefList):
628  """Generate list data references corresponding to warped exposures
629  that lie within the patch to be coadded.
630 
631  Parameters
632  ----------
633  patchRef : `dataRef`
634  Data reference for patch.
635  calExpRefList : `list`
636  List of data references for input calexps.
637 
638  Returns
639  -------
640  tempExpRefList : `list`
641  List of Warp/CoaddTempExp data references.
642  """
643  butler = patchRef.getButler()
644  groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
645  self.getTempExpDatasetName(self.warpType))
646  tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
647  g, groupData.keys) for
648  g in groupData.groups.keys()]
649  return tempExpRefList
650 
651  def prepareInputs(self, refList):
652  """Prepare the input warps for coaddition by measuring the weight for
653  each warp and the scaling for the photometric zero point.
654 
655  Each Warp has its own photometric zeropoint and background variance.
656  Before coadding these Warps together, compute a scale factor to
657  normalize the photometric zeropoint and compute the weight for each Warp.
658 
659  Parameters
660  ----------
661  refList : `list`
662  List of data references to tempExp
663 
664  Returns
665  -------
666  result : `lsst.pipe.base.Struct`
667  Result struct with components:
668 
669  - ``tempExprefList``: `list` of data references to tempExp.
670  - ``weightList``: `list` of weightings.
671  - ``imageScalerList``: `list` of image scalers.
672  """
673  statsCtrl = afwMath.StatisticsControl()
674  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
675  statsCtrl.setNumIter(self.config.clipIter)
676  statsCtrl.setAndMask(self.getBadPixelMask())
677  statsCtrl.setNanSafe(True)
678  # compute tempExpRefList: a list of tempExpRef that actually exist
679  # and weightList: a list of the weight of the associated coadd tempExp
680  # and imageScalerList: a list of scale factors for the associated coadd tempExp
681  tempExpRefList = []
682  weightList = []
683  imageScalerList = []
684  tempExpName = self.getTempExpDatasetName(self.warpType)
685  for tempExpRef in refList:
686  # Gen3's DeferredDatasetHandles are guaranteed to exist and
687  # therefore have no datasetExists() method
688  if not isinstance(tempExpRef, DeferredDatasetHandle):
689  if not tempExpRef.datasetExists(tempExpName):
690  self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
691  continue
692 
693  tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
694  # Ignore any input warp that is empty of data
695  if numpy.isnan(tempExp.image.array).all():
696  continue
697  maskedImage = tempExp.getMaskedImage()
698  imageScaler = self.scaleZeroPoint.computeImageScaler(
699  exposure=tempExp,
700  dataRef=tempExpRef,
701  )
702  try:
703  imageScaler.scaleMaskedImage(maskedImage)
704  except Exception as e:
705  self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
706  continue
707  statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
708  afwMath.MEANCLIP, statsCtrl)
709  meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
710  weight = 1.0 / float(meanVar)
711  if not numpy.isfinite(weight):
712  self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
713  continue
714  self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
715 
716  del maskedImage
717  del tempExp
718 
719  tempExpRefList.append(tempExpRef)
720  weightList.append(weight)
721  imageScalerList.append(imageScaler)
722 
723  return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
724  imageScalerList=imageScalerList)
725 
726  def prepareStats(self, mask=None):
727  """Prepare the statistics for coadding images.
728 
729  Parameters
730  ----------
731  mask : `int`, optional
732  Bit mask value to exclude from coaddition.
733 
734  Returns
735  -------
736  stats : `lsst.pipe.base.Struct`
737  Statistics structure with the following fields:
738 
739  - ``statsCtrl``: Statistics control object for coadd
740  (`lsst.afw.math.StatisticsControl`)
741  - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
742  """
743  if mask is None:
744  mask = self.getBadPixelMask()
745  statsCtrl = afwMath.StatisticsControl()
746  statsCtrl.setNumSigmaClip(self.config.sigmaClip)
747  statsCtrl.setNumIter(self.config.clipIter)
748  statsCtrl.setAndMask(mask)
749  statsCtrl.setNanSafe(True)
750  statsCtrl.setWeighted(True)
751  statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
752  for plane, threshold in self.config.maskPropagationThresholds.items():
753  bit = afwImage.Mask.getMaskPlane(plane)
754  statsCtrl.setMaskPropagationThreshold(bit, threshold)
755  statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
756  return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
757 
758  @pipeBase.timeMethod
759  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
760  altMaskList=None, mask=None, supplementaryData=None):
761  """Assemble a coadd from input warps
762 
763  Assemble the coadd using the provided list of coaddTempExps. Since
764  the full coadd covers a patch (a large area), the assembly is
765  performed over small areas on the image at a time in order to
766  conserve memory usage. Iterate over subregions within the outer
767  bbox of the patch using `assembleSubregion` to stack the corresponding
768  subregions from the coaddTempExps with the statistic specified.
769  Set the edge bits the coadd mask based on the weight map.
770 
771  Parameters
772  ----------
773  skyInfo : `lsst.pipe.base.Struct`
774  Struct with geometric information about the patch.
775  tempExpRefList : `list`
776  List of data references to Warps (previously called CoaddTempExps).
777  imageScalerList : `list`
778  List of image scalers.
779  weightList : `list`
780  List of weights
781  altMaskList : `list`, optional
782  List of alternate masks to use rather than those stored with
783  tempExp.
784  mask : `int`, optional
785  Bit mask value to exclude from coaddition.
786  supplementaryData : lsst.pipe.base.Struct, optional
787  Struct with additional data products needed to assemble coadd.
788  Only used by subclasses that implement `makeSupplementaryData`
789  and override `run`.
790 
791  Returns
792  -------
793  result : `lsst.pipe.base.Struct`
794  Result struct with components:
795 
796  - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
797  - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
798  - ``inputMap``: bit-wise map of inputs, if requested.
799  - ``warpRefList``: input list of refs to the warps (
800  ``lsst.daf.butler.DeferredDatasetHandle`` or
801  ``lsst.daf.persistence.ButlerDataRef``)
802  (unmodified)
803  - ``imageScalerList``: input list of image scalers (unmodified)
804  - ``weightList``: input list of weights (unmodified)
805  """
806  tempExpName = self.getTempExpDatasetName(self.warpType)
807  self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
808  stats = self.prepareStats(mask=mask)
809 
810  if altMaskList is None:
811  altMaskList = [None]*len(tempExpRefList)
812 
813  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
814  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
815  coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
816  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
817  coaddMaskedImage = coaddExposure.getMaskedImage()
818  subregionSizeArr = self.config.subregionSize
819  subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
820  # if nImage is requested, create a zero one which can be passed to assembleSubregion
821  if self.config.doNImage:
822  nImage = afwImage.ImageU(skyInfo.bbox)
823  else:
824  nImage = None
825  # If inputMap is requested, create the initial version that can be masked in
826  # assembleSubregion.
827  if self.config.doInputMap:
828  self.inputMapper.build_ccd_input_map(skyInfo.bbox,
829  skyInfo.wcs,
830  coaddExposure.getInfo().getCoaddInputs().ccds)
831 
832  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
833  try:
834  self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
835  weightList, altMaskList, stats.flags, stats.ctrl,
836  nImage=nImage)
837  except Exception as e:
838  self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
839 
840  # If inputMap is requested, we must finalize the map after the accumulation.
841  if self.config.doInputMap:
842  self.inputMapper.finalize_ccd_input_map_mask()
843  inputMap = self.inputMapper.ccd_input_map
844  else:
845  inputMap = None
846 
847  self.setInexactPsf(coaddMaskedImage.getMask())
848  # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
849  # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
850  coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
851  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
852  warpRefList=tempExpRefList, imageScalerList=imageScalerList,
853  weightList=weightList, inputMap=inputMap)
854 
855  def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
856  """Set the metadata for the coadd.
857 
858  This basic implementation sets the filter from the first input.
859 
860  Parameters
861  ----------
862  coaddExposure : `lsst.afw.image.Exposure`
863  The target exposure for the coadd.
864  tempExpRefList : `list`
865  List of data references to tempExp.
866  weightList : `list`
867  List of weights.
868  """
869  assert len(tempExpRefList) == len(weightList), "Length mismatch"
870  tempExpName = self.getTempExpDatasetName(self.warpType)
871  # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
872  # (and we need more than just the PropertySet that contains the header), which is not possible
873  # with the current butler (see #2777).
874  bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
875 
876  if isinstance(tempExpRefList[0], DeferredDatasetHandle):
877  # Gen 3 API
878  tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
879  else:
880  # Gen 2 API. Delete this when Gen 2 retired
881  tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
882  for tempExpRef in tempExpRefList]
883  numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
884 
885  # Set the coadd FilterLabel to the band of the first input exposure:
886  # Coadds are calibrated, so the physical label is now meaningless.
887  coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
888  coaddInputs = coaddExposure.getInfo().getCoaddInputs()
889  coaddInputs.ccds.reserve(numCcds)
890  coaddInputs.visits.reserve(len(tempExpList))
891 
892  for tempExp, weight in zip(tempExpList, weightList):
893  self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
894 
895  if self.config.doUsePsfMatchedPolygons:
896  self.shrinkValidPolygons(coaddInputs)
897 
898  coaddInputs.visits.sort()
899  if self.warpType == "psfMatched":
900  # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
901  # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
902  # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
903  # having the maximum width (sufficient because square)
904  modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
905  modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
906  psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
907  else:
908  psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
909  self.config.coaddPsf.makeControl())
910  coaddExposure.setPsf(psf)
911  apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
912  coaddExposure.getWcs())
913  coaddExposure.getInfo().setApCorrMap(apCorrMap)
914  if self.config.doAttachTransmissionCurve:
915  transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
916  coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
917 
918  def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
919  altMaskList, statsFlags, statsCtrl, nImage=None):
920  """Assemble the coadd for a sub-region.
921 
922  For each coaddTempExp, check for (and swap in) an alternative mask
923  if one is passed. Remove mask planes listed in
924  `config.removeMaskPlanes`. Finally, stack the actual exposures using
925  `lsst.afw.math.statisticsStack` with the statistic specified by
926  statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
927  a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
928  an N-sigma clipped mean where N and iterations are specified by
929  statsCtrl. Assign the stacked subregion back to the coadd.
930 
931  Parameters
932  ----------
933  coaddExposure : `lsst.afw.image.Exposure`
934  The target exposure for the coadd.
935  bbox : `lsst.geom.Box`
936  Sub-region to coadd.
937  tempExpRefList : `list`
938  List of data reference to tempExp.
939  imageScalerList : `list`
940  List of image scalers.
941  weightList : `list`
942  List of weights.
943  altMaskList : `list`
944  List of alternate masks to use rather than those stored with
945  tempExp, or None. Each element is dict with keys = mask plane
946  name to which to add the spans.
947  statsFlags : `lsst.afw.math.Property`
948  Property object for statistic for coadd.
949  statsCtrl : `lsst.afw.math.StatisticsControl`
950  Statistics control object for coadd.
951  nImage : `lsst.afw.image.ImageU`, optional
952  Keeps track of exposure count for each pixel.
953  """
954  self.log.debug("Computing coadd over %s", bbox)
955  tempExpName = self.getTempExpDatasetName(self.warpType)
956  coaddExposure.mask.addMaskPlane("REJECTED")
957  coaddExposure.mask.addMaskPlane("CLIPPED")
958  coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
959  maskMap = self.setRejectedMaskMapping(statsCtrl)
960  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
961  maskedImageList = []
962  if nImage is not None:
963  subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
964  for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
965 
966  if isinstance(tempExpRef, DeferredDatasetHandle):
967  # Gen 3 API
968  exposure = tempExpRef.get(parameters={'bbox': bbox})
969  else:
970  # Gen 2 API. Delete this when Gen 2 retired
971  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
972 
973  maskedImage = exposure.getMaskedImage()
974  mask = maskedImage.getMask()
975  if altMask is not None:
976  self.applyAltMaskPlanes(mask, altMask)
977  imageScaler.scaleMaskedImage(maskedImage)
978 
979  # Add 1 for each pixel which is not excluded by the exclude mask.
980  # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
981  if nImage is not None:
982  subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
983  if self.config.removeMaskPlanes:
984  self.removeMaskPlanes(maskedImage)
985  maskedImageList.append(maskedImage)
986 
987  if self.config.doInputMap:
988  visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
989  self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
990 
991  with self.timer("stack"):
992  coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
993  clipped, # also set output to CLIPPED if sigma-clipped
994  maskMap)
995  coaddExposure.maskedImage.assign(coaddSubregion, bbox)
996  if nImage is not None:
997  nImage.assign(subNImage, bbox)
998 
999  def removeMaskPlanes(self, maskedImage):
1000  """Unset the mask of an image for mask planes specified in the config.
1001 
1002  Parameters
1003  ----------
1004  maskedImage : `lsst.afw.image.MaskedImage`
1005  The masked image to be modified.
1006  """
1007  mask = maskedImage.getMask()
1008  for maskPlane in self.config.removeMaskPlanes:
1009  try:
1010  mask &= ~mask.getPlaneBitMask(maskPlane)
1011  except pexExceptions.InvalidParameterError:
1012  self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
1013  maskPlane)
1014 
1015  @staticmethod
1016  def setRejectedMaskMapping(statsCtrl):
1017  """Map certain mask planes of the warps to new planes for the coadd.
1018 
1019  If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1020  or CLIPPED, set it to REJECTED on the coadd.
1021  If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1022  If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1023 
1024  Parameters
1025  ----------
1026  statsCtrl : `lsst.afw.math.StatisticsControl`
1027  Statistics control object for coadd
1028 
1029  Returns
1030  -------
1031  maskMap : `list` of `tuple` of `int`
1032  A list of mappings of mask planes of the warped exposures to
1033  mask planes of the coadd.
1034  """
1035  edge = afwImage.Mask.getPlaneBitMask("EDGE")
1036  noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1037  clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1038  toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1039  maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1040  (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1041  (clipped, clipped)]
1042  return maskMap
1043 
1044  def applyAltMaskPlanes(self, mask, altMaskSpans):
1045  """Apply in place alt mask formatted as SpanSets to a mask.
1046 
1047  Parameters
1048  ----------
1049  mask : `lsst.afw.image.Mask`
1050  Original mask.
1051  altMaskSpans : `dict`
1052  SpanSet lists to apply. Each element contains the new mask
1053  plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1054  and list of SpanSets to apply to the mask.
1055 
1056  Returns
1057  -------
1058  mask : `lsst.afw.image.Mask`
1059  Updated mask.
1060  """
1061  if self.config.doUsePsfMatchedPolygons:
1062  if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1063  # Clear away any other masks outside the validPolygons. These pixels are no longer
1064  # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1065  # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1066  # This mask-clearing step must only occur *before* applying the new masks below
1067  for spanSet in altMaskSpans['NO_DATA']:
1068  spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1069 
1070  for plane, spanSetList in altMaskSpans.items():
1071  maskClipValue = mask.addMaskPlane(plane)
1072  for spanSet in spanSetList:
1073  spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1074  return mask
1075 
1076  def shrinkValidPolygons(self, coaddInputs):
1077  """Shrink coaddInputs' ccds' ValidPolygons in place.
1078 
1079  Either modify each ccd's validPolygon in place, or if CoaddInputs
1080  does not have a validPolygon, create one from its bbox.
1081 
1082  Parameters
1083  ----------
1084  coaddInputs : `lsst.afw.image.coaddInputs`
1085  Original mask.
1086 
1087  """
1088  for ccd in coaddInputs.ccds:
1089  polyOrig = ccd.getValidPolygon()
1090  validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1091  validPolyBBox.grow(-self.config.matchingKernelSize//2)
1092  if polyOrig:
1093  validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1094  else:
1095  validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1096  ccd.setValidPolygon(validPolygon)
1097 
1098  def readBrightObjectMasks(self, dataRef):
1099  """Retrieve the bright object masks.
1100 
1101  Returns None on failure.
1102 
1103  Parameters
1104  ----------
1105  dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1106  A Butler dataRef.
1107 
1108  Returns
1109  -------
1110  result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1111  Bright object mask from the Butler object, or None if it cannot
1112  be retrieved.
1113  """
1114  try:
1115  return dataRef.get(datasetType="brightObjectMask", immediate=True)
1116  except Exception as e:
1117  self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1118  return None
1119 
1120  def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1121  """Set the bright object masks.
1122 
1123  Parameters
1124  ----------
1125  exposure : `lsst.afw.image.Exposure`
1126  Exposure under consideration.
1127  dataId : `lsst.daf.persistence.dataId`
1128  Data identifier dict for patch.
1129  brightObjectMasks : `lsst.afw.table`
1130  Table of bright objects to mask.
1131  """
1132 
1133  if brightObjectMasks is None:
1134  self.log.warn("Unable to apply bright object mask: none supplied")
1135  return
1136  self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1137  mask = exposure.getMaskedImage().getMask()
1138  wcs = exposure.getWcs()
1139  plateScale = wcs.getPixelScale().asArcseconds()
1140 
1141  for rec in brightObjectMasks:
1142  center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1143  if rec["type"] == "box":
1144  assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1145  width = rec["width"].asArcseconds()/plateScale # convert to pixels
1146  height = rec["height"].asArcseconds()/plateScale # convert to pixels
1147 
1148  halfSize = geom.ExtentI(0.5*width, 0.5*height)
1149  bbox = geom.Box2I(center - halfSize, center + halfSize)
1150 
1151  bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1152  geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1153  spans = afwGeom.SpanSet(bbox)
1154  elif rec["type"] == "circle":
1155  radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1156  spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1157  else:
1158  self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1159  continue
1160  spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1161 
1162  def setInexactPsf(self, mask):
1163  """Set INEXACT_PSF mask plane.
1164 
1165  If any of the input images isn't represented in the coadd (due to
1166  clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1167  these pixels.
1168 
1169  Parameters
1170  ----------
1171  mask : `lsst.afw.image.Mask`
1172  Coadded exposure's mask, modified in-place.
1173  """
1174  mask.addMaskPlane("INEXACT_PSF")
1175  inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1176  sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1177  clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1178  rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1179  array = mask.getArray()
1180  selected = array & (sensorEdge | clipped | rejected) > 0
1181  array[selected] |= inexactPsf
1182 
1183  @classmethod
1184  def _makeArgumentParser(cls):
1185  """Create an argument parser.
1186  """
1187  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1188  parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1189  + cls.ConfigClass().warpType + "Warp",
1190  help="data ID, e.g. --id tract=12345 patch=1,2",
1191  ContainerClass=AssembleCoaddDataIdContainer)
1192  parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1193  ContainerClass=SelectDataIdContainer)
1194  return parser
1195 
1196  @staticmethod
1197  def _subBBoxIter(bbox, subregionSize):
1198  """Iterate over subregions of a bbox.
1199 
1200  Parameters
1201  ----------
1202  bbox : `lsst.geom.Box2I`
1203  Bounding box over which to iterate.
1204  subregionSize: `lsst.geom.Extent2I`
1205  Size of sub-bboxes.
1206 
1207  Yields
1208  ------
1209  subBBox : `lsst.geom.Box2I`
1210  Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1211  is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1212  the edges of ``bbox``, but it will never be empty.
1213  """
1214  if bbox.isEmpty():
1215  raise RuntimeError("bbox %s is empty" % (bbox,))
1216  if subregionSize[0] < 1 or subregionSize[1] < 1:
1217  raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1218 
1219  for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1220  for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1221  subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1222  subBBox.clip(bbox)
1223  if subBBox.isEmpty():
1224  raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1225  "colShift=%s, rowShift=%s" %
1226  (bbox, subregionSize, colShift, rowShift))
1227  yield subBBox
1228 
1229  def filterWarps(self, inputs, goodVisits):
1230  """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1231 
1232  Parameters
1233  ----------
1234  inputs : list
1235  List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1236  goodVisit : `dict`
1237  Dictionary with good visitIds as the keys. Value ignored.
1238 
1239  Returns:
1240  --------
1241  filteredInputs : `list`
1242  Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1243  """
1244  inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1245  filteredInputs = []
1246  for visit in goodVisits.keys():
1247  filteredInputs.append(inputWarpDict[visit])
1248  return filteredInputs
1249 
1250 
1251 class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1252  """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1253  """
1254 
1255  def makeDataRefList(self, namespace):
1256  """Make self.refList from self.idList.
1257 
1258  Parameters
1259  ----------
1260  namespace
1261  Results of parsing command-line (with ``butler`` and ``log`` elements).
1262  """
1263  datasetType = namespace.config.coaddName + "Coadd"
1264  keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1265 
1266  for dataId in self.idList:
1267  # tract and patch are required
1268  for key in keysCoadd:
1269  if key not in dataId:
1270  raise RuntimeError("--id must include " + key)
1271 
1272  dataRef = namespace.butler.dataRef(
1273  datasetType=datasetType,
1274  dataId=dataId,
1275  )
1276  self.refList.append(dataRef)
1277 
1278 
1279 def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1280  """Function to count the number of pixels with a specific mask in a
1281  footprint.
1282 
1283  Find the intersection of mask & footprint. Count all pixels in the mask
1284  that are in the intersection that have bitmask set but do not have
1285  ignoreMask set. Return the count.
1286 
1287  Parameters
1288  ----------
1289  mask : `lsst.afw.image.Mask`
1290  Mask to define intersection region by.
1291  footprint : `lsst.afw.detection.Footprint`
1292  Footprint to define the intersection region by.
1293  bitmask
1294  Specific mask that we wish to count the number of occurances of.
1295  ignoreMask
1296  Pixels to not consider.
1297 
1298  Returns
1299  -------
1300  result : `int`
1301  Count of number of pixels in footprint with specified mask.
1302  """
1303  bbox = footprint.getBBox()
1304  bbox.clip(mask.getBBox(afwImage.PARENT))
1305  fp = afwImage.Mask(bbox)
1306  subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1307  footprint.spans.setMask(fp, bitmask)
1308  return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1309  (subMask.getArray() & ignoreMask) == 0).sum()
1310 
1311 
1312 class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1313  """Configuration parameters for the SafeClipAssembleCoaddTask.
1314  """
1315  clipDetection = pexConfig.ConfigurableField(
1316  target=SourceDetectionTask,
1317  doc="Detect sources on difference between unclipped and clipped coadd")
1318  minClipFootOverlap = pexConfig.Field(
1319  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1320  dtype=float,
1321  default=0.6
1322  )
1323  minClipFootOverlapSingle = pexConfig.Field(
1324  doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1325  "clipped when only one visit overlaps",
1326  dtype=float,
1327  default=0.5
1328  )
1329  minClipFootOverlapDouble = pexConfig.Field(
1330  doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1331  "clipped when two visits overlap",
1332  dtype=float,
1333  default=0.45
1334  )
1335  maxClipFootOverlapDouble = pexConfig.Field(
1336  doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1337  "considering two visits",
1338  dtype=float,
1339  default=0.15
1340  )
1341  minBigOverlap = pexConfig.Field(
1342  doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1343  "when labeling clipped footprints",
1344  dtype=int,
1345  default=100
1346  )
1347 
1348  def setDefaults(self):
1349  """Set default values for clipDetection.
1350 
1351  Notes
1352  -----
1353  The numeric values for these configuration parameters were
1354  empirically determined, future work may further refine them.
1355  """
1356  AssembleCoaddConfig.setDefaults(self)
1357  self.clipDetectionclipDetection.doTempLocalBackground = False
1358  self.clipDetectionclipDetection.reEstimateBackground = False
1359  self.clipDetectionclipDetection.returnOriginalFootprints = False
1360  self.clipDetectionclipDetection.thresholdPolarity = "both"
1361  self.clipDetectionclipDetection.thresholdValue = 2
1362  self.clipDetectionclipDetection.nSigmaToGrow = 2
1363  self.clipDetectionclipDetection.minPixels = 4
1364  self.clipDetectionclipDetection.isotropicGrow = True
1365  self.clipDetectionclipDetection.thresholdType = "pixel_stdev"
1366  self.sigmaClipsigmaClip = 1.5
1367  self.clipIterclipIter = 3
1368  self.statisticstatistic = "MEAN"
1369 
1370  def validate(self):
1371  if self.doSigmaClipdoSigmaClip:
1372  log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1373  "Ignoring doSigmaClip.")
1374  self.doSigmaClipdoSigmaClip = False
1375  if self.statisticstatistic != "MEAN":
1376  raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1377  "(%s chosen). Please set statistic to MEAN."
1378  % (self.statisticstatistic))
1379  AssembleCoaddTask.ConfigClass.validate(self)
1380 
1381 
1382 class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1383  """Assemble a coadded image from a set of coadded temporary exposures,
1384  being careful to clip & flag areas with potential artifacts.
1385 
1386  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1387  we clip outliers). The problem with doing this is that when computing the
1388  coadd PSF at a given location, individual visit PSFs from visits with
1389  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1390  In this task, we correct for this behavior by creating a new
1391  ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1392  coaddTempExps and the final coadd where
1393 
1394  i. difference imaging suggests that there is an outlier and
1395  ii. this outlier appears on only one or two images.
1396 
1397  Such regions will not contribute to the final coadd. Furthermore, any
1398  routine to determine the coadd PSF can now be cognizant of clipped regions.
1399  Note that the algorithm implemented by this task is preliminary and works
1400  correctly for HSC data. Parameter modifications and or considerable
1401  redesigning of the algorithm is likley required for other surveys.
1402 
1403  ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1404  "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1405  You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1406  if you wish.
1407 
1408  Notes
1409  -----
1410  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1411  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1412  see `baseDebug` for more about ``debug.py`` files.
1413  `SafeClipAssembleCoaddTask` has no debug variables of its own.
1414  The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1415  variables. See the documetation for `SourceDetectionTask` "clipDetection"
1416  for further information.
1417 
1418  Examples
1419  --------
1420  `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1421  images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1422  running assembleCoadd.py *without* the flag '--legacyCoadd'.
1423 
1424  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1425  and filter to be coadded (specified using
1426  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1427  along with a list of coaddTempExps to attempt to coadd (specified using
1428  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1429  Only the coaddTempExps that cover the specified tract and patch will be
1430  coadded. A list of the available optional arguments can be obtained by
1431  calling assembleCoadd.py with the --help command line argument:
1432 
1433  .. code-block:: none
1434 
1435  assembleCoadd.py --help
1436 
1437  To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1438  context of multi-band processing, we will generate the HSC-I & -R band
1439  coadds from HSC engineering test data provided in the ci_hsc package.
1440  To begin, assuming that the lsst stack has been already set up, we must
1441  set up the obs_subaru and ci_hsc packages. This defines the environment
1442  variable $CI_HSC_DIR and points at the location of the package. The raw
1443  HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1444  the coadds, we must first
1445 
1446  - ``processCcd``
1447  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1448  - ``makeSkyMap``
1449  create a skymap that covers the area of the sky present in the raw exposures
1450  - ``makeCoaddTempExp``
1451  warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1452 
1453  We can perform all of these steps by running
1454 
1455  .. code-block:: none
1456 
1457  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1458 
1459  This will produce warped coaddTempExps for each visit. To coadd the
1460  warped data, we call ``assembleCoadd.py`` as follows:
1461 
1462  .. code-block:: none
1463 
1464  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1465  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1466  --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1467  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1468  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1469  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1470  --selectId visit=903988 ccd=24
1471 
1472  This will process the HSC-I band data. The results are written in
1473  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1474 
1475  You may also choose to run:
1476 
1477  .. code-block:: none
1478 
1479  scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1480  assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1481  --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1482  --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1483  --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1484  --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1485  --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1486  --selectId visit=903346 ccd=12
1487 
1488  to generate the coadd for the HSC-R band if you are interested in following
1489  multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1490  """
1491  ConfigClass = SafeClipAssembleCoaddConfig
1492  _DefaultName = "safeClipAssembleCoadd"
1493 
1494  def __init__(self, *args, **kwargs):
1495  AssembleCoaddTask.__init__(self, *args, **kwargs)
1496  schema = afwTable.SourceTable.makeMinimalSchema()
1497  self.makeSubtask("clipDetection", schema=schema)
1498 
1499  @utils.inheritDoc(AssembleCoaddTask)
1500  @pipeBase.timeMethod
1501  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1502  """Assemble the coadd for a region.
1503 
1504  Compute the difference of coadds created with and without outlier
1505  rejection to identify coadd pixels that have outlier values in some
1506  individual visits.
1507  Detect clipped regions on the difference image and mark these regions
1508  on the one or two individual coaddTempExps where they occur if there
1509  is significant overlap between the clipped region and a source. This
1510  leaves us with a set of footprints from the difference image that have
1511  been identified as having occured on just one or two individual visits.
1512  However, these footprints were generated from a difference image. It
1513  is conceivable for a large diffuse source to have become broken up
1514  into multiple footprints acrosss the coadd difference in this process.
1515  Determine the clipped region from all overlapping footprints from the
1516  detected sources in each visit - these are big footprints.
1517  Combine the small and big clipped footprints and mark them on a new
1518  bad mask plane.
1519  Generate the coadd using `AssembleCoaddTask.run` without outlier
1520  removal. Clipped footprints will no longer make it into the coadd
1521  because they are marked in the new bad mask plane.
1522 
1523  Notes
1524  -----
1525  args and kwargs are passed but ignored in order to match the call
1526  signature expected by the parent task.
1527  """
1528  exp = self.buildDifferenceImagebuildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1529  mask = exp.getMaskedImage().getMask()
1530  mask.addMaskPlane("CLIPPED")
1531 
1532  result = self.detectClipdetectClip(exp, tempExpRefList)
1533 
1534  self.log.info('Found %d clipped objects', len(result.clipFootprints))
1535 
1536  maskClipValue = mask.getPlaneBitMask("CLIPPED")
1537  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1538  # Append big footprints from individual Warps to result.clipSpans
1539  bigFootprints = self.detectClipBigdetectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1540  result.detectionFootprints, maskClipValue, maskDetValue,
1541  exp.getBBox())
1542  # Create mask of the current clipped footprints
1543  maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1544  afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1545 
1546  maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1547  afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1548  maskClip |= maskClipBig
1549 
1550  # Assemble coadd from base class, but ignoring CLIPPED pixels
1551  badMaskPlanes = self.config.badMaskPlanes[:]
1552  badMaskPlanes.append("CLIPPED")
1553  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1554  return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1555  result.clipSpans, mask=badPixelMask)
1556 
1557  def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1558  """Return an exposure that contains the difference between unclipped
1559  and clipped coadds.
1560 
1561  Generate a difference image between clipped and unclipped coadds.
1562  Compute the difference image by subtracting an outlier-clipped coadd
1563  from an outlier-unclipped coadd. Return the difference image.
1564 
1565  Parameters
1566  ----------
1567  skyInfo : `lsst.pipe.base.Struct`
1568  Patch geometry information, from getSkyInfo
1569  tempExpRefList : `list`
1570  List of data reference to tempExp
1571  imageScalerList : `list`
1572  List of image scalers
1573  weightList : `list`
1574  List of weights
1575 
1576  Returns
1577  -------
1578  exp : `lsst.afw.image.Exposure`
1579  Difference image of unclipped and clipped coadd wrapped in an Exposure
1580  """
1581  config = AssembleCoaddConfig()
1582  # getattr necessary because subtasks do not survive Config.toDict()
1583  # exclude connections because the class of self.config.connections is not
1584  # the same as AssembleCoaddConfig.connections, and the connections are not
1585  # needed to run this task anyway.
1586  configIntersection = {k: getattr(self.config, k)
1587  for k, v in self.config.toDict().items()
1588  if (k in config.keys() and k != "connections")}
1589  configIntersection['doInputMap'] = False
1590  configIntersection['doNImage'] = False
1591  config.update(**configIntersection)
1592 
1593  # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1594  config.statistic = 'MEAN'
1595  task = AssembleCoaddTask(config=config)
1596  coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1597 
1598  config.statistic = 'MEANCLIP'
1599  task = AssembleCoaddTask(config=config)
1600  coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1601 
1602  coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1603  coaddDiff -= coaddClip.getMaskedImage()
1604  exp = afwImage.ExposureF(coaddDiff)
1605  exp.setPsf(coaddMean.getPsf())
1606  return exp
1607 
1608  def detectClip(self, exp, tempExpRefList):
1609  """Detect clipped regions on an exposure and set the mask on the
1610  individual tempExp masks.
1611 
1612  Detect footprints in the difference image after smoothing the
1613  difference image with a Gaussian kernal. Identify footprints that
1614  overlap with one or two input ``coaddTempExps`` by comparing the
1615  computed overlap fraction to thresholds set in the config. A different
1616  threshold is applied depending on the number of overlapping visits
1617  (restricted to one or two). If the overlap exceeds the thresholds,
1618  the footprint is considered "CLIPPED" and is marked as such on the
1619  coaddTempExp. Return a struct with the clipped footprints, the indices
1620  of the ``coaddTempExps`` that end up overlapping with the clipped
1621  footprints, and a list of new masks for the ``coaddTempExps``.
1622 
1623  Parameters
1624  ----------
1625  exp : `lsst.afw.image.Exposure`
1626  Exposure to run detection on.
1627  tempExpRefList : `list`
1628  List of data reference to tempExp.
1629 
1630  Returns
1631  -------
1632  result : `lsst.pipe.base.Struct`
1633  Result struct with components:
1634 
1635  - ``clipFootprints``: list of clipped footprints.
1636  - ``clipIndices``: indices for each ``clippedFootprint`` in
1637  ``tempExpRefList``.
1638  - ``clipSpans``: List of dictionaries containing spanSet lists
1639  to clip. Each element contains the new maskplane name
1640  ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1641  - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1642  compressed into footprints.
1643  """
1644  mask = exp.getMaskedImage().getMask()
1645  maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1646  fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1647  # Merge positive and negative together footprints together
1648  fpSet.positive.merge(fpSet.negative)
1649  footprints = fpSet.positive
1650  self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1651  ignoreMask = self.getBadPixelMask()
1652 
1653  clipFootprints = []
1654  clipIndices = []
1655  artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1656 
1657  # for use by detectClipBig
1658  visitDetectionFootprints = []
1659 
1660  dims = [len(tempExpRefList), len(footprints.getFootprints())]
1661  overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1662  ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1663 
1664  # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1665  for i, warpRef in enumerate(tempExpRefList):
1666  tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1667  immediate=True).getMaskedImage().getMask()
1668  maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1669  afwImage.PARENT, True)
1670  maskVisitDet &= maskDetValue
1671  visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1672  visitDetectionFootprints.append(visitFootprints)
1673 
1674  for j, footprint in enumerate(footprints.getFootprints()):
1675  ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1676  overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1677 
1678  # build a list of clipped spans for each visit
1679  for j, footprint in enumerate(footprints.getFootprints()):
1680  nPixel = footprint.getArea()
1681  overlap = [] # hold the overlap with each visit
1682  indexList = [] # index of visit in global list
1683  for i in range(len(tempExpRefList)):
1684  ignore = ignoreArr[i, j]
1685  overlapDet = overlapDetArr[i, j]
1686  totPixel = nPixel - ignore
1687 
1688  # If we have more bad pixels than detection skip
1689  if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1690  continue
1691  overlap.append(overlapDet/float(totPixel))
1692  indexList.append(i)
1693 
1694  overlap = numpy.array(overlap)
1695  if not len(overlap):
1696  continue
1697 
1698  keep = False # Should this footprint be marked as clipped?
1699  keepIndex = [] # Which tempExps does the clipped footprint belong to
1700 
1701  # If footprint only has one overlap use a lower threshold
1702  if len(overlap) == 1:
1703  if overlap[0] > self.config.minClipFootOverlapSingle:
1704  keep = True
1705  keepIndex = [0]
1706  else:
1707  # This is the general case where only visit should be clipped
1708  clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1709  if len(clipIndex) == 1:
1710  keep = True
1711  keepIndex = [clipIndex[0]]
1712 
1713  # Test if there are clipped objects that overlap two different visits
1714  clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1715  if len(clipIndex) == 2 and len(overlap) > 3:
1716  clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1717  if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1718  keep = True
1719  keepIndex = clipIndex
1720 
1721  if not keep:
1722  continue
1723 
1724  for index in keepIndex:
1725  globalIndex = indexList[index]
1726  artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1727 
1728  clipIndices.append(numpy.array(indexList)[keepIndex])
1729  clipFootprints.append(footprint)
1730 
1731  return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1732  clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1733 
1734  def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1735  maskClipValue, maskDetValue, coaddBBox):
1736  """Return individual warp footprints for large artifacts and append
1737  them to ``clipList`` in place.
1738 
1739  Identify big footprints composed of many sources in the coadd
1740  difference that may have originated in a large diffuse source in the
1741  coadd. We do this by indentifying all clipped footprints that overlap
1742  significantly with each source in all the coaddTempExps.
1743 
1744  Parameters
1745  ----------
1746  clipList : `list`
1747  List of alt mask SpanSets with clipping information. Modified.
1748  clipFootprints : `list`
1749  List of clipped footprints.
1750  clipIndices : `list`
1751  List of which entries in tempExpClipList each footprint belongs to.
1752  maskClipValue
1753  Mask value of clipped pixels.
1754  maskDetValue
1755  Mask value of detected pixels.
1756  coaddBBox : `lsst.geom.Box`
1757  BBox of the coadd and warps.
1758 
1759  Returns
1760  -------
1761  bigFootprintsCoadd : `list`
1762  List of big footprints
1763  """
1764  bigFootprintsCoadd = []
1765  ignoreMask = self.getBadPixelMask()
1766  for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1767  maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1768  for footprint in visitFootprints.getFootprints():
1769  footprint.spans.setMask(maskVisitDet, maskDetValue)
1770 
1771  # build a mask of clipped footprints that are in this visit
1772  clippedFootprintsVisit = []
1773  for foot, clipIndex in zip(clipFootprints, clipIndices):
1774  if index not in clipIndex:
1775  continue
1776  clippedFootprintsVisit.append(foot)
1777  maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1778  afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1779 
1780  bigFootprintsVisit = []
1781  for foot in visitFootprints.getFootprints():
1782  if foot.getArea() < self.config.minBigOverlap:
1783  continue
1784  nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1785  if nCount > self.config.minBigOverlap:
1786  bigFootprintsVisit.append(foot)
1787  bigFootprintsCoadd.append(foot)
1788 
1789  for footprint in bigFootprintsVisit:
1790  clippedSpans["CLIPPED"].append(footprint.spans)
1791 
1792  return bigFootprintsCoadd
1793 
1794 
1796  psfMatchedWarps = pipeBase.connectionTypes.Input(
1797  doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1798  "Only PSF-Matched Warps make sense for image subtraction. "
1799  "Therefore, they must be an additional declared input."),
1800  name="{inputCoaddName}Coadd_psfMatchedWarp",
1801  storageClass="ExposureF",
1802  dimensions=("tract", "patch", "skymap", "visit"),
1803  deferLoad=True,
1804  multiple=True
1805  )
1806  templateCoadd = pipeBase.connectionTypes.Output(
1807  doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1808  "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1809  name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1810  storageClass="ExposureF",
1811  dimensions=("tract", "patch", "skymap", "band"),
1812  )
1813 
1814  def __init__(self, *, config=None):
1815  super().__init__(config=config)
1816  if not config.assembleStaticSkyModel.doWrite:
1817  self.outputs.remove("templateCoadd")
1818  config.validate()
1819 
1820 
1821 class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1822  pipelineConnections=CompareWarpAssembleCoaddConnections):
1823  assembleStaticSkyModel = pexConfig.ConfigurableField(
1824  target=AssembleCoaddTask,
1825  doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1826  " naive/first-iteration model of the static sky.",
1827  )
1828  detect = pexConfig.ConfigurableField(
1829  target=SourceDetectionTask,
1830  doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1831  )
1832  detectTemplate = pexConfig.ConfigurableField(
1833  target=SourceDetectionTask,
1834  doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1835  )
1836  maskStreaks = pexConfig.ConfigurableField(
1837  target=MaskStreaksTask,
1838  doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1839  "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1840  "streakMaskName"
1841  )
1842  streakMaskName = pexConfig.Field(
1843  dtype=str,
1844  default="STREAK",
1845  doc="Name of mask bit used for streaks"
1846  )
1847  maxNumEpochs = pexConfig.Field(
1848  doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1849  "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1850  "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1851  "For each footprint detected on the image difference between the psfMatched warp and static sky "
1852  "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1853  "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1854  "than transient and not masked.",
1855  dtype=int,
1856  default=2
1857  )
1858  maxFractionEpochsLow = pexConfig.RangeField(
1859  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1860  "Effective maxNumEpochs = "
1861  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1862  dtype=float,
1863  default=0.4,
1864  min=0., max=1.,
1865  )
1866  maxFractionEpochsHigh = pexConfig.RangeField(
1867  doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1868  "Effective maxNumEpochs = "
1869  "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1870  dtype=float,
1871  default=0.03,
1872  min=0., max=1.,
1873  )
1874  spatialThreshold = pexConfig.RangeField(
1875  doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1876  "temporal criteria. If 0, clip all. If 1, clip none.",
1877  dtype=float,
1878  default=0.5,
1879  min=0., max=1.,
1880  inclusiveMin=True, inclusiveMax=True
1881  )
1882  doScaleWarpVariance = pexConfig.Field(
1883  doc="Rescale Warp variance plane using empirical noise?",
1884  dtype=bool,
1885  default=True,
1886  )
1887  scaleWarpVariance = pexConfig.ConfigurableField(
1888  target=ScaleVarianceTask,
1889  doc="Rescale variance on warps",
1890  )
1891  doPreserveContainedBySource = pexConfig.Field(
1892  doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1893  "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1894  dtype=bool,
1895  default=True,
1896  )
1897  doPrefilterArtifacts = pexConfig.Field(
1898  doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1899  "because they will be excluded anyway. This prevents them from contributing "
1900  "to the outlier epoch count image and potentially being labeled as persistant."
1901  "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1902  dtype=bool,
1903  default=True
1904  )
1905  prefilterArtifactsMaskPlanes = pexConfig.ListField(
1906  doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1907  dtype=str,
1908  default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1909  )
1910  prefilterArtifactsRatio = pexConfig.Field(
1911  doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1912  dtype=float,
1913  default=0.05
1914  )
1915  doFilterMorphological = pexConfig.Field(
1916  doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1917  "be streaks.",
1918  dtype=bool,
1919  default=False
1920  )
1921 
1922  def setDefaults(self):
1923  AssembleCoaddConfig.setDefaults(self)
1924  self.statisticstatistic = 'MEAN'
1925  self.doUsePsfMatchedPolygonsdoUsePsfMatchedPolygons = True
1926 
1927  # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1928  # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1929  if "EDGE" in self.badMaskPlanes:
1930  self.badMaskPlanes.remove('EDGE')
1931  self.removeMaskPlanes.append('EDGE')
1932  self.assembleStaticSkyModelassembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1933  self.assembleStaticSkyModelassembleStaticSkyModel.warpType = 'psfMatched'
1934  self.assembleStaticSkyModelassembleStaticSkyModel.connections.warpType = 'psfMatched'
1935  self.assembleStaticSkyModelassembleStaticSkyModel.statistic = 'MEANCLIP'
1936  self.assembleStaticSkyModelassembleStaticSkyModel.sigmaClip = 2.5
1937  self.assembleStaticSkyModelassembleStaticSkyModel.clipIter = 3
1938  self.assembleStaticSkyModelassembleStaticSkyModel.calcErrorFromInputVariance = False
1939  self.assembleStaticSkyModelassembleStaticSkyModel.doWrite = False
1940  self.detectdetect.doTempLocalBackground = False
1941  self.detectdetect.reEstimateBackground = False
1942  self.detectdetect.returnOriginalFootprints = False
1943  self.detectdetect.thresholdPolarity = "both"
1944  self.detectdetect.thresholdValue = 5
1945  self.detectdetect.minPixels = 4
1946  self.detectdetect.isotropicGrow = True
1947  self.detectdetect.thresholdType = "pixel_stdev"
1948  self.detectdetect.nSigmaToGrow = 0.4
1949  # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1950  # Explicitly restating because ratio with detect.nSigmaToGrow matters
1951  self.detectTemplatedetectTemplate.nSigmaToGrow = 2.4
1952  self.detectTemplatedetectTemplate.doTempLocalBackground = False
1953  self.detectTemplatedetectTemplate.reEstimateBackground = False
1954  self.detectTemplatedetectTemplate.returnOriginalFootprints = False
1955 
1956  def validate(self):
1957  super().validate()
1958  if self.assembleStaticSkyModelassembleStaticSkyModel.doNImage:
1959  raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1960  "Please set assembleStaticSkyModel.doNImage=False")
1961 
1962  if self.assembleStaticSkyModelassembleStaticSkyModel.doWrite and (self.warpTypewarpType == self.assembleStaticSkyModelassembleStaticSkyModel.warpType):
1963  raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1964  "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1965  "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1966  "'PsfMatched'" % (self.warpTypewarpType, self.assembleStaticSkyModelassembleStaticSkyModel.warpType))
1967 
1968 
1969 class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1970  """Assemble a compareWarp coadded image from a set of warps
1971  by masking artifacts detected by comparing PSF-matched warps.
1972 
1973  In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1974  we clip outliers). The problem with doing this is that when computing the
1975  coadd PSF at a given location, individual visit PSFs from visits with
1976  outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1977  In this task, we correct for this behavior by creating a new badMaskPlane
1978  'CLIPPED' which marks pixels in the individual warps suspected to contain
1979  an artifact. We populate this plane on the input warps by comparing
1980  PSF-matched warps with a PSF-matched median coadd which serves as a
1981  model of the static sky. Any group of pixels that deviates from the
1982  PSF-matched template coadd by more than config.detect.threshold sigma,
1983  is an artifact candidate. The candidates are then filtered to remove
1984  variable sources and sources that are difficult to subtract such as
1985  bright stars. This filter is configured using the config parameters
1986  ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1987  the maximum fraction of epochs that the deviation can appear in and still
1988  be considered an artifact. The spatialThreshold is the maximum fraction of
1989  pixels in the footprint of the deviation that appear in other epochs
1990  (where other epochs is defined by the temporalThreshold). If the deviant
1991  region meets this criteria of having a significant percentage of pixels
1992  that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1993  set in the mask. These regions will not contribute to the final coadd.
1994  Furthermore, any routine to determine the coadd PSF can now be cognizant
1995  of clipped regions. Note that the algorithm implemented by this task is
1996  preliminary and works correctly for HSC data. Parameter modifications and
1997  or considerable redesigning of the algorithm is likley required for other
1998  surveys.
1999 
2000  ``CompareWarpAssembleCoaddTask`` sub-classes
2001  ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
2002  as a subtask to generate the TemplateCoadd (the model of the static sky).
2003 
2004  Notes
2005  -----
2006  The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2007  flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2008  ``baseDebug`` for more about ``debug.py`` files.
2009 
2010  This task supports the following debug variables:
2011 
2012  - ``saveCountIm``
2013  If True then save the Epoch Count Image as a fits file in the `figPath`
2014  - ``figPath``
2015  Path to save the debug fits images and figures
2016 
2017  For example, put something like:
2018 
2019  .. code-block:: python
2020 
2021  import lsstDebug
2022  def DebugInfo(name):
2023  di = lsstDebug.getInfo(name)
2024  if name == "lsst.pipe.tasks.assembleCoadd":
2025  di.saveCountIm = True
2026  di.figPath = "/desired/path/to/debugging/output/images"
2027  return di
2028  lsstDebug.Info = DebugInfo
2029 
2030  into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2031  ``--debug`` flag. Some subtasks may have their own debug variables;
2032  see individual Task documentation.
2033 
2034  Examples
2035  --------
2036  ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2037  coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2038  ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2039  Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2040  and filter to be coadded (specified using
2041  '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2042  along with a list of coaddTempExps to attempt to coadd (specified using
2043  '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2044  Only the warps that cover the specified tract and patch will be coadded.
2045  A list of the available optional arguments can be obtained by calling
2046  ``assembleCoadd.py`` with the ``--help`` command line argument:
2047 
2048  .. code-block:: none
2049 
2050  assembleCoadd.py --help
2051 
2052  To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2053  context of multi-band processing, we will generate the HSC-I & -R band
2054  oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2055  To begin, assuming that the lsst stack has been already set up, we must
2056  set up the ``obs_subaru`` and ``ci_hsc`` packages.
2057  This defines the environment variable ``$CI_HSC_DIR`` and points at the
2058  location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2059  directory. To begin assembling the coadds, we must first
2060 
2061  - processCcd
2062  process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2063  - makeSkyMap
2064  create a skymap that covers the area of the sky present in the raw exposures
2065  - makeCoaddTempExp
2066  warp the individual calibrated exposures to the tangent plane of the coadd
2067 
2068  We can perform all of these steps by running
2069 
2070  .. code-block:: none
2071 
2072  $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2073 
2074  This will produce warped ``coaddTempExps`` for each visit. To coadd the
2075  warped data, we call ``assembleCoadd.py`` as follows:
2076 
2077  .. code-block:: none
2078 
2079  assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2080  --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2081  --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2082  --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2083  --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2084  --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2085  --selectId visit=903988 ccd=24
2086 
2087  This will process the HSC-I band data. The results are written in
2088  ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2089  """
2090  ConfigClass = CompareWarpAssembleCoaddConfig
2091  _DefaultName = "compareWarpAssembleCoadd"
2092 
2093  def __init__(self, *args, **kwargs):
2094  AssembleCoaddTask.__init__(self, *args, **kwargs)
2095  self.makeSubtask("assembleStaticSkyModel")
2096  detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2097  self.makeSubtask("detect", schema=detectionSchema)
2098  if self.config.doPreserveContainedBySource:
2099  self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2100  if self.config.doScaleWarpVariance:
2101  self.makeSubtask("scaleWarpVariance")
2102  if self.config.doFilterMorphological:
2103  self.makeSubtask("maskStreaks")
2104 
2105  @utils.inheritDoc(AssembleCoaddTask)
2106  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2107  """
2108  Generate a templateCoadd to use as a naive model of static sky to
2109  subtract from PSF-Matched warps.
2110 
2111  Returns
2112  -------
2113  result : `lsst.pipe.base.Struct`
2114  Result struct with components:
2115 
2116  - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2117  - ``nImage`` : N Image (``lsst.afw.image.Image``)
2118  """
2119  # Ensure that psfMatchedWarps are used as input warps for template generation
2120  staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2121  staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2122 
2123  # Because subtasks don't have connections we have to make one.
2124  # The main task's `templateCoadd` is the subtask's `coaddExposure`
2125  staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2126  if self.config.assembleStaticSkyModel.doWrite:
2127  staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2128  # Remove template coadd from both subtask's and main tasks outputs,
2129  # because it is handled by the subtask as `coaddExposure`
2130  del outputRefs.templateCoadd
2131  del staticSkyModelOutputRefs.templateCoadd
2132 
2133  # A PSF-Matched nImage does not exist as a dataset type
2134  if 'nImage' in staticSkyModelOutputRefs.keys():
2135  del staticSkyModelOutputRefs.nImage
2136 
2137  templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2138  staticSkyModelOutputRefs)
2139  if templateCoadd is None:
2140  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2141 
2142  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2143  nImage=templateCoadd.nImage,
2144  warpRefList=templateCoadd.warpRefList,
2145  imageScalerList=templateCoadd.imageScalerList,
2146  weightList=templateCoadd.weightList)
2147 
2148  @utils.inheritDoc(AssembleCoaddTask)
2149  def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2150  """
2151  Generate a templateCoadd to use as a naive model of static sky to
2152  subtract from PSF-Matched warps.
2153 
2154  Returns
2155  -------
2156  result : `lsst.pipe.base.Struct`
2157  Result struct with components:
2158 
2159  - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2160  - ``nImage``: N Image (``lsst.afw.image.Image``)
2161  """
2162  templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2163  if templateCoadd is None:
2164  raise RuntimeError(self._noTemplateMessage_noTemplateMessage(self.assembleStaticSkyModel.warpType))
2165 
2166  return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2167  nImage=templateCoadd.nImage,
2168  warpRefList=templateCoadd.warpRefList,
2169  imageScalerList=templateCoadd.imageScalerList,
2170  weightList=templateCoadd.weightList)
2171 
2172  def _noTemplateMessage(self, warpType):
2173  warpName = (warpType[0].upper() + warpType[1:])
2174  message = """No %(warpName)s warps were found to build the template coadd which is
2175  required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2176  first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2177  coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2178 
2179  Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2180  another algorithm like:
2181 
2182  from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2183  config.assemble.retarget(SafeClipAssembleCoaddTask)
2184  """ % {"warpName": warpName}
2185  return message
2186 
2187  @utils.inheritDoc(AssembleCoaddTask)
2188  @pipeBase.timeMethod
2189  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2190  supplementaryData, *args, **kwargs):
2191  """Assemble the coadd.
2192 
2193  Find artifacts and apply them to the warps' masks creating a list of
2194  alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2195  plane. Then pass these alternative masks to the base class's `run`
2196  method.
2197 
2198  The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2199  that must contain a ``templateCoadd`` that serves as the
2200  model of the static sky.
2201  """
2202 
2203  # Check and match the order of the supplementaryData
2204  # (PSF-matched) inputs to the order of the direct inputs,
2205  # so that the artifact mask is applied to the right warp
2206  dataIds = [ref.dataId for ref in tempExpRefList]
2207  psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2208 
2209  if dataIds != psfMatchedDataIds:
2210  self.log.info("Reordering and or/padding PSF-matched visit input list")
2211  supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2212  psfMatchedDataIds, dataIds)
2213  supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2214  psfMatchedDataIds, dataIds)
2215 
2216  # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2217  spanSetMaskList = self.findArtifactsfindArtifacts(supplementaryData.templateCoadd,
2218  supplementaryData.warpRefList,
2219  supplementaryData.imageScalerList)
2220 
2221  badMaskPlanes = self.config.badMaskPlanes[:]
2222  badMaskPlanes.append("CLIPPED")
2223  badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2224 
2225  result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2226  spanSetMaskList, mask=badPixelMask)
2227 
2228  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2229  # Psf-Matching moves the real edge inwards
2230  self.applyAltEdgeMaskapplyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2231  return result
2232 
2233  def applyAltEdgeMask(self, mask, altMaskList):
2234  """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2235 
2236  Parameters
2237  ----------
2238  mask : `lsst.afw.image.Mask`
2239  Original mask.
2240  altMaskList : `list`
2241  List of Dicts containing ``spanSet`` lists.
2242  Each element contains the new mask plane name (e.g. "CLIPPED
2243  and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2244  the mask.
2245  """
2246  maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2247  for visitMask in altMaskList:
2248  if "EDGE" in visitMask:
2249  for spanSet in visitMask['EDGE']:
2250  spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2251 
2252  def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2253  """Find artifacts.
2254 
2255  Loop through warps twice. The first loop builds a map with the count
2256  of how many epochs each pixel deviates from the templateCoadd by more
2257  than ``config.chiThreshold`` sigma. The second loop takes each
2258  difference image and filters the artifacts detected in each using
2259  count map to filter out variable sources and sources that are
2260  difficult to subtract cleanly.
2261 
2262  Parameters
2263  ----------
2264  templateCoadd : `lsst.afw.image.Exposure`
2265  Exposure to serve as model of static sky.
2266  tempExpRefList : `list`
2267  List of data references to warps.
2268  imageScalerList : `list`
2269  List of image scalers.
2270 
2271  Returns
2272  -------
2273  altMasks : `list`
2274  List of dicts containing information about CLIPPED
2275  (i.e., artifacts), NO_DATA, and EDGE pixels.
2276  """
2277 
2278  self.log.debug("Generating Count Image, and mask lists.")
2279  coaddBBox = templateCoadd.getBBox()
2280  slateIm = afwImage.ImageU(coaddBBox)
2281  epochCountImage = afwImage.ImageU(coaddBBox)
2282  nImage = afwImage.ImageU(coaddBBox)
2283  spanSetArtifactList = []
2284  spanSetNoDataMaskList = []
2285  spanSetEdgeList = []
2286  spanSetBadMorphoList = []
2287  badPixelMask = self.getBadPixelMask()
2288 
2289  # mask of the warp diffs should = that of only the warp
2290  templateCoadd.mask.clearAllMaskPlanes()
2291 
2292  if self.config.doPreserveContainedBySource:
2293  templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2294  else:
2295  templateFootprints = None
2296 
2297  for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2298  warpDiffExp = self._readAndComputeWarpDiff_readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2299  if warpDiffExp is not None:
2300  # This nImage only approximates the final nImage because it uses the PSF-matched mask
2301  nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2302  * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2303  fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2304  fpSet.positive.merge(fpSet.negative)
2305  footprints = fpSet.positive
2306  slateIm.set(0)
2307  spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2308 
2309  # Remove artifacts due to defects before they contribute to the epochCountImage
2310  if self.config.doPrefilterArtifacts:
2311  spanSetList = self.prefilterArtifactsprefilterArtifacts(spanSetList, warpDiffExp)
2312 
2313  # Clear mask before adding prefiltered spanSets
2314  self.detect.clearMask(warpDiffExp.mask)
2315  for spans in spanSetList:
2316  spans.setImage(slateIm, 1, doClip=True)
2317  spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2318  epochCountImage += slateIm
2319 
2320  if self.config.doFilterMorphological:
2321  maskName = self.config.streakMaskName
2322  _ = self.maskStreaks.run(warpDiffExp)
2323  streakMask = warpDiffExp.mask
2324  spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2325  streakMask.getPlaneBitMask(maskName)).split()
2326 
2327  # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2328  # undergo a second convolution. Pixels with data in the direct warp
2329  # but not in the PSF-matched warp will not have their artifacts detected.
2330  # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2331  nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2332  nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2333  nansMask.setXY0(warpDiffExp.getXY0())
2334  edgeMask = warpDiffExp.mask
2335  spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2336  edgeMask.getPlaneBitMask("EDGE")).split()
2337  else:
2338  # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2339  # In this case, mask the whole epoch
2340  nansMask = afwImage.MaskX(coaddBBox, 1)
2341  spanSetList = []
2342  spanSetEdgeMask = []
2343  spanSetStreak = []
2344 
2345  spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2346 
2347  spanSetNoDataMaskList.append(spanSetNoDataMask)
2348  spanSetArtifactList.append(spanSetList)
2349  spanSetEdgeList.append(spanSetEdgeMask)
2350  if self.config.doFilterMorphological:
2351  spanSetBadMorphoList.append(spanSetStreak)
2352 
2353  if lsstDebug.Info(__name__).saveCountIm:
2354  path = self._dataRef2DebugPath_dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2355  epochCountImage.writeFits(path)
2356 
2357  for i, spanSetList in enumerate(spanSetArtifactList):
2358  if spanSetList:
2359  filteredSpanSetList = self.filterArtifactsfilterArtifacts(spanSetList, epochCountImage, nImage,
2360  templateFootprints)
2361  spanSetArtifactList[i] = filteredSpanSetList
2362  if self.config.doFilterMorphological:
2363  spanSetArtifactList[i] += spanSetBadMorphoList[i]
2364 
2365  altMasks = []
2366  for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2367  altMasks.append({'CLIPPED': artifacts,
2368  'NO_DATA': noData,
2369  'EDGE': edge})
2370  return altMasks
2371 
2372  def prefilterArtifacts(self, spanSetList, exp):
2373  """Remove artifact candidates covered by bad mask plane.
2374 
2375  Any future editing of the candidate list that does not depend on
2376  temporal information should go in this method.
2377 
2378  Parameters
2379  ----------
2380  spanSetList : `list`
2381  List of SpanSets representing artifact candidates.
2382  exp : `lsst.afw.image.Exposure`
2383  Exposure containing mask planes used to prefilter.
2384 
2385  Returns
2386  -------
2387  returnSpanSetList : `list`
2388  List of SpanSets with artifacts.
2389  """
2390  badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2391  goodArr = (exp.mask.array & badPixelMask) == 0
2392  returnSpanSetList = []
2393  bbox = exp.getBBox()
2394  x0, y0 = exp.getXY0()
2395  for i, span in enumerate(spanSetList):
2396  y, x = span.clippedTo(bbox).indices()
2397  yIndexLocal = numpy.array(y) - y0
2398  xIndexLocal = numpy.array(x) - x0
2399  goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2400  if goodRatio > self.config.prefilterArtifactsRatio:
2401  returnSpanSetList.append(span)
2402  return returnSpanSetList
2403 
2404  def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2405  """Filter artifact candidates.
2406 
2407  Parameters
2408  ----------
2409  spanSetList : `list`
2410  List of SpanSets representing artifact candidates.
2411  epochCountImage : `lsst.afw.image.Image`
2412  Image of accumulated number of warpDiff detections.
2413  nImage : `lsst.afw.image.Image`
2414  Image of the accumulated number of total epochs contributing.
2415 
2416  Returns
2417  -------
2418  maskSpanSetList : `list`
2419  List of SpanSets with artifacts.
2420  """
2421 
2422  maskSpanSetList = []
2423  x0, y0 = epochCountImage.getXY0()
2424  for i, span in enumerate(spanSetList):
2425  y, x = span.indices()
2426  yIdxLocal = [y1 - y0 for y1 in y]
2427  xIdxLocal = [x1 - x0 for x1 in x]
2428  outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2429  totalN = nImage.array[yIdxLocal, xIdxLocal]
2430 
2431  # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2432  effMaxNumEpochsHighN = (self.config.maxNumEpochs
2433  + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2434  effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2435  effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2436  nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2437  & (outlierN <= effectiveMaxNumEpochs))
2438  percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2439  if percentBelowThreshold > self.config.spatialThreshold:
2440  maskSpanSetList.append(span)
2441 
2442  if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2443  # If a candidate is contained by a footprint on the template coadd, do not clip
2444  filteredMaskSpanSetList = []
2445  for span in maskSpanSetList:
2446  doKeep = True
2447  for footprint in footprintsToExclude.positive.getFootprints():
2448  if footprint.spans.contains(span):
2449  doKeep = False
2450  break
2451  if doKeep:
2452  filteredMaskSpanSetList.append(span)
2453  maskSpanSetList = filteredMaskSpanSetList
2454 
2455  return maskSpanSetList
2456 
2457  def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2458  """Fetch a warp from the butler and return a warpDiff.
2459 
2460  Parameters
2461  ----------
2462  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2463  Butler dataRef for the warp.
2464  imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2465  An image scaler object.
2466  templateCoadd : `lsst.afw.image.Exposure`
2467  Exposure to be substracted from the scaled warp.
2468 
2469  Returns
2470  -------
2471  warp : `lsst.afw.image.Exposure`
2472  Exposure of the image difference between the warp and template.
2473  """
2474 
2475  # If the PSF-Matched warp did not exist for this direct warp
2476  # None is holding its place to maintain order in Gen 3
2477  if warpRef is None:
2478  return None
2479  # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2480  warpName = self.getTempExpDatasetName('psfMatched')
2481  if not isinstance(warpRef, DeferredDatasetHandle):
2482  if not warpRef.datasetExists(warpName):
2483  self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2484  return None
2485  warp = warpRef.get(datasetType=warpName, immediate=True)
2486  # direct image scaler OK for PSF-matched Warp
2487  imageScaler.scaleMaskedImage(warp.getMaskedImage())
2488  mi = warp.getMaskedImage()
2489  if self.config.doScaleWarpVariance:
2490  try:
2491  self.scaleWarpVariance.run(mi)
2492  except Exception as exc:
2493  self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2494  mi -= templateCoadd.getMaskedImage()
2495  return warp
2496 
2497  def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2498  """Return a path to which to write debugging output.
2499 
2500  Creates a hyphen-delimited string of dataId values for simple filenames.
2501 
2502  Parameters
2503  ----------
2504  prefix : `str`
2505  Prefix for filename.
2506  warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2507  Butler dataRef to make the path from.
2508  coaddLevel : `bool`, optional.
2509  If True, include only coadd-level keys (e.g., 'tract', 'patch',
2510  'filter', but no 'visit').
2511 
2512  Returns
2513  -------
2514  result : `str`
2515  Path for debugging output.
2516  """
2517  if coaddLevel:
2518  keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2519  else:
2520  keys = warpRef.dataId.keys()
2521  keyList = sorted(keys, reverse=True)
2522  directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2523  filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2524  return os.path.join(directory, filename)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, *args, **kwargs)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs)
Base class for coaddition.
Definition: coaddBase.py:141
def prepareStats(self, mask=None)
def readBrightObjectMasks(self, dataRef)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def removeMaskPlanes(self, maskedImage)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def filterWarps(self, inputs, goodVisits)
def prepareInputs(self, refList)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeCoaddSuffix(warpType="direct")
Definition: coaddBase.py:346
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60